You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/10/03 17:06:10 UTC
[01/35] hadoop git commit: HADOOP-14822. hadoop-project/pom.xml is
executable. Contributed by Ajay Kumar.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 579434a83 -> 75ce3fcc7
HADOOP-14822. hadoop-project/pom.xml is executable. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd378775
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd378775
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd378775
Branch: refs/heads/HDFS-7240
Commit: dd378775df5bd79e23c12120535404c4ca0fb386
Parents: 0da29cb
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Sep 27 14:25:07 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Sep 27 14:25:07 2017 +0900
----------------------------------------------------------------------
hadoop-project/pom.xml | 0
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd378775/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
old mode 100755
new mode 100644
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/35] hadoop git commit: YARN-7259. Add size-based rolling policy
to LogAggregationIndexedFileController. (xgong via wangda)
Posted by ae...@apache.org.
YARN-7259. Add size-based rolling policy to LogAggregationIndexedFileController. (xgong via wangda)
Change-Id: Ifaf82c0aee6b73b9b6ebf103aa72e131e3942f31
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/280080fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/280080fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/280080fa
Branch: refs/heads/HDFS-7240
Commit: 280080fad01304c85a9ede4d4f7b707eb36c0155
Parents: 2c62ff7
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Oct 2 15:30:22 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Oct 2 15:30:22 2017 -0700
----------------------------------------------------------------------
.../ifile/IndexedFileAggregatedLogsBlock.java | 14 +-
.../LogAggregationIndexedFileController.java | 397 +++++++++++++------
.../TestLogAggregationIndexFileController.java | 67 +++-
3 files changed, 340 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/280080fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
index c4cbfda..5439b53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
@@ -101,10 +101,9 @@ public class IndexedFileAggregatedLogsBlock extends LogAggregationHtmlBlock {
return;
}
- Map<String, FileStatus> checkSumFiles;
+ Map<String, Long> checkSumFiles;
try {
- checkSumFiles = fileController.filterFiles(nodeFiles,
- LogAggregationIndexedFileController.CHECK_SUM_FILE_SUFFIX);
+ checkSumFiles = fileController.parseCheckSumFiles(nodeFiles);
} catch (IOException ex) {
LOG.error("Error getting logs for " + logEntity, ex);
html.h1("Error getting logs for " + logEntity);
@@ -125,12 +124,11 @@ public class IndexedFileAggregatedLogsBlock extends LogAggregationHtmlBlock {
String desiredLogType = $(CONTAINER_LOG_TYPE);
try {
for (FileStatus thisNodeFile : fileToRead) {
- FileStatus checkSum = fileController.getAllChecksumFiles(
- checkSumFiles, thisNodeFile.getPath().getName());
+ Long checkSumIndex = checkSumFiles.get(
+ thisNodeFile.getPath().getName());
long endIndex = -1;
- if (checkSum != null) {
- endIndex = fileController.loadIndexedLogsCheckSum(
- checkSum.getPath());
+ if (checkSumIndex != null) {
+ endIndex = checkSumIndex.longValue();
}
IndexedLogsMeta indexedLogsMeta = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/280080fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
index 243945e..800c0a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
@@ -29,6 +29,8 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.nio.charset.Charset;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
@@ -41,7 +43,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import java.util.UUID;
import org.apache.commons.lang.SerializationUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -54,6 +55,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.HarFs;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SecureIOUtils;
@@ -77,6 +79,8 @@ import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.View.ViewContext;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block;
@@ -102,7 +106,8 @@ public class LogAggregationIndexedFileController
"indexedFile.fs.op.num-retries";
private static final String FS_RETRY_INTERVAL_MS_ATTR =
"indexedFile.fs.retry-interval-ms";
- private static final int UUID_LENGTH = 36;
+ private static final String LOG_ROLL_OVER_MAX_FILE_SIZE_GB =
+ "indexedFile.log.roll-over.max-file-size-gb";
@VisibleForTesting
public static final String CHECK_SUM_FILE_SUFFIX = "-checksum";
@@ -121,7 +126,10 @@ public class LogAggregationIndexedFileController
private Path remoteLogCheckSumFile;
private FileContext fc;
private UserGroupInformation ugi;
- private String uuid = null;
+ private byte[] uuid = null;
+ private final int UUID_LENGTH = 32;
+ private long logRollOverMaxFileSize;
+ private Clock sysClock;
public LogAggregationIndexedFileController() {}
@@ -164,6 +172,8 @@ public class LogAggregationIndexedFileController
compressName);
this.fsNumRetries = conf.getInt(FS_NUM_RETRIES_ATTR, 3);
this.fsRetryInterval = conf.getLong(FS_RETRY_INTERVAL_MS_ATTR, 1000L);
+ this.logRollOverMaxFileSize = getRollOverLogMaxSize(conf);
+ this.sysClock = getSystemClock();
}
@Override
@@ -173,11 +183,12 @@ public class LogAggregationIndexedFileController
final UserGroupInformation userUgi = context.getUserUgi();
final Map<ApplicationAccessType, String> appAcls = context.getAppAcls();
final String nodeId = context.getNodeId().toString();
+ final ApplicationId appId = context.getAppId();
final Path remoteLogFile = context.getRemoteNodeLogFileForApp();
this.ugi = userUgi;
logAggregationSuccessfullyInThisCyCle = false;
logsMetaInThisCycle = new IndexedPerAggregationLogMeta();
- logAggregationTimeInThisCycle = System.currentTimeMillis();
+ logAggregationTimeInThisCycle = this.sysClock.getTime();
logsMetaInThisCycle.setUploadTimeStamp(logAggregationTimeInThisCycle);
logsMetaInThisCycle.setRemoteNodeFile(remoteLogFile.getName());
try {
@@ -187,57 +198,6 @@ public class LogAggregationIndexedFileController
fc = FileContext.getFileContext(
remoteRootLogDir.toUri(), conf);
fc.setUMask(APP_LOG_FILE_UMASK);
- boolean fileExist = fc.util().exists(remoteLogFile);
- if (fileExist && context.isLogAggregationInRolling()) {
- fsDataOStream = fc.create(remoteLogFile,
- EnumSet.of(CreateFlag.APPEND),
- new Options.CreateOpts[] {});
- if (uuid == null) {
- FSDataInputStream fsDataInputStream = null;
- try {
- fsDataInputStream = fc.open(remoteLogFile);
- byte[] b = new byte[UUID_LENGTH];
- int actual = fsDataInputStream.read(b);
- if (actual != UUID_LENGTH) {
- // Get an error when parse the UUID from existed log file.
- // Simply OverWrite the existed log file and re-create the
- // UUID.
- fsDataOStream = fc.create(remoteLogFile,
- EnumSet.of(CreateFlag.OVERWRITE),
- new Options.CreateOpts[] {});
- uuid = UUID.randomUUID().toString();
- fsDataOStream.write(uuid.getBytes(Charset.forName("UTF-8")));
- fsDataOStream.flush();
- } else {
- uuid = new String(b, Charset.forName("UTF-8"));
- }
- } finally {
- IOUtils.cleanupWithLogger(LOG, fsDataInputStream);
- }
- }
- // if the remote log file exists, but we do not have any
- // indexedLogsMeta. We need to re-load indexedLogsMeta from
- // the existing remote log file. If the re-load fails, we simply
- // re-create a new indexedLogsMeta object. And will re-load
- // the indexedLogsMeta from checksum file later.
- if (indexedLogsMeta == null) {
- try {
- indexedLogsMeta = loadIndexedLogsMeta(remoteLogFile);
- } catch (IOException ex) {
- // DO NOTHING
- }
- }
- } else {
- fsDataOStream = fc.create(remoteLogFile,
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
- new Options.CreateOpts[] {});
- if (uuid == null) {
- uuid = UUID.randomUUID().toString();
- }
- byte[] b = uuid.getBytes(Charset.forName("UTF-8"));
- fsDataOStream.write(b);
- fsDataOStream.flush();
- }
if (indexedLogsMeta == null) {
indexedLogsMeta = new IndexedLogsMeta();
indexedLogsMeta.setVersion(VERSION);
@@ -249,44 +209,24 @@ public class LogAggregationIndexedFileController
YarnConfiguration.DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE);
indexedLogsMeta.setCompressName(compressName);
}
- final long currentAggregatedLogFileLength = fc
- .getFileStatus(remoteLogFile).getLen();
- // only check the check-sum file when we are in append mode
+ Path aggregatedLogFile = null;
if (context.isLogAggregationInRolling()) {
- // check whether the checksum file exists to figure out
- // whether the previous log aggregation process is successful
- // and the aggregated log file is corrupted or not.
- remoteLogCheckSumFile = new Path(remoteLogFile.getParent(),
- (remoteLogFile.getName() + CHECK_SUM_FILE_SUFFIX));
- boolean exist = fc.util().exists(remoteLogCheckSumFile);
- if (!exist) {
- FSDataOutputStream checksumFileOutputStream = null;
- try {
- checksumFileOutputStream = fc.create(remoteLogCheckSumFile,
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
- new Options.CreateOpts[] {});
- checksumFileOutputStream.writeLong(
- currentAggregatedLogFileLength);
- } finally {
- IOUtils.cleanupWithLogger(LOG, checksumFileOutputStream);
- }
- } else {
- FSDataInputStream checksumFileInputStream = null;
- try {
- checksumFileInputStream = fc.open(remoteLogCheckSumFile);
- long endIndex = checksumFileInputStream.readLong();
- IndexedLogsMeta recoveredLogsMeta = loadIndexedLogsMeta(
- remoteLogFile, endIndex);
- if (recoveredLogsMeta == null) {
- indexedLogsMeta.getLogMetas().clear();
- } else {
- indexedLogsMeta = recoveredLogsMeta;
- }
- } finally {
- IOUtils.cleanupWithLogger(LOG, checksumFileInputStream);
- }
+ aggregatedLogFile = initializeWriterInRolling(
+ remoteLogFile, appId, nodeId);
+ } else {
+ aggregatedLogFile = remoteLogFile;
+ fsDataOStream = fc.create(remoteLogFile,
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
+ new Options.CreateOpts[] {});
+ if (uuid == null) {
+ uuid = createUUID(appId);
}
+ fsDataOStream.write(uuid);
+ fsDataOStream.flush();
}
+
+ long aggregatedLogFileLength = fc.getFileStatus(
+ aggregatedLogFile).getLen();
// append a simple character("\n") to move the writer cursor, so
// we could get the correct position when we call
// fsOutputStream.getStartPos()
@@ -294,11 +234,11 @@ public class LogAggregationIndexedFileController
fsDataOStream.write(dummyBytes);
fsDataOStream.flush();
- if (fsDataOStream.getPos() >= (currentAggregatedLogFileLength
+ if (fsDataOStream.getPos() >= (aggregatedLogFileLength
+ dummyBytes.length)) {
currentOffSet = 0;
} else {
- currentOffSet = currentAggregatedLogFileLength;
+ currentOffSet = aggregatedLogFileLength;
}
return null;
}
@@ -308,6 +248,104 @@ public class LogAggregationIndexedFileController
}
}
+ private Path initializeWriterInRolling(final Path remoteLogFile,
+ final ApplicationId appId, final String nodeId) throws Exception {
+ Path aggregatedLogFile = null;
+ // check uuid
+ // if we can not find uuid, we would load the uuid
+ // from previous aggregated log files, and at the same
+ // time, we would delete any aggregated log files which
+ // has invalid uuid.
+ if (uuid == null) {
+ uuid = loadUUIDFromLogFile(fc, remoteLogFile.getParent(),
+ appId, nodeId);
+ }
+ Path currentRemoteLogFile = getCurrentRemoteLogFile(
+ fc, remoteLogFile.getParent(), nodeId);
+ // check checksum file
+ boolean overwriteCheckSum = true;
+ remoteLogCheckSumFile = new Path(remoteLogFile.getParent(),
+ (remoteLogFile.getName() + CHECK_SUM_FILE_SUFFIX));
+ if(fc.util().exists(remoteLogCheckSumFile)) {
+ // if the checksum file exists, we should reset cached
+ // indexedLogsMeta.
+ indexedLogsMeta.getLogMetas().clear();
+ if (currentRemoteLogFile != null) {
+ FSDataInputStream checksumFileInputStream = null;
+ try {
+ checksumFileInputStream = fc.open(remoteLogCheckSumFile);
+ int nameLength = checksumFileInputStream.readInt();
+ byte[] b = new byte[nameLength];
+ int actualLength = checksumFileInputStream.read(b);
+ if (actualLength == nameLength) {
+ String recoveredLogFile = new String(
+ b, Charset.forName("UTF-8"));
+ if (recoveredLogFile.equals(
+ currentRemoteLogFile.getName())) {
+ overwriteCheckSum = false;
+ long endIndex = checksumFileInputStream.readLong();
+ IndexedLogsMeta recoveredLogsMeta = null;
+ try {
+ truncateFileWithRetries(fc, currentRemoteLogFile,
+ endIndex);
+ recoveredLogsMeta = loadIndexedLogsMeta(
+ currentRemoteLogFile);
+ } catch (Exception ex) {
+ recoveredLogsMeta = loadIndexedLogsMeta(
+ currentRemoteLogFile, endIndex);
+ }
+ if (recoveredLogsMeta != null) {
+ indexedLogsMeta = recoveredLogsMeta;
+ }
+ }
+ }
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, checksumFileInputStream);
+ }
+ }
+ }
+ // check whether we need roll over old logs
+ if (currentRemoteLogFile == null || isRollover(
+ fc, currentRemoteLogFile)) {
+ indexedLogsMeta.getLogMetas().clear();
+ overwriteCheckSum = true;
+ aggregatedLogFile = new Path(remoteLogFile.getParent(),
+ remoteLogFile.getName() + "_" + sysClock.getTime());
+ fsDataOStream = fc.create(aggregatedLogFile,
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
+ new Options.CreateOpts[] {});
+ // writes the uuid
+ fsDataOStream.write(uuid);
+ fsDataOStream.flush();
+ } else {
+ aggregatedLogFile = currentRemoteLogFile;
+ fsDataOStream = fc.create(currentRemoteLogFile,
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND),
+ new Options.CreateOpts[] {});
+ }
+ // recreate checksum file if needed before aggregate the logs
+ if (overwriteCheckSum) {
+ final long currentAggregatedLogFileLength = fc
+ .getFileStatus(aggregatedLogFile).getLen();
+ FSDataOutputStream checksumFileOutputStream = null;
+ try {
+ checksumFileOutputStream = fc.create(remoteLogCheckSumFile,
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
+ new Options.CreateOpts[] {});
+ String fileName = aggregatedLogFile.getName();
+ checksumFileOutputStream.writeInt(fileName.length());
+ checksumFileOutputStream.write(fileName.getBytes(
+ Charset.forName("UTF-8")));
+ checksumFileOutputStream.writeLong(
+ currentAggregatedLogFileLength);
+ checksumFileOutputStream.flush();
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, checksumFileOutputStream);
+ }
+ }
+ return aggregatedLogFile;
+ }
+
@Override
public void closeWriter() {
IOUtils.cleanupWithLogger(LOG, this.fsDataOStream);
@@ -390,8 +428,7 @@ public class LogAggregationIndexedFileController
this.fsDataOStream.write(b);
int length = b.length;
this.fsDataOStream.writeInt(length);
- byte[] separator = this.uuid.getBytes(Charset.forName("UTF-8"));
- this.fsDataOStream.write(separator);
+ this.fsDataOStream.write(uuid);
if (logAggregationSuccessfullyInThisCyCle &&
record.isLogAggregationInRolling()) {
deleteFileWithRetries(fc, ugi, remoteLogCheckSumFile);
@@ -410,6 +447,30 @@ public class LogAggregationIndexedFileController
}.runWithRetries();
}
+ private void deleteFileWithRetries(final FileContext fileContext,
+ final Path deletePath) throws Exception {
+ new FSAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ if (fileContext.util().exists(deletePath)) {
+ fileContext.delete(deletePath, false);
+ }
+ return null;
+ }
+ }.runWithRetries();
+ }
+
+ private void truncateFileWithRetries(final FileContext fileContext,
+ final Path truncatePath, final long newLength) throws Exception {
+ new FSAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ fileContext.truncate(truncatePath, newLength);
+ return null;
+ }
+ }.runWithRetries();
+ }
+
private Object deleteFileWithPrivilege(final FileContext fileContext,
final UserGroupInformation userUgi, final Path fileToDelete)
throws Exception {
@@ -449,18 +510,16 @@ public class LogAggregationIndexedFileController
throw new IOException("There is no available log fils for "
+ "application:" + appId);
}
- Map<String, FileStatus> checkSumFiles = filterFiles(
- nodeFiles, CHECK_SUM_FILE_SUFFIX);
+ Map<String, Long> checkSumFiles = parseCheckSumFiles(nodeFiles);
List<FileStatus> fileToRead = getNodeLogFileToRead(
nodeFiles, nodeIdStr, appId);
byte[] buf = new byte[65535];
for (FileStatus thisNodeFile : fileToRead) {
String nodeName = thisNodeFile.getPath().getName();
- FileStatus checkSum = getAllChecksumFiles(checkSumFiles,
- thisNodeFile.getPath().getName());
+ Long checkSumIndex = checkSumFiles.get(nodeName);
long endIndex = -1;
- if (checkSum != null) {
- endIndex = loadIndexedLogsCheckSum(checkSum.getPath());
+ if (checkSumIndex != null) {
+ endIndex = checkSumIndex.longValue();
}
IndexedLogsMeta indexedLogsMeta = null;
try {
@@ -565,17 +624,16 @@ public class LogAggregationIndexedFileController
throw new IOException("There is no available log fils for "
+ "application:" + appId);
}
- Map<String, FileStatus> checkSumFiles = filterFiles(
- nodeFiles, CHECK_SUM_FILE_SUFFIX);
+ Map<String, Long> checkSumFiles = parseCheckSumFiles(nodeFiles);
List<FileStatus> fileToRead = getNodeLogFileToRead(
nodeFiles, nodeIdStr, appId);
for(FileStatus thisNodeFile : fileToRead) {
try {
- FileStatus checkSum = getAllChecksumFiles(checkSumFiles,
+ Long checkSumIndex = checkSumFiles.get(
thisNodeFile.getPath().getName());
long endIndex = -1;
- if (checkSum != null) {
- endIndex = loadIndexedLogsCheckSum(checkSum.getPath());
+ if (checkSumIndex != null) {
+ endIndex = checkSumIndex.longValue();
}
IndexedLogsMeta current = loadIndexedLogsMeta(
thisNodeFile.getPath(), endIndex);
@@ -627,21 +685,46 @@ public class LogAggregationIndexedFileController
}
@Private
- public Map<String, FileStatus> filterFiles(
- List<FileStatus> fileList, final String suffix) throws IOException {
- Map<String, FileStatus> checkSumFiles = new HashMap<>();
+ public Map<String, Long> parseCheckSumFiles(
+ List<FileStatus> fileList) throws IOException {
+ Map<String, Long> checkSumFiles = new HashMap<>();
Set<FileStatus> status = new HashSet<FileStatus>(fileList);
Iterable<FileStatus> mask =
Iterables.filter(status, new Predicate<FileStatus>() {
@Override
public boolean apply(FileStatus next) {
return next.getPath().getName().endsWith(
- suffix);
+ CHECK_SUM_FILE_SUFFIX);
}
});
status = Sets.newHashSet(mask);
+ FileContext fc = null;
for (FileStatus file : status) {
- checkSumFiles.put(file.getPath().getName(), file);
+ FSDataInputStream checksumFileInputStream = null;
+ try {
+ if (fc == null) {
+ fc = FileContext.getFileContext(file.getPath().toUri(), conf);
+ }
+ String nodeName = null;
+ long index = 0L;
+ checksumFileInputStream = fc.open(file.getPath());
+ int nameLength = checksumFileInputStream.readInt();
+ byte[] b = new byte[nameLength];
+ int actualLength = checksumFileInputStream.read(b);
+ if (actualLength == nameLength) {
+ nodeName = new String(b, Charset.forName("UTF-8"));
+ index = checksumFileInputStream.readLong();
+ } else {
+ continue;
+ }
+ if (nodeName != null && !nodeName.isEmpty()) {
+ checkSumFiles.put(nodeName, Long.valueOf(index));
+ }
+ } catch (IOException ex) {
+ continue;
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, checksumFileInputStream);
+ }
}
return checkSumFiles;
}
@@ -755,20 +838,6 @@ public class LogAggregationIndexedFileController
return loadIndexedLogsMeta(remoteLogPath, -1);
}
- @Private
- public long loadIndexedLogsCheckSum(Path remoteLogCheckSumPath)
- throws IOException {
- FileContext fileContext =
- FileContext.getFileContext(remoteLogCheckSumPath.toUri(), conf);
- FSDataInputStream fsDataIStream = null;
- try {
- fsDataIStream = fileContext.open(remoteLogCheckSumPath);
- return fsDataIStream.readLong();
- } finally {
- IOUtils.cleanupWithLogger(LOG, fsDataIStream);
- }
- }
-
/**
* This IndexedLogsMeta includes all the meta information
* for the aggregated log file.
@@ -1034,6 +1103,13 @@ public class LogAggregationIndexedFileController
return conf.getInt(FS_INPUT_BUF_SIZE_ATTR, 256 * 1024);
}
+ @Private
+ @VisibleForTesting
+ public long getRollOverLogMaxSize(Configuration conf) {
+ return 1024L * 1024 * 1024 * conf.getInt(
+ LOG_ROLL_OVER_MAX_FILE_SIZE_GB, 10);
+ }
+
private abstract class FSAction<T> {
abstract T run() throws Exception;
@@ -1054,4 +1130,77 @@ public class LogAggregationIndexedFileController
}
}
}
+
+ private Path getCurrentRemoteLogFile(final FileContext fc,
+ final Path parent, final String nodeId) throws IOException {
+ RemoteIterator<FileStatus> files = fc.listStatus(parent);
+ long maxTime = 0L;
+ Path returnPath = null;
+ while(files.hasNext()) {
+ FileStatus candidate = files.next();
+ String fileName = candidate.getPath().getName();
+ if (fileName.contains(LogAggregationUtils.getNodeString(nodeId))
+ && !fileName.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX) &&
+ !fileName.endsWith(CHECK_SUM_FILE_SUFFIX)) {
+ if (candidate.getModificationTime() > maxTime) {
+ maxTime = candidate.getModificationTime();
+ returnPath = candidate.getPath();
+ }
+ }
+ }
+ return returnPath;
+ }
+
+ private byte[] loadUUIDFromLogFile(final FileContext fc,
+ final Path parent, final ApplicationId appId, final String nodeId)
+ throws Exception {
+ byte[] id = null;
+ RemoteIterator<FileStatus> files = fc.listStatus(parent);
+ FSDataInputStream fsDataInputStream = null;
+ byte[] uuid = createUUID(appId);
+ while(files.hasNext()) {
+ try {
+ Path checkPath = files.next().getPath();
+ if (checkPath.getName().contains(LogAggregationUtils
+ .getNodeString(nodeId)) && !checkPath.getName()
+ .endsWith(CHECK_SUM_FILE_SUFFIX)) {
+ fsDataInputStream = fc.open(checkPath);
+ byte[] b = new byte[uuid.length];
+ int actual = fsDataInputStream.read(b);
+ if (actual != uuid.length || Arrays.equals(b, uuid)) {
+ deleteFileWithRetries(fc, checkPath);
+ } else if (id == null){
+ id = uuid;
+ }
+ }
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, fsDataInputStream);
+ }
+ }
+ return id == null ? uuid : id;
+ }
+
+ @Private
+ @VisibleForTesting
+ public boolean isRollover(final FileContext fc,
+ final Path candidate) throws IOException {
+ FileStatus fs = fc.getFileStatus(candidate);
+ return fs.getLen() >= this.logRollOverMaxFileSize;
+ }
+
+ @Private
+ @VisibleForTesting
+ public Clock getSystemClock() {
+ return SystemClock.getInstance();
+ }
+
+ private byte[] createUUID(ApplicationId appId) throws IOException {
+ try {
+ MessageDigest digest = MessageDigest.getInstance("SHA-256");
+ return digest.digest(appId.toString().getBytes(
+ Charset.forName("UTF-8")));
+ } catch (NoSuchAlgorithmException ex) {
+ throw new IOException(ex);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/280080fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
index f77ad96..7d0205b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
@@ -21,13 +21,13 @@ package org.apache.hadoop.yarn.logaggregation.filecontroller.ifile;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintStream;
import java.io.Writer;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -37,6 +37,8 @@ import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -54,6 +56,8 @@ import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue;
import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.ControlledClock;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -143,8 +147,27 @@ public class TestLogAggregationIndexFileController {
LogValue value = mock(LogValue.class);
when(value.getPendingLogFilesToUploadForThisContainer()).thenReturn(files);
+ final ControlledClock clock = new ControlledClock();
+ clock.setTime(System.currentTimeMillis());
LogAggregationIndexedFileController fileFormat
- = new LogAggregationIndexedFileController();
+ = new LogAggregationIndexedFileController() {
+ private int rollOverCheck = 0;
+ @Override
+ public Clock getSystemClock() {
+ return clock;
+ }
+
+ @Override
+ public boolean isRollover(final FileContext fc,
+ final Path candidate) throws IOException {
+ rollOverCheck++;
+ if (rollOverCheck >= 3) {
+ return true;
+ }
+ return false;
+ }
+ };
+
fileFormat.initialize(conf, "Indexed");
Map<ApplicationAccessType, String> appAcls = new HashMap<>();
@@ -203,7 +226,11 @@ public class TestLogAggregationIndexFileController {
+ LogAggregationIndexedFileController.CHECK_SUM_FILE_SUFFIX);
FSDataOutputStream fInput = null;
try {
+ String nodeName = logPath.getName() + "_" + clock.getTime();
fInput = FileSystem.create(fs, checksumFile, LOG_FILE_UMASK);
+ fInput.writeInt(nodeName.length());
+ fInput.write(nodeName.getBytes(
+ Charset.forName("UTF-8")));
fInput.writeLong(0);
} finally {
IOUtils.closeQuietly(fInput);
@@ -236,9 +263,9 @@ public class TestLogAggregationIndexFileController {
// We did not call postWriter which we would keep the checksum file.
// We can only get the logs/logmeta from the first write.
- fileFormat.readAggregatedLogsMeta(
+ meta = fileFormat.readAggregatedLogsMeta(
logRequest);
- Assert.assertEquals(meta.size(), meta.size(), 1);
+ Assert.assertEquals(meta.size(), 1);
for (ContainerLogMeta log : meta) {
Assert.assertTrue(log.getContainerId().equals(containerId.toString()));
Assert.assertTrue(log.getNodeId().equals(nodeId.toString()));
@@ -267,9 +294,37 @@ public class TestLogAggregationIndexFileController {
fileFormat.write(key1, value2);
fileFormat.postWrite(context);
fileFormat.closeWriter();
- fileFormat.readAggregatedLogsMeta(
+ meta = fileFormat.readAggregatedLogsMeta(
logRequest);
- Assert.assertEquals(meta.size(), meta.size(), 2);
+ Assert.assertEquals(meta.size(), 2);
+ for (ContainerLogMeta log : meta) {
+ Assert.assertTrue(log.getContainerId().equals(containerId.toString()));
+ Assert.assertTrue(log.getNodeId().equals(nodeId.toString()));
+ for (ContainerLogFileInfo file : log.getContainerLogMeta()) {
+ fileNames.add(file.getFileName());
+ }
+ }
+ fileNames.removeAll(newLogTypes);
+ Assert.assertTrue(fileNames.isEmpty());
+ foundLogs = fileFormat.readAggregatedLogs(logRequest, System.out);
+ Assert.assertTrue(foundLogs);
+ for (String logType : newLogTypes) {
+ Assert.assertTrue(sysOutStream.toString().contains(logMessage(
+ containerId, logType)));
+ }
+ sysOutStream.reset();
+
+ // start to roll over old logs
+ clock.setTime(System.currentTimeMillis());
+ fileFormat.initializeWriter(context);
+ fileFormat.write(key1, value2);
+ fileFormat.postWrite(context);
+ fileFormat.closeWriter();
+ FileStatus[] status = fs.listStatus(logPath.getParent());
+ Assert.assertTrue(status.length == 2);
+ meta = fileFormat.readAggregatedLogsMeta(
+ logRequest);
+ Assert.assertEquals(meta.size(), 3);
for (ContainerLogMeta log : meta) {
Assert.assertTrue(log.getContainerId().equals(containerId.toString()));
Assert.assertTrue(log.getNodeId().equals(nodeId.toString()));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/35] hadoop git commit: HDFS-12458. TestReencryptionWithKMS fails
regularly. Contributed by Xiao Chen.
Posted by ae...@apache.org.
HDFS-12458. TestReencryptionWithKMS fails regularly. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c34ceaf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c34ceaf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c34ceaf
Branch: refs/heads/HDFS-7240
Commit: 7c34ceaf4fa28e2ecabd6626860bb1c7418e4b8d
Parents: 28c4957
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Thu Sep 28 05:09:37 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Sep 28 05:10:26 2017 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/TestReencryption.java | 13 +++++++------
.../hdfs/server/namenode/TestReencryptionHandler.java | 7 ++++---
2 files changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c34ceaf/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
index 33c52bf..4bf6aa4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
@@ -28,7 +28,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -129,11 +128,9 @@ public class TestReencryption {
conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, 3);
// Adjust configs for re-encrypt test cases
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REENCRYPT_BATCH_SIZE_KEY, 5);
- conf.setTimeDuration(
- DFSConfigKeys.DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_KEY, 1,
- TimeUnit.SECONDS);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
cluster.waitActive();
+ cluster.waitClusterUp();
fs = cluster.getFileSystem();
fsn = cluster.getNamesystem();
fsWrapper = new FileSystemTestWrapper(fs);
@@ -1284,6 +1281,7 @@ public class TestReencryption {
fsn = cluster.getNamesystem();
getEzManager().pauseReencryptForTesting();
cluster.waitActive();
+ cluster.waitClusterUp();
}
private void waitForReencryptedZones(final int expected)
@@ -1519,6 +1517,7 @@ public class TestReencryption {
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForQueuedZones(1);
+ getEzManager().pauseReencryptUpdaterForTesting();
getEzManager().resumeReencryptForTesting();
LOG.info("Waiting for re-encrypt callables to run");
@@ -1529,7 +1528,6 @@ public class TestReencryption {
}
}, 100, 10000);
- getEzManager().pauseReencryptUpdaterForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.CANCEL);
// now resume updater and verify status.
@@ -1609,7 +1607,7 @@ public class TestReencryption {
cluster.getConfiguration(0)
.unset(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
cluster.restartNameNodes();
- cluster.waitActive();
+ cluster.waitClusterUp();
// test re-encrypt should fail
try {
@@ -1673,6 +1671,9 @@ public class TestReencryption {
}
fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+ // trigger the background thread to run, without having to
+ // wait for DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_KEY
+ getHandler().notifyNewSubmission();
waitForReencryptedFiles(zone.toString(), 10);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c34ceaf/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
index f0dd92c..e2035ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
@@ -101,9 +101,9 @@ public class TestReencryptionHandler {
final StopWatch sw = new StopWatch().start();
rh.throttle();
sw.stop();
- assertTrue("should have throttled for at least 4 second",
+ assertTrue("should have throttled for at least 8 second",
sw.now(TimeUnit.MILLISECONDS) > 8000);
- assertTrue("should have throttled for at most 6 second",
+ assertTrue("should have throttled for at most 12 second",
sw.now(TimeUnit.MILLISECONDS) < 12000);
}
@@ -191,7 +191,8 @@ public class TestReencryptionHandler {
removeTaskThread.start();
rh.throttle();
sw.stop();
+ LOG.info("Throttle completed, consumed {}", sw.now(TimeUnit.MILLISECONDS));
assertTrue("should have throttled for at least 3 second",
- sw.now(TimeUnit.MILLISECONDS) > 3000);
+ sw.now(TimeUnit.MILLISECONDS) >= 3000);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/35] hadoop git commit: HDFS-12560. Remove the extra word it in
HdfsUserGuide.md. Contributed by fang zhenyi.
Posted by ae...@apache.org.
HDFS-12560. Remove the extra word it in HdfsUserGuide.md. Contributed by fang zhenyi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e1bd114
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e1bd114
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e1bd114
Branch: refs/heads/HDFS-7240
Commit: 8e1bd114ee718c42cd3f3275bee20f5d401a1d99
Parents: 7c34cea
Author: Chen Liang <cl...@apache.org>
Authored: Thu Sep 28 10:22:27 2017 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Thu Sep 28 10:22:27 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e1bd114/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
index 7a06c41..6f707f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
@@ -257,7 +257,7 @@ During start up the NameNode loads the file system state from the fsimage and th
fsck
----
-HDFS supports the fsck command to check for various inconsistencies. It it is designed for reporting problems with various files, for example, missing blocks for a file or under-replicated blocks. Unlike a traditional fsck utility for native file systems, this command does not correct the errors it detects. Normally NameNode automatically corrects most of the recoverable failures. By default fsck ignores open files but provides an option to select all files during reporting. The HDFS fsck command is not a Hadoop shell command. It can be run as `bin/hdfs fsck`. For command usage, see [fsck](./HDFSCommands.html#fsck). fsck can be run on the whole file system or on a subset of files.
+HDFS supports the fsck command to check for various inconsistencies. It is designed for reporting problems with various files, for example, missing blocks for a file or under-replicated blocks. Unlike a traditional fsck utility for native file systems, this command does not correct the errors it detects. Normally NameNode automatically corrects most of the recoverable failures. By default fsck ignores open files but provides an option to select all files during reporting. The HDFS fsck command is not a Hadoop shell command. It can be run as `bin/hdfs fsck`. For command usage, see [fsck](./HDFSCommands.html#fsck). fsck can be run on the whole file system or on a subset of files.
fetchdt
-------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/35] hadoop git commit: YARN-7248. NM returns new SCHEDULED
container status to older clients. Contributed by Arun Suresh
Posted by ae...@apache.org.
YARN-7248. NM returns new SCHEDULED container status to older clients. Contributed by Arun Suresh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85d81ae5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85d81ae5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85d81ae5
Branch: refs/heads/HDFS-7240
Commit: 85d81ae58ec4361a944c84753a900460a0888b9b
Parents: a530e7a
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Sep 28 14:10:15 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Sep 28 14:10:15 2017 -0500
----------------------------------------------------------------------
.../hadoop/yarn/api/records/ContainerState.java | 10 +--
.../yarn/api/records/ContainerStatus.java | 22 ++++++
.../yarn/api/records/ContainerSubState.java | 57 ++++++++++++++
.../src/main/proto/yarn_protos.proto | 34 ++++++++-
.../records/impl/pb/ContainerStatusPBImpl.java | 24 +++++-
.../yarn/api/records/impl/pb/ProtoUtils.java | 20 ++++-
.../container/ContainerImpl.java | 33 ++++++++-
.../container/ContainerState.java | 9 +++
.../yarn/server/nodemanager/TestEventFlow.java | 2 +-
.../nodemanager/TestNodeManagerShutdown.java | 3 +-
.../TestContainerSchedulerQueuing.java | 78 ++++++++------------
.../resourcemanager/rmnode/RMNodeImpl.java | 6 +-
.../TestResourceTrackerService.java | 2 +-
13 files changed, 229 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
index 45e5bd4..e22204e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.api.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Stable;
/**
@@ -35,12 +34,5 @@ public enum ContainerState {
RUNNING,
/** Completed container */
- COMPLETE,
-
- /** Scheduled (awaiting resources) at the NM. */
- @InterfaceStability.Unstable
- SCHEDULED,
-
- /** Paused at the NM. */
- PAUSED
+ COMPLETE
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
index d7c75f3..edc62fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
@@ -201,4 +201,26 @@ public abstract class ContainerStatus {
throw new UnsupportedOperationException(
"subclass must implement this method");
}
+
+ /**
+ * Add Extra state information of the container (SCHEDULED, LOCALIZING etc.).
+ * @param subState Extra State Information.
+ */
+ @Private
+ @Unstable
+ public void setContainerSubState(ContainerSubState subState) {
+ throw new UnsupportedOperationException(
+ "subclass must implement this method");
+ }
+
+ /**
+ * Get Extra state information of the container (SCHEDULED, LOCALIZING etc.).
+ * @return Extra State information.
+ */
+ @Private
+ @Unstable
+ public ContainerSubState getContainerSubState() {
+ throw new UnsupportedOperationException(
+ "subclass must implement this method");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerSubState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerSubState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerSubState.java
new file mode 100644
index 0000000..fed4537
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerSubState.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Container Sub-State.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public enum ContainerSubState {
+ /*
+ * NEW, LOCALIZING, SCHEDULED,
+ * REINITIALIZING_AWAITING_KILL, RELAUNCHING,
+ */
+ SCHEDULED,
+
+ /*
+ * RUNNING, REINITIALIZING, PAUSING, KILLING
+ */
+ RUNNING,
+
+ /*
+ * PAUSED, RESUMING
+ */
+ PAUSED,
+
+ /*
+ * LOCALIZATION_FAILED, EXITED_WITH_SUCCESS,
+ * EXITED_WITH_FAILURE,
+ * CONTAINER_CLEANEDUP_AFTER_KILL,
+ * CONTAINER_RESOURCES_CLEANINGUP
+ */
+ COMPLETING,
+
+ /*
+ * DONE
+ */
+ DONE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index fb340d1..e69c07b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -109,8 +109,37 @@ enum ContainerStateProto {
C_NEW = 1;
C_RUNNING = 2;
C_COMPLETE = 3;
- C_SCHEDULED = 4;
- C_PAUSED = 5;
+}
+
+enum ContainerSubStateProto {
+ /**
+ * NEW, LOCALIZING, SCHEDULED,
+ * REINITIALIZING_AWAITING_KILL, RELAUNCHING,
+ */
+ CSS_SCHEDULED = 1;
+
+ /**
+ * RUNNING, REINITIALIZING, PAUSING, KILLING
+ */
+ CSS_RUNNING = 2;
+
+ /**
+ * PAUSED, RESUMING
+ */
+ CSS_PAUSED = 3;
+
+ /**
+ * LOCALIZATION_FAILED, EXITED_WITH_SUCCESS,
+ * EXITED_WITH_FAILURE,
+ * CONTAINER_CLEANEDUP_AFTER_KILL,
+ * CONTAINER_RESOURCES_CLEANINGUP
+ */
+ CSS_COMPLETING = 4;
+
+ /**
+ * DONE
+ */
+ CSS_DONE = 5;
}
message ContainerProto {
@@ -616,6 +645,7 @@ message ContainerStatusProto {
optional ResourceProto capability = 5;
optional ExecutionTypeProto executionType = 6 [default = GUARANTEED];
repeated StringStringMapProto container_attributes = 7;
+ optional ContainerSubStateProto container_sub_state = 8;
}
enum ContainerExitStatusProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
index 11a3d86..a6668da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos;
@@ -96,7 +97,8 @@ public class ContainerStatusPBImpl extends ContainerStatus {
sb.append("Diagnostics: ").append(getDiagnostics()).append(", ");
sb.append("ExitStatus: ").append(getExitStatus()).append(", ");
sb.append("IP: ").append(getIPs()).append(", ");
- sb.append("Host: ").append(getHost());
+ sb.append("Host: ").append(getHost()).append(", ");
+ sb.append("ContainerSubState: ").append(getContainerSubState());
sb.append("]");
return sb.toString();
}
@@ -213,6 +215,26 @@ public class ContainerStatusPBImpl extends ContainerStatus {
}
builder.setState(convertToProtoFormat(state));
}
+
+ @Override
+ public synchronized ContainerSubState getContainerSubState() {
+ ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasContainerSubState()) {
+ return null;
+ }
+ return ProtoUtils.convertFromProtoFormat(p.getContainerSubState());
+ }
+
+ @Override
+ public synchronized void setContainerSubState(ContainerSubState subState) {
+ maybeInitBuilder();
+ if (subState == null) {
+ builder.clearContainerSubState();
+ return;
+ }
+ builder.setContainerSubState(ProtoUtils.convertToProtoFormat(subState));
+ }
+
@Override
public synchronized ContainerId getContainerId() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 158c2ae..528cf8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.ExecutionType;
@@ -61,6 +62,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportPro
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationTimeoutTypeProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerSubStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceTypeProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto;
@@ -92,7 +94,7 @@ public class ProtoUtils {
/*
* ContainerState
*/
- private static String CONTAINER_STATE_PREFIX = "C_";
+ private final static String CONTAINER_STATE_PREFIX = "C_";
public static ContainerStateProto convertToProtoFormat(ContainerState e) {
return ContainerStateProto.valueOf(CONTAINER_STATE_PREFIX + e.name());
}
@@ -101,9 +103,23 @@ public class ProtoUtils {
}
/*
+ * Container SubState
+ */
+ private final static String CONTAINER_SUB_STATE_PREFIX = "CSS_";
+ public static ContainerSubStateProto convertToProtoFormat(
+ ContainerSubState e) {
+ return ContainerSubStateProto.valueOf(
+ CONTAINER_SUB_STATE_PREFIX + e.name());
+ }
+ public static ContainerSubState convertFromProtoFormat(
+ ContainerSubStateProto e) {
+ return ContainerSubState.valueOf(
+ e.name().substring(CONTAINER_SUB_STATE_PREFIX.length()));
+ }
+ /*
* NodeState
*/
- private static String NODE_STATE_PREFIX = "NS_";
+ private final static String NODE_STATE_PREFIX = "NS_";
public static NodeStateProto convertToProtoFormat(NodeState e) {
return NodeStateProto.valueOf(NODE_STATE_PREFIX + e.name());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 2bbc267..df83789 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -34,6 +34,7 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.UpdateContainerSchedulerEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -714,7 +715,6 @@ public class ContainerImpl implements Container {
case SCHEDULED:
case PAUSED:
case RESUMING:
- return org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED;
case RUNNING:
case RELAUNCHING:
case REINITIALIZING:
@@ -732,6 +732,36 @@ public class ContainerImpl implements Container {
}
}
+ // NOTE: Please update the doc in the ContainerSubState class as
+ // well as the yarn_protos.proto file if this mapping is ever modified.
+ private ContainerSubState getContainerSubState() {
+ switch (stateMachine.getCurrentState()) {
+ case NEW:
+ case LOCALIZING:
+ case SCHEDULED:
+ case REINITIALIZING_AWAITING_KILL:
+ case RELAUNCHING:
+ return ContainerSubState.SCHEDULED;
+ case REINITIALIZING:
+ case PAUSING:
+ case KILLING:
+ case RUNNING:
+ return ContainerSubState.RUNNING;
+ case PAUSED:
+ case RESUMING:
+ return ContainerSubState.PAUSED;
+ case LOCALIZATION_FAILED:
+ case EXITED_WITH_SUCCESS:
+ case EXITED_WITH_FAILURE:
+ case CONTAINER_CLEANEDUP_AFTER_KILL:
+ case CONTAINER_RESOURCES_CLEANINGUP:
+ return ContainerSubState.COMPLETING;
+ case DONE:
+ default:
+ return ContainerSubState.DONE;
+ }
+ }
+
public NMTimelinePublisher getNMTimelinePublisher() {
return context.getNMTimelinePublisher();
}
@@ -800,6 +830,7 @@ public class ContainerImpl implements Container {
this.containerTokenIdentifier.getExecutionType());
status.setIPs(ips == null ? null : Arrays.asList(ips.split(",")));
status.setHost(host);
+ status.setContainerSubState(getContainerSubState());
return status;
} finally {
this.readLock.unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
index 5644d03..a0db646 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
@@ -18,7 +18,16 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+/**
+ * States used by the container state machine.
+ */
public enum ContainerState {
+ // NOTE: In case of future additions / deletions / modifications to this
+ // enum, please ensure that the following are also correspondingly
+ // updated:
+ // 1. ContainerImpl::getContainerSubState().
+ // 2. the doc in the ContainerSubState class.
+ // 3. the doc in the yarn_protos.proto file.
NEW, LOCALIZING, LOCALIZATION_FAILED, SCHEDULED, RUNNING, RELAUNCHING,
REINITIALIZING, REINITIALIZING_AWAITING_KILL,
EXITED_WITH_SUCCESS, EXITED_WITH_FAILURE, KILLING,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
index 8e4522b..54e090a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
@@ -159,7 +159,7 @@ public class TestEventFlow {
containerManager.startContainers(allRequests);
BaseContainerManagerTest.waitForContainerState(containerManager, cID,
- Arrays.asList(ContainerState.RUNNING, ContainerState.SCHEDULED), 20);
+ Arrays.asList(ContainerState.RUNNING), 20);
List<ContainerId> containerIds = new ArrayList<ContainerId>();
containerIds.add(cID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
index 03e06d2..25dbc1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
@@ -256,8 +256,7 @@ public class TestNodeManagerShutdown {
GetContainerStatusesRequest.newInstance(containerIds);
ContainerStatus containerStatus =
containerManager.getContainerStatuses(request).getContainerStatuses().get(0);
- Assert.assertTrue(
- EnumSet.of(ContainerState.RUNNING, ContainerState.SCHEDULED)
+ Assert.assertTrue(EnumSet.of(ContainerState.RUNNING)
.contains(containerStatus.getState()));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
index 4b380ff..ed4f290 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -311,9 +312,8 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
List<ContainerStatus> containerStatuses = containerManager
.getContainerStatuses(statRequest).getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED,
- status.getState());
+ Assert.assertEquals(ContainerSubState.SCHEDULED,
+ status.getContainerSubState());
}
ContainerScheduler containerScheduler =
@@ -378,13 +378,11 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
.getContainerStatuses(statRequest).getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
if (status.getContainerId().equals(createContainerId(0))) {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
- status.getState());
+ Assert.assertEquals(ContainerSubState.RUNNING,
+ status.getContainerSubState());
} else {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED,
- status.getState());
+ Assert.assertEquals(ContainerSubState.SCHEDULED,
+ status.getContainerSubState());
}
}
@@ -458,17 +456,15 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
.getContainerStatuses(statRequest).getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
if (status.getContainerId().equals(createContainerId(0))) {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
- status.getState());
+ Assert.assertEquals(ContainerSubState.RUNNING,
+ status.getContainerSubState());
} else if (status.getContainerId().equals(createContainerId(
maxOppQueueLength + 1))) {
Assert.assertTrue(status.getDiagnostics().contains(
"Opportunistic container queue is full"));
} else {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED,
- status.getState());
+ Assert.assertEquals(ContainerSubState.SCHEDULED,
+ status.getContainerSubState());
}
System.out.println("\nStatus : [" + status + "]\n");
}
@@ -545,13 +541,11 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
Assert.assertTrue(status.getDiagnostics().contains(
"Container Killed to make room for Guaranteed Container"));
} else if (status.getContainerId().equals(createContainerId(1))) {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED,
- status.getState());
+ Assert.assertEquals(ContainerSubState.SCHEDULED,
+ status.getContainerSubState());
} else if (status.getContainerId().equals(createContainerId(2))) {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
- status.getState());
+ Assert.assertEquals(ContainerSubState.RUNNING,
+ status.getContainerSubState());
}
System.out.println("\nStatus : [" + status + "]\n");
}
@@ -799,8 +793,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
"Container De-queued to meet NM queuing limits")) {
deQueuedContainers++;
}
- if (status.getState() ==
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED) {
+ if (ContainerSubState.SCHEDULED == status.getContainerSubState()) {
numQueuedOppContainers++;
}
}
@@ -1079,11 +1072,9 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
List<ContainerStatus> containerStatuses = containerManager
.getContainerStatuses(statRequest).getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
- if (status.getState() ==
- org.apache.hadoop.yarn.api.records.ContainerState.RUNNING) {
+ if (ContainerSubState.RUNNING == status.getContainerSubState()) {
runningContainersNo++;
- } else if (status.getState() ==
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED) {
+ } else if (ContainerSubState.SCHEDULED == status.getContainerSubState()) {
queuedContainersNo++;
}
System.out.println("\nStatus : [" + status + "]\n");
@@ -1106,34 +1097,27 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
}
statRequest = GetContainerStatusesRequest.newInstance(statList);
- HashMap<org.apache.hadoop.yarn.api.records.ContainerState, ContainerStatus>
- map = new HashMap<>();
+ HashMap<ContainerSubState, ContainerStatus> map = new HashMap<>();
for (int i=0; i < 10; i++) {
containerStatuses = containerManager.getContainerStatuses(statRequest)
.getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
System.out.println("\nStatus : [" + status + "]\n");
- map.put(status.getState(), status);
- if (map.containsKey(
- org.apache.hadoop.yarn.api.records.ContainerState.RUNNING) &&
- map.containsKey(
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED) &&
- map.containsKey(
- org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE)) {
+ map.put(status.getContainerSubState(), status);
+ if (map.containsKey(ContainerSubState.RUNNING) &&
+ map.containsKey(ContainerSubState.SCHEDULED) &&
+ map.containsKey(ContainerSubState.DONE)) {
break;
}
Thread.sleep(1000);
}
}
Assert.assertEquals(createContainerId(0),
- map.get(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING)
- .getContainerId());
+ map.get(ContainerSubState.RUNNING).getContainerId());
Assert.assertEquals(createContainerId(1),
- map.get(org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE)
- .getContainerId());
+ map.get(ContainerSubState.DONE).getContainerId());
Assert.assertEquals(createContainerId(2),
- map.get(org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED)
- .getContainerId());
+ map.get(ContainerSubState.SCHEDULED).getContainerId());
}
/**
@@ -1186,13 +1170,11 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
.getContainerStatuses(statRequest).getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
if (status.getContainerId().equals(createContainerId(0))) {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
- status.getState());
+ Assert.assertEquals(ContainerSubState.RUNNING,
+ status.getContainerSubState());
} else {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED,
- status.getState());
+ Assert.assertEquals(ContainerSubState.SCHEDULED,
+ status.getContainerSubState());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index c547128..d0bfecf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -856,8 +856,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
containers = startEvent.getNMContainerStatuses();
if (containers != null && !containers.isEmpty()) {
for (NMContainerStatus container : containers) {
- if (container.getContainerState() == ContainerState.RUNNING ||
- container.getContainerState() == ContainerState.SCHEDULED) {
+ if (container.getContainerState() == ContainerState.RUNNING) {
rmNode.launchedContainers.add(container.getContainerId());
}
}
@@ -1412,8 +1411,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
// Process running containers
- if (remoteContainer.getState() == ContainerState.RUNNING ||
- remoteContainer.getState() == ContainerState.SCHEDULED) {
+ if (remoteContainer.getState() == ContainerState.RUNNING) {
++numRemoteRunningContainers;
if (!launchedContainers.contains(containerId)) {
// Just launched container. RM knows about it the first time.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 41078d0..2bd745b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -2085,7 +2085,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
ContainerId c2 = ContainerId.newContainerId(appAttemptId, 2);
ContainerId c3 = ContainerId.newContainerId(appAttemptId, 3);
NMContainerStatus queuedOpp =
- NMContainerStatus.newInstance(c1, 1, ContainerState.SCHEDULED,
+ NMContainerStatus.newInstance(c1, 1, ContainerState.RUNNING,
Resource.newInstance(1024, 1), "Dummy Queued OC",
ContainerExitStatus.INVALID, Priority.newInstance(5), 1234, "",
ExecutionType.OPPORTUNISTIC);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/35] hadoop git commit: YARN-6333. Improve doc for
minSharePreemptionTimeout,
fairSharePreemptionTimeout and fairSharePreemptionThreshold. (Contributed by
Chetna Chaudhari via Yufei Gu)
Posted by ae...@apache.org.
YARN-6333. Improve doc for minSharePreemptionTimeout, fairSharePreemptionTimeout and fairSharePreemptionThreshold. (Contributed by Chetna Chaudhari via Yufei Gu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66c41716
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66c41716
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66c41716
Branch: refs/heads/HDFS-7240
Commit: 66c417167ae0d24467360ae3b222a10f00fa2700
Parents: ec2ae30
Author: Yufei Gu <yu...@apache.org>
Authored: Fri Sep 29 18:07:33 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Fri Sep 29 18:07:33 2017 -0700
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/FairScheduler.md | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66c41716/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index 748fda9..2ccd5ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -101,11 +101,11 @@ The allocation file must be in XML format. The format contains five types of ele
* **aclAdministerApps**: a list of users and/or groups that can administer a queue. Currently the only administrative action is killing an application. Refer to the ACLs section below for more info on the format of this list and how queue ACLs work.
- * **minSharePreemptionTimeout**: number of seconds the queue is under its minimum share before it will try to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue.
+ * **minSharePreemptionTimeout**: number of seconds the queue is under its minimum share before it will try to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. Default value is Long.MAX_VALUE, which means that it will not preempt containers until you set a meaningful value.
- * **fairSharePreemptionTimeout**: number of seconds the queue is under its fair share threshold before it will try to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue.
+ * **fairSharePreemptionTimeout**: number of seconds the queue is under its fair share threshold before it will try to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. Default value is Long.MAX_VALUE, which means that it will not preempt containers until you set a meaningful value.
- * **fairSharePreemptionThreshold**: the fair share preemption threshold for the queue. If the queue waits fairSharePreemptionTimeout without receiving fairSharePreemptionThreshold\*fairShare resources, it is allowed to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue.
+ * **fairSharePreemptionThreshold**: the fair share preemption threshold for the queue. If the queue waits fairSharePreemptionTimeout without receiving fairSharePreemptionThreshold\*fairShare resources, it is allowed to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. Default value is 0.5f.
* **allowPreemptionFrom**: determines whether the scheduler is allowed to preempt resources from the queue. The default is true. If a queue has this property set to false, this property will apply recursively to all child queues.
@@ -113,11 +113,11 @@ The allocation file must be in XML format. The format contains five types of ele
* **A userMaxAppsDefault element**: which sets the default running app limit for any users whose limit is not otherwise specified.
-* **A defaultFairSharePreemptionTimeout element**: which sets the fair share preemption timeout for the root queue; overridden by fairSharePreemptionTimeout element in root queue.
+* **A defaultFairSharePreemptionTimeout element**: which sets the fair share preemption timeout for the root queue; overridden by fairSharePreemptionTimeout element in root queue. Default is set to Long.MAX_VALUE.
-* **A defaultMinSharePreemptionTimeout element**: which sets the min share preemption timeout for the root queue; overridden by minSharePreemptionTimeout element in root queue.
+* **A defaultMinSharePreemptionTimeout element**: which sets the min share preemption timeout for the root queue; overridden by minSharePreemptionTimeout element in root queue. Default is set to Long.MAX_VALUE.
-* **A defaultFairSharePreemptionThreshold element**: which sets the fair share preemption threshold for the root queue; overridden by fairSharePreemptionThreshold element in root queue.
+* **A defaultFairSharePreemptionThreshold element**: which sets the fair share preemption threshold for the root queue; overridden by fairSharePreemptionThreshold element in root queue. Default is set to 0.5f.
* **A queueMaxAppsDefault element**: which sets the default running app limit for queues; overridden by maxRunningApps element in each queue.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/35] hadoop git commit: YARN-7253. Shared Cache Manager daemon
command listed as admin subcmd in yarn script.
Posted by ae...@apache.org.
YARN-7253. Shared Cache Manager daemon command listed as admin subcmd in yarn script.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c87db8d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c87db8d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c87db8d1
Branch: refs/heads/HDFS-7240
Commit: c87db8d154ab2501e786b4f1669b205759ece5c3
Parents: 8facf1f
Author: Chris Trezzo <ct...@apache.org>
Authored: Wed Sep 27 11:32:09 2017 -0700
Committer: Chris Trezzo <ct...@apache.org>
Committed: Wed Sep 27 11:32:09 2017 -0700
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/yarn | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c87db8d1/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index dcde0dc..52138c5 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -48,7 +48,7 @@ function hadoop_usage
hadoop_add_subcommand "rmadmin" admin "admin tools"
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
- hadoop_add_subcommand "sharedcachemanager" admin "run the SharedCacheManager daemon"
+ hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
hadoop_add_subcommand "top" client "view cluster information"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/35] hadoop git commit: HADOOP-14915. method name is incorrect in
ConfServlet. Contributed by Bharat Viswanadham.
Posted by ae...@apache.org.
HADOOP-14915. method name is incorrect in ConfServlet. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/563dcdfc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/563dcdfc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/563dcdfc
Branch: refs/heads/HDFS-7240
Commit: 563dcdfc1de7ea9ee7ce296163cf2678dfe5349c
Parents: 06df6ab
Author: Chen Liang <cl...@apache.org>
Authored: Mon Oct 2 10:58:19 2017 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Mon Oct 2 10:58:19 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/conf/ConfServlet.java | 4 ++--
.../src/test/java/org/apache/hadoop/conf/TestConfServlet.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/563dcdfc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
index cfd7b97..2128de7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
@@ -69,7 +69,7 @@ public class ConfServlet extends HttpServlet {
return;
}
- String format = parseAccecptHeader(request);
+ String format = parseAcceptHeader(request);
if (FORMAT_XML.equals(format)) {
response.setContentType("text/xml; charset=utf-8");
} else if (FORMAT_JSON.equals(format)) {
@@ -89,7 +89,7 @@ public class ConfServlet extends HttpServlet {
}
@VisibleForTesting
- static String parseAccecptHeader(HttpServletRequest request) {
+ static String parseAcceptHeader(HttpServletRequest request) {
String format = request.getHeader(HttpHeaders.ACCEPT);
return format != null && format.contains(FORMAT_JSON) ?
FORMAT_JSON : FORMAT_XML;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/563dcdfc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
index 53089ed..cf42219 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
@@ -98,7 +98,7 @@ public class TestConfServlet {
Mockito.when(request.getHeader(HttpHeaders.ACCEPT))
.thenReturn(contentTypeExpected);
assertEquals(contenTypeActual,
- ConfServlet.parseAccecptHeader(request));
+ ConfServlet.parseAcceptHeader(request));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/35] hadoop git commit: YARN-7226. Whitelisted variables do not
support delayed variable expansion. Contributed by Jason Lowe
Posted by ae...@apache.org.
YARN-7226. Whitelisted variables do not support delayed variable expansion. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7eb84999
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7eb84999
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7eb84999
Branch: refs/heads/HDFS-7240
Commit: 7eb8499996869cdb63743f1c9eca2ba91d57ad08
Parents: d4d2fd1
Author: Sidharta S <si...@apache.org>
Authored: Mon Oct 2 19:04:49 2017 -0700
Committer: Sidharta S <si...@apache.org>
Committed: Mon Oct 2 19:04:49 2017 -0700
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 44 ++++++++----
.../nodemanager/LinuxContainerExecutor.java | 8 +++
.../launcher/ContainerLaunch.java | 25 -------
.../runtime/DefaultLinuxContainerRuntime.java | 6 ++
.../DelegatingLinuxContainerRuntime.java | 11 +++
.../runtime/DockerLinuxContainerRuntime.java | 7 ++
.../runtime/ContainerRuntime.java | 11 +++
.../launcher/TestContainerLaunch.java | 73 ++++++++++++++++++--
8 files changed, 141 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 5fd059d..0b4dbc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -26,10 +26,8 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -95,10 +93,15 @@ public abstract class ContainerExecutor implements Configurable {
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
private final ReadLock readLock = lock.readLock();
private final WriteLock writeLock = lock.writeLock();
+ private String[] whitelistVars;
@Override
public void setConf(Configuration conf) {
this.conf = conf;
+ if (conf != null) {
+ whitelistVars = conf.get(YarnConfiguration.NM_ENV_WHITELIST,
+ YarnConfiguration.DEFAULT_NM_ENV_WHITELIST).split(",");
+ }
}
@Override
@@ -331,6 +334,8 @@ public abstract class ContainerExecutor implements Configurable {
public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
Map<Path, List<String>> resources, List<String> command, Path logDir,
String user, String outFilename) throws IOException {
+ updateEnvForWhitelistVars(environment);
+
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
@@ -341,22 +346,11 @@ public abstract class ContainerExecutor implements Configurable {
sb.stdout(logDir, CONTAINER_PRE_LAUNCH_STDOUT);
sb.stderr(logDir, CONTAINER_PRE_LAUNCH_STDERR);
- Set<String> whitelist = new HashSet<>();
-
- String[] nmWhiteList = conf.get(YarnConfiguration.NM_ENV_WHITELIST,
- YarnConfiguration.DEFAULT_NM_ENV_WHITELIST).split(",");
- for (String param : nmWhiteList) {
- whitelist.add(param);
- }
if (environment != null) {
sb.echo("Setting up env variables");
for (Map.Entry<String, String> env : environment.entrySet()) {
- if (!whitelist.contains(env.getKey())) {
- sb.env(env.getKey(), env.getValue());
- } else {
- sb.whitelistedEnv(env.getKey(), env.getValue());
- }
+ sb.env(env.getKey(), env.getValue());
}
}
@@ -658,6 +652,28 @@ public abstract class ContainerExecutor implements Configurable {
}
/**
+ * Propagate variables from the nodemanager's environment into the
+ * container's environment if unspecified by the container.
+ * @param env the environment to update
+ * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST
+ */
+ protected void updateEnvForWhitelistVars(Map<String, String> env) {
+ for(String var : whitelistVars) {
+ if (!env.containsKey(var)) {
+ String val = getNMEnvVar(var);
+ if (val != null) {
+ env.put(var, val);
+ }
+ }
+ }
+ }
+
+ @VisibleForTesting
+ protected String getNMEnvVar(String varname) {
+ return System.getenv(varname);
+ }
+
+ /**
* Mark the container as active.
*
* @param containerId the container ID
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 9e004e7..9f1ec60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -62,6 +62,7 @@ import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -442,6 +443,13 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
+ protected void updateEnvForWhitelistVars(Map<String, String> env) {
+ if (linuxContainerRuntime.useWhitelistEnv(env)) {
+ super.updateEnvForWhitelistVars(env);
+ }
+ }
+
+ @Override
public int launchContainer(ContainerStartContext ctx)
throws IOException, ConfigurationException {
Container container = ctx.getContainer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index f929dfc..f63b1b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -89,7 +89,6 @@ import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.yarn.util.ConverterUtils;
public class ContainerLaunch implements Callable<Integer> {
@@ -1022,8 +1021,6 @@ public class ContainerLaunch implements Callable<Integer> {
public abstract void command(List<String> command) throws IOException;
- public abstract void whitelistedEnv(String key, String value) throws IOException;
-
protected static final String ENV_PRELAUNCH_STDOUT = "PRELAUNCH_OUT";
protected static final String ENV_PRELAUNCH_STDERR = "PRELAUNCH_ERR";
@@ -1163,11 +1160,6 @@ public class ContainerLaunch implements Callable<Integer> {
}
@Override
- public void whitelistedEnv(String key, String value) throws IOException {
- line("export ", key, "=${", key, ":-", "\"", value, "\"}");
- }
-
- @Override
public void setStdOut(final Path stdout) throws IOException {
line("export ", ENV_PRELAUNCH_STDOUT, "=\"", stdout.toString(), "\"");
// tee is needed for DefaultContainerExecutor error propagation to stdout
@@ -1262,12 +1254,6 @@ public class ContainerLaunch implements Callable<Integer> {
errorCheck();
}
- @Override
- public void whitelistedEnv(String key, String value) throws IOException {
- lineWithLenCheck("@set ", key, "=", value);
- errorCheck();
- }
-
//Dummy implementation
@Override
protected void setStdOut(final Path stdout) throws IOException {
@@ -1389,17 +1375,6 @@ public class ContainerLaunch implements Callable<Integer> {
environment.put("JVM_PID", "$$");
}
- /**
- * Modifiable environment variables
- */
-
- // allow containers to override these variables
- String[] whitelist = conf.get(YarnConfiguration.NM_ENV_WHITELIST, YarnConfiguration.DEFAULT_NM_ENV_WHITELIST).split(",");
-
- for(String whitelistEnvVariable : whitelist) {
- putEnvIfAbsent(environment, whitelistEnvVariable.trim());
- }
-
// variables here will be forced in, even if the container has specified them.
Apps.setEnvFromInputString(environment, conf.get(
YarnConfiguration.NM_ADMIN_USER_ENV,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index 6c3ae85..e9c58b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -36,6 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.Map;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -72,6 +73,11 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
}
@Override
+ public boolean useWhitelistEnv(Map<String, String> env) {
+ return true;
+ }
+
+ @Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
//nothing to do here at the moment.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index 9fe4927..517a4e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -93,6 +93,17 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
}
}
+ @Override
+ public boolean useWhitelistEnv(Map<String, String> env) {
+ try {
+ LinuxContainerRuntime runtime = pickContainerRuntime(env);
+ return runtime.useWhitelistEnv(env);
+ } catch (ContainerExecutionException e) {
+ LOG.debug("Unable to determine runtime");
+ return false;
+ }
+ }
+
@VisibleForTesting
LinuxContainerRuntime pickContainerRuntime(
Map<String, String> environment) throws ContainerExecutionException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 29c6fe9..2013306 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -282,6 +282,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
@Override
+ public boolean useWhitelistEnv(Map<String, String> env) {
+ // Avoid propagating nodemanager environment variables into the container
+ // so those variables can be picked up from the Docker image instead.
+ return false;
+ }
+
+ @Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
index 7caa0ed..aa294fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import java.util.Map;
+
/**
* An abstraction for various container runtime implementations. Examples
* include Process Tree, Docker, Appc runtimes etc. These implementations
@@ -83,4 +85,13 @@ public interface ContainerRuntime {
* and hostname
*/
String[] getIpAndHost(Container container) throws ContainerExecutionException;
+
+ /**
+ * Whether to propagate the whitelist of environment variables from the
+ * nodemanager environment into the container environment.
+ * @param env the container's environment variables
+ * @return true if whitelist variables should be propagated, false otherwise
+ * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST
+ */
+ boolean useWhitelistEnv(Map<String, String> env);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 588fdcd..1ee6eb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -91,6 +91,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
@@ -99,6 +100,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.ShellScriptBuilder;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
@@ -304,8 +307,18 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
FileOutputStream fos = new FileOutputStream(shellFile);
List<String> commands = new ArrayList<String>();
+ final Map<String, String> nmEnv = new HashMap<>();
+ nmEnv.put("HADOOP_COMMON_HOME", "nodemanager_common_home");
+ nmEnv.put("HADOOP_HDFS_HOME", "nodemanager_hdfs_home");
+ nmEnv.put("HADOOP_YARN_HOME", "nodemanager_yarn_home");
+ nmEnv.put("HADOOP_MAPRED_HOME", "nodemanager_mapred_home");
DefaultContainerExecutor defaultContainerExecutor =
- new DefaultContainerExecutor();
+ new DefaultContainerExecutor() {
+ @Override
+ protected String getNMEnvVar(String varname) {
+ return nmEnv.get(varname);
+ }
+ };
YarnConfiguration conf = new YarnConfiguration();
conf.set(YarnConfiguration.NM_ENV_WHITELIST,
"HADOOP_MAPRED_HOME,HADOOP_YARN_HOME");
@@ -317,10 +330,60 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
StandardCharsets.UTF_8);
Assert.assertTrue(shellContent
.contains("export HADOOP_COMMON_HOME=\"/opt/hadoopcommon\""));
- // Not available in env and whitelist
- Assert.assertTrue(shellContent.contains("export HADOOP_MAPRED_HOME="
- + "${HADOOP_MAPRED_HOME:-\"/opt/hadoopbuild\"}"));
- // Not available in env but in whitelist
+ // Whitelisted variable overridden by container
+ Assert.assertTrue(shellContent.contains(
+ "export HADOOP_MAPRED_HOME=\"/opt/hadoopbuild\""));
+ // Available in env but not in whitelist
+ Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME"));
+ // Available in env and in whitelist
+ Assert.assertTrue(shellContent.contains(
+ "export HADOOP_YARN_HOME=\"nodemanager_yarn_home\""));
+ fos.flush();
+ fos.close();
+ }
+
+ @Test(timeout = 20000)
+ public void testWriteEnvExportDocker() throws Exception {
+ // Valid only for unix
+ assumeNotWindows();
+ File shellFile = Shell.appendScriptExtension(tmpDir, "hello");
+ Map<String, String> env = new HashMap<String, String>();
+ env.put("HADOOP_COMMON_HOME", "/opt/hadoopcommon");
+ env.put("HADOOP_MAPRED_HOME", "/opt/hadoopbuild");
+ Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
+ FileOutputStream fos = new FileOutputStream(shellFile);
+ List<String> commands = new ArrayList<String>();
+ final Map<String, String> nmEnv = new HashMap<>();
+ nmEnv.put("HADOOP_COMMON_HOME", "nodemanager_common_home");
+ nmEnv.put("HADOOP_HDFS_HOME", "nodemanager_hdfs_home");
+ nmEnv.put("HADOOP_YARN_HOME", "nodemanager_yarn_home");
+ nmEnv.put("HADOOP_MAPRED_HOME", "nodemanager_mapred_home");
+ DockerLinuxContainerRuntime dockerRuntime =
+ new DockerLinuxContainerRuntime(
+ mock(PrivilegedOperationExecutor.class));
+ LinuxContainerExecutor lce =
+ new LinuxContainerExecutor(dockerRuntime) {
+ @Override
+ protected String getNMEnvVar(String varname) {
+ return nmEnv.get(varname);
+ }
+ };
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_ENV_WHITELIST,
+ "HADOOP_MAPRED_HOME,HADOOP_YARN_HOME");
+ lce.setConf(conf);
+ lce.writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()), "user");
+ String shellContent =
+ new String(Files.readAllBytes(Paths.get(shellFile.getAbsolutePath())),
+ StandardCharsets.UTF_8);
+ Assert.assertTrue(shellContent
+ .contains("export HADOOP_COMMON_HOME=\"/opt/hadoopcommon\""));
+ // Whitelisted variable overridden by container
+ Assert.assertTrue(shellContent.contains(
+ "export HADOOP_MAPRED_HOME=\"/opt/hadoopbuild\""));
+ // Verify no whitelisted variables inherited from NM env
+ Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME"));
Assert.assertFalse(shellContent.contains("HADOOP_YARN_HOME"));
fos.flush();
fos.close();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/35] hadoop git commit: YARN-7084.
TestSchedulingMonitor#testRMStarts fails sporadically. Contributed by Jason
Lowe.
Posted by ae...@apache.org.
YARN-7084. TestSchedulingMonitor#testRMStarts fails sporadically. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7f6118f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7f6118f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7f6118f9
Branch: refs/heads/HDFS-7240
Commit: 7f6118f918e0fbe81cb64e8b9f6959b8d66679f6
Parents: d3b1c63
Author: Eric Payne <ep...@apache.org>
Authored: Fri Sep 29 10:33:57 2017 -0500
Committer: Eric Payne <ep...@apache.org>
Committed: Fri Sep 29 10:33:57 2017 -0500
----------------------------------------------------------------------
.../monitor/TestSchedulingMonitor.java | 34 +++++++-------------
1 file changed, 11 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f6118f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
index d6ac346..c38236d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
@@ -25,42 +25,30 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy;
import org.junit.Test;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
public class TestSchedulingMonitor {
@Test(timeout = 10000)
- public void testRMStarts() {
+ public void testRMStarts() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
ResourceManager rm = new MockRM();
- try {
- rm.init(conf);
- } catch (Exception e) {
- fail("ResourceManager does not start when " +
- YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS + " is set to true");
- }
-
+ rm.init(conf);
SchedulingEditPolicy mPolicy = mock(SchedulingEditPolicy.class);
when(mPolicy.getMonitoringInterval()).thenReturn(1000L);
SchedulingMonitor monitor = new SchedulingMonitor(rm.getRMContext(),
mPolicy);
- try {
- monitor.serviceInit(conf);
- monitor.serviceStart();
- } catch (Exception e) {
- fail("SchedulingMonitor failes to start.");
- }
- verify(mPolicy, times(1)).editSchedule();
- try {
- monitor.close();
- rm.close();
- } catch (Exception e) {
- fail("Failed to close.");
- }
+ monitor.serviceInit(conf);
+ monitor.serviceStart();
+ verify(mPolicy, timeout(10000)).editSchedule();
+ monitor.close();
+ rm.close();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/35] hadoop git commit: YARN-7257. AggregatedLogsBlock reports a
bad 'end' value as a bad 'start' value. Contributed by Jason Lowe
Posted by ae...@apache.org.
YARN-7257. AggregatedLogsBlock reports a bad 'end' value as a bad
'start' value. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28c4957f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28c4957f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28c4957f
Branch: refs/heads/HDFS-7240
Commit: 28c4957fccebe2d7e63ec9fe9af58313b4f21d4f
Parents: c87db8d
Author: Xuan <xg...@apache.org>
Authored: Wed Sep 27 15:46:18 2017 -0700
Committer: Xuan <xg...@apache.org>
Committed: Wed Sep 27 15:46:18 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28c4957f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
index f6fea74..ef6876a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
@@ -67,7 +67,7 @@ public class AggregatedLogsBlock extends HtmlBlock {
LogAggregationWebUtils.getLogEndIndex(
html, $("end"));
} catch (NumberFormatException ne) {
- html.h1().__("Invalid log start value: " + $("end")).__();
+ html.h1().__("Invalid log end value: " + $("end")).__();
isValid = false;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/35] hadoop git commit: YARN-6623. Add support to turn off
launching privileged containers in the container-executor. (Varun Vasudev via
wangda)
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
new file mode 100644
index 0000000..860320d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -0,0 +1,998 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <ctype.h>
+#include "../modules/common/module-configs.h"
+#include "docker-util.h"
+#include "string-utils.h"
+#include "util.h"
+
+static int read_and_verify_command_file(const char *command_file, const char *docker_command,
+ struct configuration *command_config) {
+ int ret = 0;
+ ret = read_config(command_file, command_config);
+ if (ret != 0) {
+ return INVALID_COMMAND_FILE;
+ }
+ char *command = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, command_config);
+ if (command == NULL || (strcmp(command, docker_command) != 0)) {
+ ret = INCORRECT_COMMAND;
+ }
+ free(command);
+ return ret;
+}
+
+static int add_to_buffer(char *buff, const size_t bufflen, const char *string) {
+ size_t current_len = strlen(buff);
+ size_t string_len = strlen(string);
+ if (current_len + string_len < bufflen - 1) {
+ strncpy(buff + current_len, string, string_len);
+ buff[current_len + string_len] = '\0';
+ return 0;
+ }
+ return -1;
+}
+
+static int add_param_to_command(const struct configuration *command_config, const char *key, const char *param,
+ const int with_argument, char *out, const size_t outlen) {
+ size_t tmp_buffer_size = 4096;
+ int ret = 0;
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char *value = get_configuration_value(key, DOCKER_COMMAND_FILE_SECTION, command_config);
+ if (value != NULL) {
+ if (with_argument) {
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, param, value);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ } else if (strcmp(value, "true") == 0) {
+ ret = add_to_buffer(out, outlen, param);
+ }
+ free(value);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ }
+ }
+ free(tmp_buffer);
+ return ret;
+}
+
+static int add_param_to_command_if_allowed(const struct configuration *command_config,
+ const struct configuration *executor_cfg,
+ const char *key, const char *allowed_key, const char *param,
+ const int multiple_values, const char prefix,
+ char *out, const size_t outlen) {
+ size_t tmp_buffer_size = 4096;
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char *tmp_ptr = NULL;
+ char **values = NULL;
+ char **permitted_values = get_configuration_values_delimiter(allowed_key,
+ CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, executor_cfg,
+ ",");
+ int i = 0, j = 0, permitted = 0, ret = 0;
+ if (multiple_values) {
+ values = get_configuration_values_delimiter(key, DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ } else {
+ values = (char **) alloc_and_clear_memory(2, sizeof(char *));
+ values[0] = get_configuration_value(key, DOCKER_COMMAND_FILE_SECTION, command_config);
+ values[1] = NULL;
+ if (values[0] == NULL) {
+ ret = 0;
+ goto free_and_exit;
+ }
+ }
+
+ if (values != NULL) {
+ if (permitted_values != NULL) {
+ for (i = 0; values[i] != NULL; ++i) {
+ memset(tmp_buffer, 0, tmp_buffer_size);
+ permitted = 0;
+ if(prefix != 0) {
+ tmp_ptr = strchr(values[i], prefix);
+ if (tmp_ptr == NULL) {
+ fprintf(ERRORFILE, "Prefix char '%c' not found in '%s'\n",
+ prefix, values[i]);
+ ret = -1;
+ goto free_and_exit;
+ }
+ }
+ for (j = 0; permitted_values[j] != NULL; ++j) {
+ if (prefix == 0) {
+ ret = strcmp(values[i], permitted_values[j]);
+ } else {
+ ret = strncmp(values[i], permitted_values[j], tmp_ptr - values[i]);
+ }
+ if (ret == 0) {
+ permitted = 1;
+ break;
+ }
+ }
+ if (permitted == 1) {
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, param, values[i]);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Output buffer too small\n");
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
+ }
+ } else {
+ fprintf(ERRORFILE, "Invalid param '%s' requested\n", values[i]);
+ ret = -1;
+ goto free_and_exit;
+ }
+ }
+ } else {
+ fprintf(ERRORFILE, "Invalid param '%s' requested, "
+ "permitted values list is empty\n", values[0]);
+ ret = -1;
+ goto free_and_exit;
+ }
+ }
+
+ free_and_exit:
+ free_values(values);
+ free_values(permitted_values);
+ free(tmp_buffer);
+ if (ret != 0) {
+ memset(out, 0, outlen);
+ }
+ return ret;
+}
+
+static int add_docker_config_param(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "docker-config", "--config=", 1, out, outlen);
+}
+
+static int validate_container_name(const char *container_name) {
+ const char *CONTAINER_NAME_PREFIX = "container_";
+ if (0 == strncmp(container_name, CONTAINER_NAME_PREFIX, strlen(CONTAINER_NAME_PREFIX))) {
+ if (1 == validate_container_id(container_name)) {
+ return 0;
+ }
+ }
+ fprintf(ERRORFILE, "Specified container_id=%s is invalid\n", container_name);
+ fflush(ERRORFILE);
+ return INVALID_DOCKER_CONTAINER_NAME;
+}
+
+const char *get_docker_error_message(const int error_code) {
+
+ switch (error_code) {
+ case INVALID_COMMAND_FILE:
+ return "Invalid command file passed";
+ case INCORRECT_COMMAND:
+ return "Incorrect command";
+ case BUFFER_TOO_SMALL:
+ return "Command buffer too small";
+ case INVALID_DOCKER_CONTAINER_NAME:
+ return "Invalid docker container name";
+ case INVALID_DOCKER_IMAGE_NAME:
+ return "Invalid docker image name";
+ case INVALID_DOCKER_USER_NAME:
+ return "Invalid docker user name";
+ case INVALID_DOCKER_INSPECT_FORMAT:
+ return "Invalid docker inspect format";
+ case UNKNOWN_DOCKER_COMMAND:
+ return "Unknown docker command";
+ case INVALID_DOCKER_NETWORK:
+ return "Invalid docker network";
+ case INVALID_DOCKER_CAPABILITY:
+ return "Invalid docker capability";
+ case PRIVILEGED_CONTAINERS_DISABLED:
+ return "Privileged containers are disabled";
+ case INVALID_DOCKER_MOUNT:
+ return "Invalid docker mount";
+ case INVALID_DOCKER_RO_MOUNT:
+ return "Invalid docker read-only mount";
+ case INVALID_DOCKER_RW_MOUNT:
+ return "Invalid docker read-write mount";
+ case MOUNT_ACCESS_ERROR:
+ return "Mount access error";
+ case INVALID_DOCKER_DEVICE:
+ return "Invalid docker device";
+ default:
+ return "Unknown error";
+ }
+}
+
+char *get_docker_binary(const struct configuration *conf) {
+ char *docker_binary = NULL;
+ docker_binary = get_configuration_value(DOCKER_BINARY_KEY, CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
+ if (docker_binary == NULL) {
+ docker_binary = get_configuration_value(DOCKER_BINARY_KEY, "", conf);
+ if (docker_binary == NULL) {
+ docker_binary = strdup("/usr/bin/docker");
+ }
+ }
+ return docker_binary;
+}
+
+int docker_module_enabled(const struct configuration *conf) {
+ struct section *section = get_configuration_section(CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
+ if (section != NULL) {
+ return module_enabled(section, CONTAINER_EXECUTOR_CFG_DOCKER_SECTION);
+ }
+ return 0;
+}
+
+int get_docker_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ struct configuration command_config = {0, NULL};
+ ret = read_config(command_file, &command_config);
+ if (ret != 0) {
+ return INVALID_COMMAND_FILE;
+ }
+
+ char *command = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (strcmp(DOCKER_INSPECT_COMMAND, command) == 0) {
+ return get_docker_inspect_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_LOAD_COMMAND, command) == 0) {
+ return get_docker_load_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_PULL_COMMAND, command) == 0) {
+ return get_docker_pull_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_RM_COMMAND, command) == 0) {
+ return get_docker_rm_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_RUN_COMMAND, command) == 0) {
+ return get_docker_run_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_STOP_COMMAND, command) == 0) {
+ return get_docker_stop_command(command_file, conf, out, outlen);
+ } else {
+ return UNKNOWN_DOCKER_COMMAND;
+ }
+}
+
+int get_docker_inspect_command(const char *command_file, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ const char *valid_format_strings[] = { "{{.State.Status}}",
+ "{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}" };
+ int ret = 0, i = 0, valid_format = 0;
+ char *format = NULL, *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_INSPECT_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ format = get_configuration_value("format", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (format == NULL) {
+ free(container_name);
+ return INVALID_DOCKER_INSPECT_FORMAT;
+ }
+ for (i = 0; i < 2; ++i) {
+ if (strcmp(format, valid_format_strings[i]) == 0) {
+ valid_format = 1;
+ break;
+ }
+ }
+ if (valid_format != 1) {
+ fprintf(ERRORFILE, "Invalid format option '%s' not permitted\n", format);
+ free(container_name);
+ free(format);
+ return INVALID_DOCKER_INSPECT_FORMAT;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ free(container_name);
+ free(format);
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_INSPECT_COMMAND);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, " --format=");
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, format);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, container_name);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ free(format);
+ free(container_name);
+ return 0;
+
+ free_and_exit:
+ free(format);
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+}
+
+int get_docker_load_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *image_name = NULL;
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_LOAD_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (image_name == NULL) {
+ return INVALID_DOCKER_IMAGE_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ free(image_name);
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_LOAD_COMMAND);
+ if (ret == 0) {
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, " --i=", image_name);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ free(tmp_buffer);
+ free(image_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(image_name);
+ return BUFFER_TOO_SMALL;
+}
+
+static int validate_docker_image_name(const char *image_name) {
+ const char *regex_str = "^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$";
+ return execute_regex_match(regex_str, image_name);
+}
+
+int get_docker_pull_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *image_name = NULL;
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_PULL_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (image_name == NULL || validate_docker_image_name(image_name) != 0) {
+ return INVALID_DOCKER_IMAGE_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_PULL_COMMAND);
+ if (ret == 0) {
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, " ", image_name);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ free(tmp_buffer);
+ free(image_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(image_name);
+ return BUFFER_TOO_SMALL;
+}
+
+int get_docker_rm_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_RM_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_RM_COMMAND);
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, container_name);
+ }
+ free(container_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+}
+
+int get_docker_stop_command(const char *command_file, const struct configuration *conf,
+ char *out, const size_t outlen) {
+ int ret = 0;
+ size_t len = 0, i = 0;
+ char *value = NULL;
+ char *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_STOP_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_STOP_COMMAND);
+ if (ret == 0) {
+ value = get_configuration_value("time", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (value != NULL) {
+ len = strlen(value);
+ for (i = 0; i < len; ++i) {
+ if (isdigit(value[i]) == 0) {
+ fprintf(ERRORFILE, "Value for time is not a number '%s'\n", value);
+ free(container_name);
+ memset(out, 0, outlen);
+ return INVALID_DOCKER_STOP_COMMAND;
+ }
+ }
+ ret = add_to_buffer(out, outlen, " --time=");
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, value);
+ }
+ if (ret != 0) {
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+ }
+ }
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, container_name);
+ }
+ free(container_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+}
+
+static int detach_container(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "detach", "-d ", 0, out, outlen);
+}
+
+static int rm_container_on_exit(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "rm", "--rm ", 0, out, outlen);
+}
+
+static int set_container_workdir(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "workdir", "--workdir=", 1, out, outlen);
+}
+
+static int set_cgroup_parent(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "cgroup-parent", "--cgroup-parent=", 1, out, outlen);
+}
+
+static int set_hostname(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "hostname", "--hostname=", 1, out, outlen);
+}
+
+static int set_group_add(const struct configuration *command_config, char *out, const size_t outlen) {
+ int i = 0, ret = 0;
+ char **group_add = get_configuration_values_delimiter("group-add", DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ size_t tmp_buffer_size = 4096;
+ char *tmp_buffer = NULL;
+
+ if (group_add != NULL) {
+ for (i = 0; group_add[i] != NULL; ++i) {
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "--group-add ", group_add[i]);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ }
+ }
+ return ret;
+}
+
+static int set_network(const struct configuration *command_config,
+ const struct configuration *conf, char *out,
+ const size_t outlen) {
+
+ int ret = 0;
+ ret = add_param_to_command_if_allowed(command_config, conf, "net",
+ "docker.allowed.networks", "--net=",
+ 0, 0, out, outlen);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Could not find requested network in allowed networks\n");
+ ret = INVALID_DOCKER_NETWORK;
+ memset(out, 0, outlen);
+ }
+
+ return ret;
+}
+
+static int set_capabilities(const struct configuration *command_config,
+ const struct configuration *conf, char *out,
+ const size_t outlen) {
+
+ int ret = 0;
+
+ ret = add_to_buffer(out, outlen, "--cap-drop='ALL' ");
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ ret = add_param_to_command_if_allowed(command_config, conf, "cap-add",
+ "docker.allowed.capabilities",
+ "--cap-add=", 1, 0,
+ out, outlen);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Invalid docker capability requested\n");
+ ret = INVALID_DOCKER_CAPABILITY;
+ memset(out, 0, outlen);
+ }
+
+ return ret;
+}
+
+static int set_devices(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ int ret = 0;
+ ret = add_param_to_command_if_allowed(command_config, conf, "devices", "docker.allowed.devices", "--device=", 1, ':',
+ out, outlen);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Invalid docker device requested\n");
+ ret = INVALID_DOCKER_DEVICE;
+ memset(out, 0, outlen);
+ }
+
+ return ret;
+}
+
+/**
+ * Helper function to help normalize mounts for checking if mounts are
+ * permitted. The function does the following -
+ * 1. Find the canonical path for mount using realpath
+ * 2. If the path is a directory, add a '/' at the end (if not present)
+ * 3. Return a copy of the canonicalised path(to be freed by the caller)
+ * @param mount path to be canonicalised
+ * @return pointer to canonicalised path, NULL on error
+ */
+static char* normalize_mount(const char* mount) {
+ int ret = 0;
+ struct stat buff;
+ char *ret_ptr = NULL, *real_mount = NULL;
+ if (mount == NULL) {
+ return NULL;
+ }
+ real_mount = realpath(mount, NULL);
+ if (real_mount == NULL) {
+ fprintf(ERRORFILE, "Could not determine real path of mount '%s'\n", mount);
+ free(real_mount);
+ return NULL;
+ }
+ ret = stat(real_mount, &buff);
+ if (ret == 0) {
+ if (S_ISDIR(buff.st_mode)) {
+ size_t len = strlen(real_mount);
+ if (len <= 0) {
+ return NULL;
+ }
+ if (real_mount[len - 1] != '/') {
+ ret_ptr = (char *) alloc_and_clear_memory(len + 2, sizeof(char));
+ strncpy(ret_ptr, real_mount, len);
+ ret_ptr[len] = '/';
+ ret_ptr[len + 1] = '\0';
+ } else {
+ ret_ptr = strdup(real_mount);
+ }
+ } else {
+ ret_ptr = strdup(real_mount);
+ }
+ } else {
+ fprintf(ERRORFILE, "Could not stat path '%s'\n", real_mount);
+ ret_ptr = NULL;
+ }
+ free(real_mount);
+ return ret_ptr;
+}
+
+static int normalize_mounts(char **mounts) {
+ int i = 0;
+ char *tmp = NULL;
+ if (mounts == NULL) {
+ return 0;
+ }
+ for (i = 0; mounts[i] != NULL; ++i) {
+ tmp = normalize_mount(mounts[i]);
+ if (tmp == NULL) {
+ return -1;
+ }
+ free(mounts[i]);
+ mounts[i] = tmp;
+ }
+ return 0;
+}
+
+static int check_mount_permitted(const char **permitted_mounts, const char *requested) {
+ int i = 0, ret = 0;
+ size_t permitted_mount_len = 0;
+ char *normalized_path = normalize_mount(requested);
+ if (permitted_mounts == NULL) {
+ return 0;
+ }
+ if (normalized_path == NULL) {
+ return -1;
+ }
+ for (i = 0; permitted_mounts[i] != NULL; ++i) {
+ if (strcmp(normalized_path, permitted_mounts[i]) == 0) {
+ ret = 1;
+ break;
+ }
+ // directory check
+ permitted_mount_len = strlen(permitted_mounts[i]);
+ if (permitted_mount_len > 0
+ && permitted_mounts[i][permitted_mount_len - 1] == '/') {
+ if (strncmp(normalized_path, permitted_mounts[i], permitted_mount_len) == 0) {
+ ret = 1;
+ break;
+ }
+ }
+
+ }
+ free(normalized_path);
+ return ret;
+}
+
+static char* get_mount_source(const char *mount) {
+ char *src_mount = NULL;
+ const char *tmp = NULL;
+ tmp = strchr(mount, ':');
+ if (tmp == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s'\n", mount);
+ return NULL;
+ }
+ src_mount = strndup(mount, tmp - mount);
+ return src_mount;
+}
+
+static int add_mounts(const struct configuration *command_config, const struct configuration *conf, const char *key,
+ const int ro, char *out, const size_t outlen) {
+ size_t tmp_buffer_size = 1024;
+ const char *ro_suffix = "";
+ const char *tmp_path_buffer[2] = {NULL, NULL};
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char **permitted_ro_mounts = get_configuration_values_delimiter("docker.allowed.ro-mounts",
+ CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
+ char **permitted_rw_mounts = get_configuration_values_delimiter("docker.allowed.rw-mounts",
+ CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
+ char **values = get_configuration_values_delimiter(key, DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ char *tmp_buffer_2 = NULL, *mount_src = NULL;
+ const char *container_executor_cfg_path = normalize_mount(get_config_path(""));
+ int i = 0, permitted_rw = 0, permitted_ro = 0, ret = 0;
+ if (ro != 0) {
+ ro_suffix = ":ro";
+ }
+
+ if (values != NULL) {
+ ret = normalize_mounts(permitted_ro_mounts);
+ ret |= normalize_mounts(permitted_rw_mounts);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Unable to find permitted docker mounts on disk\n");
+ ret = MOUNT_ACCESS_ERROR;
+ goto free_and_exit;
+ }
+ for (i = 0; values[i] != NULL; ++i) {
+ mount_src = get_mount_source(values[i]);
+ if (mount_src == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_MOUNT;
+ goto free_and_exit;
+ }
+ permitted_rw = check_mount_permitted((const char **) permitted_rw_mounts, mount_src);
+ permitted_ro = check_mount_permitted((const char **) permitted_ro_mounts, mount_src);
+ if (permitted_ro == -1 || permitted_rw == -1) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_MOUNT;
+ goto free_and_exit;
+ }
+ // rw mount
+ if (ro == 0) {
+ if (permitted_rw == 0) {
+ fprintf(ERRORFILE, "Invalid docker rw mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_RW_MOUNT;
+ goto free_and_exit;
+ } else {
+ // determine if the user can modify the container-executor.cfg file
+ tmp_path_buffer[0] = normalize_mount(mount_src);
+ // just re-use the function, flip the args to check if the container-executor path is in the requested
+ // mount point
+ ret = check_mount_permitted(tmp_path_buffer, container_executor_cfg_path);
+ free((void *) tmp_path_buffer[0]);
+ if (ret == 1) {
+ fprintf(ERRORFILE, "Attempting to mount a parent directory '%s' of container-executor.cfg as read-write\n",
+ values[i]);
+ ret = INVALID_DOCKER_RW_MOUNT;
+ goto free_and_exit;
+ }
+ }
+ }
+ //ro mount
+ if (ro != 0 && permitted_ro == 0 && permitted_rw == 0) {
+ fprintf(ERRORFILE, "Invalid docker ro mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_RO_MOUNT;
+ goto free_and_exit;
+ }
+ tmp_buffer_2 = (char *) alloc_and_clear_memory(strlen(values[i]) + strlen(ro_suffix) + 1, sizeof(char));
+ strncpy(tmp_buffer_2, values[i], strlen(values[i]));
+ strncpy(tmp_buffer_2 + strlen(values[i]), ro_suffix, strlen(ro_suffix));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "-v ", tmp_buffer_2);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ free(tmp_buffer_2);
+ free(mount_src);
+ tmp_buffer_2 = NULL;
+ mount_src = NULL;
+ memset(tmp_buffer, 0, tmp_buffer_size);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
+ }
+ }
+ }
+
+ free_and_exit:
+ free_values(permitted_ro_mounts);
+ free_values(permitted_rw_mounts);
+ free_values(values);
+ free(mount_src);
+ free((void *) container_executor_cfg_path);
+ free(tmp_buffer);
+ if (ret != 0) {
+ memset(out, 0, outlen);
+ }
+ return ret;
+}
+
+static int add_ro_mounts(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ return add_mounts(command_config, conf, "ro-mounts", 1, out, outlen);
+}
+
+static int add_rw_mounts(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ return add_mounts(command_config, conf, "rw-mounts", 0, out, outlen);
+}
+
+static int set_privileged(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char *value = get_configuration_value("privileged", DOCKER_COMMAND_FILE_SECTION, command_config);
+ char *privileged_container_enabled
+ = get_configuration_value("docker.privileged-containers.enabled", CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
+ int ret = 0;
+
+ if (value != NULL && strcmp(value, "true") == 0) {
+ if (privileged_container_enabled != NULL) {
+ if (strcmp(privileged_container_enabled, "1") == 0) {
+ ret = add_to_buffer(out, outlen, "--privileged ");
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ }
+ } else {
+ fprintf(ERRORFILE, "Privileged containers are disabled\n");
+ ret = PRIVILEGED_CONTAINERS_DISABLED;
+ goto free_and_exit;
+ }
+ } else {
+ fprintf(ERRORFILE, "Privileged containers are disabled\n");
+ ret = PRIVILEGED_CONTAINERS_DISABLED;
+ goto free_and_exit;
+ }
+ }
+
+ free_and_exit:
+ free(tmp_buffer);
+ free(value);
+ free(privileged_container_enabled);
+ if (ret != 0) {
+ memset(out, 0, outlen);
+ }
+ return ret;
+}
+
+int get_docker_run_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0, i = 0;
+ char *container_name = NULL, *user = NULL, *image = NULL;
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = NULL;
+ char **launch_command = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_RUN_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+ user = get_configuration_value("user", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (user == NULL) {
+ return INVALID_DOCKER_USER_NAME;
+ }
+ image = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (image == NULL || validate_docker_image_name(image) != 0) {
+ return INVALID_DOCKER_IMAGE_NAME;
+ }
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_RUN_COMMAND);
+ if(ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, " --name=", container_name);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ memset(tmp_buffer, 0, tmp_buffer_size);
+
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "--user=", user);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ memset(tmp_buffer, 0, tmp_buffer_size);
+
+ ret = detach_container(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = rm_container_on_exit(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_container_workdir(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_network(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = add_ro_mounts(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = add_rw_mounts(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_cgroup_parent(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_privileged(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_capabilities(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_hostname(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_group_add(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_devices(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "", image);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ launch_command = get_configuration_values_delimiter("launch-command", DOCKER_COMMAND_FILE_SECTION, &command_config,
+ ",");
+ if (launch_command != NULL) {
+ for (i = 0; launch_command[i] != NULL; ++i) {
+ memset(tmp_buffer, 0, tmp_buffer_size);
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "", launch_command[i]);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ free_values(launch_command);
+ free(tmp_buffer);
+ return BUFFER_TOO_SMALL;
+ }
+ }
+ free_values(launch_command);
+ }
+ free(tmp_buffer);
+ return 0;
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
new file mode 100644
index 0000000..37ec880
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_DOCKER_UTIL_H__
+#define __YARN_POSIX_CONTAINER_EXECUTOR_DOCKER_UTIL_H__
+
+#include "configuration.h"
+
+#define CONTAINER_EXECUTOR_CFG_DOCKER_SECTION "docker"
+#define DOCKER_BINARY_KEY "docker.binary"
+#define DOCKER_COMMAND_FILE_SECTION "docker-command-execution"
+#define DOCKER_INSPECT_COMMAND "inspect"
+#define DOCKER_LOAD_COMMAND "load"
+#define DOCKER_PULL_COMMAND "pull"
+#define DOCKER_RM_COMMAND "rm"
+#define DOCKER_RUN_COMMAND "run"
+#define DOCKER_STOP_COMMAND "stop"
+
+
+enum docker_error_codes {
+ INVALID_COMMAND_FILE = 1,
+ INCORRECT_COMMAND,
+ BUFFER_TOO_SMALL,
+ INVALID_DOCKER_CONTAINER_NAME,
+ INVALID_DOCKER_IMAGE_NAME,
+ INVALID_DOCKER_USER_NAME,
+ INVALID_DOCKER_INSPECT_FORMAT,
+ UNKNOWN_DOCKER_COMMAND,
+ INVALID_DOCKER_NETWORK,
+ INVALID_DOCKER_CAPABILITY,
+ PRIVILEGED_CONTAINERS_DISABLED,
+ INVALID_DOCKER_MOUNT,
+ INVALID_DOCKER_RO_MOUNT,
+ INVALID_DOCKER_RW_MOUNT,
+ MOUNT_ACCESS_ERROR,
+ INVALID_DOCKER_DEVICE,
+ INVALID_DOCKER_STOP_COMMAND
+};
+
+/**
+ * Get the full path for the docker binary.
+ * @param conf Configuration for the container-executor
+ * @return String containing the docker binary to be freed by the user.
+ */
+char *get_docker_binary(const struct configuration *conf);
+
+/**
+ * Get the Docker command line string. The function will inspect the params file to determine the command to be run.
+ * @param command_file File containing the params for the Docker command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the Docker command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker inspect command line string. The function will verify that the params file is meant for the
+ * inspect command.
+ * @param command_file File containing the params for the Docker inspect command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the inspect command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_inspect_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker load command line string. The function will verify that the params file is meant for the load command.
+ * @param command_file File containing the params for the Docker load command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the load command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_load_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker pull command line string. The function will verify that the params file is meant for the pull command.
+ * @param command_file File containing the params for the Docker pull command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the pull command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_pull_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker rm command line string. The function will verify that the params file is meant for the rm command.
+ * @param command_file File containing the params for the Docker rm command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the rm command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_rm_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker run command line string. The function will verify that the params file is meant for the run command.
+ * @param command_file File containing the params for the Docker run command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the run command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_run_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker stop command line string. The function will verify that the params file is meant for the stop command.
+ * @param command_file File containing the params for the Docker stop command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the stop command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_stop_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Give an error message for the supplied error code
+ * @param error_code the error code
+ * @return const string containing the error message
+ */
+const char *get_docker_error_message(const int error_code);
+
+/**
+ * Determine if the docker module is enabled in the config.
+ * @param conf Configuration structure
+ * @return 1 if enabled, 0 otherwise
+ */
+int docker_module_enabled(const struct configuration *conf);
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
index d19c084..40b2c25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
@@ -21,7 +21,6 @@
#include <errno.h>
#include <strings.h>
#include <string.h>
-#include <stdio.h>
#include <stdlib.h>
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg
new file mode 100644
index 0000000..339bec9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg
@@ -0,0 +1,13 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[docker]
+privieged-containers.enabled=0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 64ee717..609dd2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -19,6 +19,7 @@
#include "container-executor.h"
#include "utils/string-utils.h"
#include "util.h"
+#include "get_executable.h"
#include "test/test-container-executor-common.h"
#include <inttypes.h>
@@ -1155,145 +1156,6 @@ void test_trim_function() {
free(trimmed);
}
-void test_sanitize_docker_command() {
-
- char *input[] = {
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run ''''''''",
- "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}' container_e111_1111111111111_1111_01_111111",
- "rm container_e111_1111111111111_1111_01_111111",
- "stop container_e111_1111111111111_1111_01_111111",
- "pull ubuntu",
- "pull registry.com/user/ubuntu",
- "--config=/yarn/local/cdir/ pull registry.com/user/ubuntu"
- };
- char *expected_output[] = {
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run ''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"'' ",
- "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}' container_e111_1111111111111_1111_01_111111",
- "rm container_e111_1111111111111_1111_01_111111",
- "stop container_e111_1111111111111_1111_01_111111",
- "pull ubuntu",
- "pull registry.com/user/ubuntu",
- "--config=/yarn/local/cdir/ pull registry.com/user/ubuntu"
- };
-
- int input_size = sizeof(input) / sizeof(char *);
- int i = 0;
- for(i = 0; i < input_size; i++) {
- char *command = (char *) calloc(strlen(input[i]) + 1 , sizeof(char));
- strncpy(command, input[i], strlen(input[i]));
- char *op = sanitize_docker_command(command);
- if(strncmp(expected_output[i], op, strlen(expected_output[i])) != 0) {
- printf("FAIL: expected output %s does not match actual output '%s'\n", expected_output[i], op);
- exit(1);
- }
- free(command);
- }
-}
-
-void test_validate_docker_image_name() {
-
- char *good_input[] = {
- "ubuntu",
- "ubuntu:latest",
- "ubuntu:14.04",
- "ubuntu:LATEST",
- "registry.com:5000/user/ubuntu",
- "registry.com:5000/user/ubuntu:latest",
- "registry.com:5000/user/ubuntu:0.1.2.3",
- "registry.com/user/ubuntu",
- "registry.com/user/ubuntu:latest",
- "registry.com/user/ubuntu:0.1.2.3",
- "registry.com/user/ubuntu:test-image",
- "registry.com/user/ubuntu:test_image",
- "registry.com/ubuntu",
- "user/ubuntu",
- "user/ubuntu:0.1.2.3",
- "user/ubuntu:latest",
- "user/ubuntu:test_image",
- "user/ubuntu.test:test_image",
- "user/ubuntu-test:test-image",
- "registry.com/ubuntu/ubuntu/ubuntu"
- };
-
- char *bad_input[] = {
- "UBUNTU",
- "registry.com|5000/user/ubuntu",
- "registry.com | 5000/user/ubuntu",
- "ubuntu' || touch /tmp/file #",
- "ubuntu || touch /tmp/file #",
- "''''''''",
- "bad_host_name:5000/user/ubuntu",
- "registry.com:foo/ubuntu/ubuntu/ubuntu",
- "registry.com/ubuntu:foo/ubuntu/ubuntu"
- };
-
- int good_input_size = sizeof(good_input) / sizeof(char *);
- int i = 0;
- for(i = 0; i < good_input_size; i++) {
- int op = validate_docker_image_name(good_input[i]);
- if(0 != op) {
- printf("\nFAIL: docker image name %s is invalid", good_input[i]);
- exit(1);
- }
- }
-
- int bad_input_size = sizeof(bad_input) / sizeof(char *);
- int j = 0;
- for(j = 0; j < bad_input_size; j++) {
- int op = validate_docker_image_name(bad_input[j]);
- if(1 != op) {
- printf("\nFAIL: docker image name %s is valid, expected invalid", bad_input[j]);
- exit(1);
- }
- }
-}
-
-void test_validate_container_id() {
- char *good_input[] = {
- "container_e134_1499953498516_50875_01_000007",
- "container_1499953498516_50875_01_000007",
- "container_e1_12312_11111_02_000001"
- };
-
- char *bad_input[] = {
- "CONTAINER",
- "container_e1_12312_11111_02_000001 | /tmp/file"
- "container_e1_12312_11111_02_000001 || # /tmp/file",
- "container_e1_12312_11111_02_000001 # /tmp/file",
- "container_e1_12312_11111_02_000001' || touch /tmp/file #",
- "ubuntu || touch /tmp/file #",
- "''''''''"
- };
-
- int good_input_size = sizeof(good_input) / sizeof(char *);
- int i = 0;
- for(i = 0; i < good_input_size; i++) {
- int op = validate_container_id(good_input[i]);
- if(1 != op) {
- printf("FAIL: docker container name %s is invalid\n", good_input[i]);
- exit(1);
- }
- }
-
- int bad_input_size = sizeof(bad_input) / sizeof(char *);
- int j = 0;
- for(j = 0; j < bad_input_size; j++) {
- int op = validate_container_id(bad_input[j]);
- if(0 != op) {
- printf("FAIL: docker container name %s is valid, expected invalid\n", bad_input[j]);
- exit(1);
- }
- }
-}
-
// This test is expected to be executed either by a regular
// user or by root. If executed by a regular user it doesn't
// test all the functions that would depend on changing the
@@ -1388,15 +1250,6 @@ int main(int argc, char **argv) {
printf("\nTesting is_feature_enabled()\n");
test_is_feature_enabled();
- printf("\nTesting sanitize docker commands()\n");
- test_sanitize_docker_command();
-
- printf("\nTesting validate_docker_image_name()\n");
- test_validate_docker_image_name();
-
- printf("\nTesting validate_container_id()\n");
- test_validate_container_id();
-
test_check_user(0);
#ifdef __APPLE__
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
index 2ec7b2a..b96dea1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
@@ -17,7 +17,7 @@
*/
#include <gtest/gtest.h>
-#include <sstream>
+#include <vector>
extern "C" {
#include "util.h"
@@ -135,4 +135,39 @@ namespace ContainerExecutor {
ASSERT_STREQ("foo", trimmed);
free(trimmed);
}
+
+ TEST_F(TestUtil, test_escape_single_quote) {
+ std::vector<std::pair<std::string, std::string> > input_output_vec;
+ input_output_vec.push_back(std::make_pair<std::string, std::string>("'abcd'", "'\"'\"'abcd'\"'\"'"));
+ input_output_vec.push_back(std::make_pair<std::string, std::string>("'", "'\"'\"'"));
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ for (itr = input_output_vec.begin(); itr != input_output_vec.end(); ++itr) {
+ char *ret = escape_single_quote(itr->first.c_str());
+ ASSERT_STREQ(itr->second.c_str(), ret);
+ free(ret);
+ }
+ }
+
+ TEST_F(TestUtil, test_quote_and_append_arg) {
+
+ char *tmp = static_cast<char *>(malloc(4096));
+ size_t tmp_size = 4096;
+
+ memset(tmp, 0, tmp_size);
+ quote_and_append_arg(&tmp, &tmp_size, "param=", "argument1");
+ ASSERT_STREQ("param='argument1' ", tmp);
+
+ memset(tmp, 0, tmp_size);
+ quote_and_append_arg(&tmp, &tmp_size, "param=", "ab'cd");
+ ASSERT_STREQ("param='ab'\"'\"'cd' ", tmp);
+ free(tmp);
+
+ tmp = static_cast<char *>(malloc(4));
+ tmp_size = 4;
+ memset(tmp, 0, tmp_size);
+ quote_and_append_arg(&tmp, &tmp_size, "param=", "argument1");
+ ASSERT_STREQ("param='argument1' ", tmp);
+ ASSERT_EQ(1040, tmp_size);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
index 037816a..b259c6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
@@ -88,6 +88,39 @@
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
std::cout << "Testing input=" << input << "\n";
ASSERT_TRUE(0 != rc) << "Should failed\n";
-}
+ }
+
+ TEST_F(TestStringUtils, test_validate_container_id) {
+
+ const char *good_input[] = {
+ "container_e134_1499953498516_50875_01_000007",
+ "container_1499953498516_50875_01_000007",
+ "container_e1_12312_11111_02_000001"
+ };
+
+ const char *bad_input[] = {
+ "CONTAINER",
+ "container_e1_12312_11111_02_000001 | /tmp/file"
+ "container_e1_12312_11111_02_000001 || # /tmp/file",
+ "container_e1_12312_11111_02_000001 # /tmp/file",
+ "container_e1_12312_11111_02_000001' || touch /tmp/file #",
+ "ubuntu || touch /tmp/file #",
+ "''''''''"
+ };
+
+ int good_input_size = sizeof(good_input) / sizeof(char *);
+ int i = 0;
+ for (i = 0; i < good_input_size; i++) {
+ int op = validate_container_id(good_input[i]);
+ ASSERT_EQ(1, op);
+ }
+
+ int bad_input_size = sizeof(bad_input) / sizeof(char *);
+ int j = 0;
+ for (j = 0; j < bad_input_size; j++) {
+ int op = validate_container_id(bad_input[j]);
+ ASSERT_EQ(0, op);
+ }
+ }
-} // namespace ContainerExecutor
\ No newline at end of file
+} // namespace ContainerExecutor
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/35] hadoop git commit: YARN-6871. Add additional deSelects params
in RMWebServices#getAppReport. Contributed by Tanuj Nayak.
Posted by ae...@apache.org.
YARN-6871. Add additional deSelects params in RMWebServices#getAppReport. Contributed by Tanuj Nayak.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8facf1f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8facf1f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8facf1f9
Branch: refs/heads/HDFS-7240
Commit: 8facf1f976d7e12a846f12baabf54be1b7a49f9d
Parents: dd37877
Author: Sunil G <su...@apache.org>
Authored: Wed Sep 27 14:37:32 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Wed Sep 27 14:37:32 2017 +0530
----------------------------------------------------------------------
.../resourcemanager/webapp/DeSelectFields.java | 11 +-
.../resourcemanager/webapp/dao/AppInfo.java | 108 +++++++++++++------
.../webapp/TestRMWebServicesApps.java | 77 ++++++++++++-
3 files changed, 160 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8facf1f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
index 258bbfa..c991766 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
@@ -91,7 +91,16 @@ public class DeSelectFields {
* <code>RESOURCE_REQUESTS</code> is the first
* supported type from YARN-6280.
*/
- RESOURCE_REQUESTS("resourceRequests");
+ RESOURCE_REQUESTS("resourceRequests"),
+ /**
+ * <code>APP_TIMEOUTS, APP_NODE_LABEL_EXPRESSION, AM_NODE_LABEL_EXPRESSION,
+ * RESOURCE_INFO</code> are additionally supported parameters added in
+ * YARN-6871.
+ */
+ TIMEOUTS("timeouts"),
+ APP_NODE_LABEL_EXPRESSION("appNodeLabelExpression"),
+ AM_NODE_LABEL_EXPRESSION("amNodeLabelExpression"),
+ RESOURCE_INFO("resourceInfo");
private final String literals;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8facf1f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 236c467..40d7b4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -123,7 +123,7 @@ public class AppInfo {
protected String amNodeLabelExpression;
protected ResourcesInfo resourceInfo = null;
- protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo();
+ private AppTimeoutsInfo timeouts;
public AppInfo() {
} // JAXB needs this
@@ -247,47 +247,87 @@ public class AppInfo {
unmanagedApplication = appSubmissionContext.getUnmanagedAM();
appNodeLabelExpression =
app.getApplicationSubmissionContext().getNodeLabelExpression();
- amNodeLabelExpression = (unmanagedApplication) ? null
- : app.getAMResourceRequests().get(0).getNodeLabelExpression();
+ /*
+ * When the deSelects parameter contains "amNodeLabelExpression", objects
+ * pertaining to the amNodeLabelExpression are not returned. By default,
+ * this is not skipped. (YARN-6871)
+ */
+ if(!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) {
+ amNodeLabelExpression = (unmanagedApplication) ?
+ null :
+ app.getAMResourceRequests().get(0).getNodeLabelExpression();
+ }
+ /*
+ * When the deSelects parameter contains "appNodeLabelExpression", objects
+ * pertaining to the appNodeLabelExpression are not returned. By default,
+ * this is not skipped. (YARN-6871)
+ */
+ if (!deSelects.contains(DeSelectType.APP_NODE_LABEL_EXPRESSION)) {
+ appNodeLabelExpression =
+ app.getApplicationSubmissionContext().getNodeLabelExpression();
+ }
+ /*
+ * When the deSelects parameter contains "amNodeLabelExpression", objects
+ * pertaining to the amNodeLabelExpression are not returned. By default,
+ * this is not skipped. (YARN-6871)
+ */
+ if (!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) {
+ amNodeLabelExpression = (unmanagedApplication) ?
+ null :
+ app.getAMResourceRequests().get(0).getNodeLabelExpression();
+ }
+ /*
+ * When the deSelects parameter contains "resourceInfo", ResourceInfo
+ * objects are not returned. Default behavior is no skipping. (YARN-6871)
+ */
// Setting partition based resource usage of application
- ResourceScheduler scheduler = rm.getRMContext().getScheduler();
- if (scheduler instanceof CapacityScheduler) {
- RMAppAttempt attempt = app.getCurrentAppAttempt();
- if (null != attempt) {
- FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler)
- .getApplicationAttempt(attempt.getAppAttemptId());
- resourceInfo = null != ficaAppAttempt
- ? new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage())
- : null;
+ if (!deSelects.contains(DeSelectType.RESOURCE_INFO)) {
+ ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+ if (scheduler instanceof CapacityScheduler) {
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ if (null != attempt) {
+ FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(attempt.getAppAttemptId());
+ resourceInfo = null != ficaAppAttempt ?
+ new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) :
+ null;
+ }
}
}
- Map<ApplicationTimeoutType, Long> applicationTimeouts =
- app.getApplicationTimeouts();
- if (applicationTimeouts.isEmpty()) {
- // If application is not set timeout, lifetime should be sent as default
- // with expiryTime=UNLIMITED and remainingTime=-1
- AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
- timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
- timeouts.add(timeoutInfo);
- } else {
- for (Map.Entry<ApplicationTimeoutType, Long> entry : app
- .getApplicationTimeouts().entrySet()) {
- AppTimeoutInfo timeout = new AppTimeoutInfo();
- timeout.setTimeoutType(entry.getKey());
- long timeoutInMillis = entry.getValue().longValue();
- timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
- if (app.isAppInCompletedStates()) {
- timeout.setRemainingTime(0);
- } else {
- timeout.setRemainingTime(Math
- .max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
+ /*
+ * When the deSelects parameter contains "appTimeouts", objects pertaining
+ * to app timeouts are not returned. By default, this is not skipped.
+ * (YARN-6871)
+ */
+ if (!deSelects.contains(DeSelectType.TIMEOUTS)) {
+ Map<ApplicationTimeoutType, Long> applicationTimeouts =
+ app.getApplicationTimeouts();
+ if (applicationTimeouts.isEmpty()) {
+ // If application is not set timeout, lifetime should be sent
+ // as default with expiryTime=UNLIMITED and remainingTime=-1
+ AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
+ timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
+ timeouts = new AppTimeoutsInfo();
+ timeouts.add(timeoutInfo);
+ } else {
+ for (Map.Entry<ApplicationTimeoutType, Long> entry : app
+ .getApplicationTimeouts().entrySet()) {
+ AppTimeoutInfo timeout = new AppTimeoutInfo();
+ timeout.setTimeoutType(entry.getKey());
+ long timeoutInMillis = entry.getValue().longValue();
+ timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
+ if (app.isAppInCompletedStates()) {
+ timeout.setRemainingTime(0);
+ } else {
+ timeout.setRemainingTime(Math.max(
+ (timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
+ }
+ timeouts.add(timeout);
}
- timeouts.add(timeout);
}
}
-
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8facf1f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 07f74a3..f0704ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1113,8 +1113,83 @@ public class TestRMWebServicesApps extends JerseyTestBase {
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
JSONObject app = array.getJSONObject(0);
- assertTrue("resource requests shouldn't exits",
+ assertTrue("resource requests shouldn't exist",
!app.has("resourceRequests"));
+
+ params.clear();
+ params.add("deSelects",
+ DeSelectFields.DeSelectType.AM_NODE_LABEL_EXPRESSION.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("AMNodeLabelExpression shouldn't exist",
+ !app.has("amNodeLabelExpression"));
+
+ params.clear();
+ params.add("deSelects", DeSelectFields.DeSelectType.TIMEOUTS.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("Timeouts shouldn't exist", !app.has("timeouts"));
+ rm.stop();
+
+ params.clear();
+ params.add("deSelects",
+ DeSelectFields.DeSelectType.APP_NODE_LABEL_EXPRESSION.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("AppNodeLabelExpression shouldn't exist",
+ !app.has("appNodeLabelExpression"));
+ rm.stop();
+
+ params.clear();
+ params
+ .add("deSelects", DeSelectFields.DeSelectType.RESOURCE_INFO.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("Resource info shouldn't exist", !app.has("resourceInfo"));
rm.stop();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/35] hadoop git commit: HDFS-12569. Unset EC policy logs empty
payload in edit log. (Lei (Eddy) Xu)
Posted by ae...@apache.org.
HDFS-12569. Unset EC policy logs empty payload in edit log. (Lei (Eddy) Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27ffd43b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27ffd43b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27ffd43b
Branch: refs/heads/HDFS-7240
Commit: 27ffd43b6419c9ebe697536bcb6abb858ce791d2
Parents: 280080f
Author: Lei Xu <le...@apache.org>
Authored: Mon Oct 2 15:31:20 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Mon Oct 2 15:35:49 2017 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/FSDirErasureCodingOp.java | 3 +--
.../hadoop/hdfs/server/namenode/FSDirXAttrOp.java | 4 ++++
.../hdfs/TestUnsetAndChangeDirectoryEcPolicy.java | 13 +++++++++++++
3 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffd43b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 181b147..391e392 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -298,8 +298,7 @@ final class FSDirErasureCodingOp {
final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
xattrs.add(ecXAttr);
- FSDirXAttrOp.unprotectedRemoveXAttrs(fsd, srcIIP, xattrs);
- return xattrs;
+ return FSDirXAttrOp.unprotectedRemoveXAttrs(fsd, srcIIP, xattrs);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffd43b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index acdade7..3223467 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -184,6 +184,10 @@ class FSDirXAttrOp {
return fsd.getAuditFileInfo(iip);
}
+ /**
+ * Remove xattrs from the inode, and return the <em>removed</em> xattrs.
+ * @return the <em>removed</em> xattrs.
+ */
static List<XAttr> unprotectedRemoveXAttrs(
FSDirectory fsd, final INodesInPath iip, final List<XAttr> toRemove)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffd43b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
index 529a110..52cf163 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
@@ -362,4 +362,17 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
+ ecFilePath, e);
}
}
+
+ /**
+ * Test unsetEcPolicy is persisted correctly in edit log.
+ */
+ @Test
+ public void testUnsetEcPolicyInEditLog() throws IOException {
+ fs.getClient().setErasureCodingPolicy("/", ecPolicy.getName());
+ Assert.assertEquals(ecPolicy, fs.getErasureCodingPolicy(new Path("/")));
+ fs.getClient().unsetErasureCodingPolicy("/");
+
+ cluster.restartNameNode(true);
+ Assert.assertNull(fs.getErasureCodingPolicy(new Path("/")));
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/35] hadoop git commit: HADOOP-14752.
TestCopyFromLocal#testCopyFromLocalWithThreads is fleaky. Contributed by
Andras Bokor.
Posted by ae...@apache.org.
HADOOP-14752. TestCopyFromLocal#testCopyFromLocalWithThreads is fleaky. Contributed by Andras Bokor.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/442ea855
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/442ea855
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/442ea855
Branch: refs/heads/HDFS-7240
Commit: 442ea85545ce3591f45211773a10346340abaed5
Parents: ceca969
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Oct 2 13:11:47 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Oct 2 13:11:47 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/shell/TestCopyFromLocal.java | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/442ea855/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
index 8d354b4..8e60540 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
@@ -115,10 +115,10 @@ public class TestCopyFromLocal {
Path dir = new Path("dir" + RandomStringUtils.randomNumeric(4));
int numFiles = TestCopyFromLocal.initialize(dir);
int maxThreads = Runtime.getRuntime().availableProcessors() * 2;
- int randThreads = RandomUtils.nextInt(maxThreads);
- int numActualThreads = randThreads == 0 ? 1 : randThreads;
- String numThreads = Integer.toString(numActualThreads);
- run(new TestMultiThreadedCopy(numActualThreads, numFiles), "-t", numThreads,
+ int randThreads = RandomUtils.nextInt(maxThreads - 1) + 1;
+ String numThreads = Integer.toString(randThreads);
+ run(new TestMultiThreadedCopy(randThreads,
+ randThreads == 1 ? 0 : numFiles), "-t", numThreads,
new Path(dir, FROM_DIR_NAME).toString(),
new Path(dir, TO_DIR_NAME).toString());
}
@@ -144,6 +144,7 @@ public class TestCopyFromLocal {
}
private class TestMultiThreadedCopy extends CopyFromLocal {
+ public static final String NAME = "testCopyFromLocal";
private int expectedThreads;
private int expectedCompletedTaskCount;
@@ -164,9 +165,9 @@ public class TestCopyFromLocal {
// 2) There are no active tasks in the executor
// 3) Executor has shutdown correctly
ThreadPoolExecutor executor = getExecutor();
- Assert.assertEquals(executor.getCompletedTaskCount(),
- expectedCompletedTaskCount);
- Assert.assertEquals(executor.getActiveCount(), 0);
+ Assert.assertEquals(expectedCompletedTaskCount,
+ executor.getCompletedTaskCount());
+ Assert.assertEquals(0, executor.getActiveCount());
Assert.assertTrue(executor.isTerminated());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/35] hadoop git commit: YARN-6550. Capture launch_container.sh
logs to a separate log file. (Suma Shivaprasad via wangda)
Posted by ae...@apache.org.
YARN-6550. Capture launch_container.sh logs to a separate log file. (Suma Shivaprasad via wangda)
Change-Id: I0ee0b1bb459437432a22cf68861a6354f0decabb
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/febeead5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/febeead5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/febeead5
Branch: refs/heads/HDFS-7240
Commit: febeead5f95c6fc245ea3735f5b538d4bb4dc8a4
Parents: 373d0a5
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Sep 29 16:39:46 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Sep 29 16:39:46 2017 -0700
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 20 +-
.../nodemanager/DefaultContainerExecutor.java | 3 +-
.../nodemanager/LinuxContainerExecutor.java | 3 +-
.../launcher/ContainerLaunch.java | 189 ++++++++++++++++---
.../launcher/TestContainerLaunch.java | 115 ++++++++++-
5 files changed, 290 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/febeead5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index da50d7a..5fd059d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -64,6 +64,9 @@ import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.CONTAINER_PRE_LAUNCH_STDOUT;
+
/**
* This class is abstraction of the mechanism used to launch a container on the
* underlying OS. All executor implementations must extend ContainerExecutor.
@@ -330,6 +333,14 @@ public abstract class ContainerExecutor implements Configurable {
String user, String outFilename) throws IOException {
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
+
+ // Add "set -o pipefail -e" to validate launch_container script.
+ sb.setExitOnFailure();
+
+ //Redirect stdout and stderr for launch_container script
+ sb.stdout(logDir, CONTAINER_PRE_LAUNCH_STDOUT);
+ sb.stderr(logDir, CONTAINER_PRE_LAUNCH_STDERR);
+
Set<String> whitelist = new HashSet<>();
String[] nmWhiteList = conf.get(YarnConfiguration.NM_ENV_WHITELIST,
@@ -338,10 +349,8 @@ public abstract class ContainerExecutor implements Configurable {
whitelist.add(param);
}
- // Add "set -o pipefail -e" to validate launch_container script.
- sb.setExitOnFailure();
-
if (environment != null) {
+ sb.echo("Setting up env variables");
for (Map.Entry<String, String> env : environment.entrySet()) {
if (!whitelist.contains(env.getKey())) {
sb.env(env.getKey(), env.getValue());
@@ -352,6 +361,7 @@ public abstract class ContainerExecutor implements Configurable {
}
if (resources != null) {
+ sb.echo("Setting up job resources");
for (Map.Entry<Path, List<String>> resourceEntry :
resources.entrySet()) {
for (String linkName : resourceEntry.getValue()) {
@@ -373,15 +383,15 @@ public abstract class ContainerExecutor implements Configurable {
if (getConf() != null &&
getConf().getBoolean(YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO,
YarnConfiguration.DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO)) {
+ sb.echo("Copying debugging information");
sb.copyDebugInformation(new Path(outFilename),
new Path(logDir, outFilename));
sb.listDebugInformation(new Path(logDir, DIRECTORY_CONTENTS));
}
-
+ sb.echo("Launching container");
sb.command(command);
PrintStream pout = null;
-
try {
pout = new PrintStream(out, false, "UTF-8");
sb.write(pout);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/febeead5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 8c58056..2b32234 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -320,8 +320,7 @@ public class DefaultContainerExecutor extends ContainerExecutor {
builder.append("Exception message: ");
builder.append(e.getMessage()).append("\n");
}
- builder.append("Stack trace: ");
- builder.append(StringUtils.stringifyException(e)).append("\n");
+
if (!shExec.getOutput().isEmpty()) {
builder.append("Shell output: ");
builder.append(shExec.getOutput()).append("\n");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/febeead5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 2971f83..9e004e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -532,8 +532,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
if (!Optional.fromNullable(e.getErrorOutput()).or("").isEmpty()) {
builder.append("Exception message: " + e.getErrorOutput() + "\n");
}
- builder.append("Stack trace: "
- + StringUtils.stringifyException(e) + "\n");
+ //Skip stack trace
String output = e.getOutput();
if (output != null && !e.getOutput().isEmpty()) {
builder.append("Shell output: " + output + "\n");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/febeead5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index e254887..f929dfc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -87,6 +87,8 @@ import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.yarn.util.ConverterUtils;
public class ContainerLaunch implements Callable<Integer> {
@@ -94,8 +96,13 @@ public class ContainerLaunch implements Callable<Integer> {
private static final Logger LOG =
LoggerFactory.getLogger(ContainerLaunch.class);
+ private static final String CONTAINER_PRE_LAUNCH_PREFIX = "prelaunch";
+ public static final String CONTAINER_PRE_LAUNCH_STDOUT = CONTAINER_PRE_LAUNCH_PREFIX + ".out";
+ public static final String CONTAINER_PRE_LAUNCH_STDERR = CONTAINER_PRE_LAUNCH_PREFIX + ".err";
+
public static final String CONTAINER_SCRIPT =
Shell.appendScriptExtension("launch_container");
+
public static final String FINAL_CONTAINER_TOKENS_FILE = "container_tokens";
private static final String PID_FILE_NAME_FMT = "%s.pid";
@@ -147,7 +154,7 @@ public class ContainerLaunch implements Callable<Integer> {
Path containerLogDir) {
var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
containerLogDir.toString());
- var = var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
+ var = var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
File.pathSeparator);
// replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
@@ -371,7 +378,7 @@ public class ContainerLaunch implements Callable<Integer> {
String relativeContainerLogDir = ContainerLaunch
.getRelativeContainerLogDir(appIdStr, containerIdStr);
- for(String logDir : logDirs) {
+ for (String logDir : logDirs) {
containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir);
}
@@ -516,6 +523,7 @@ public class ContainerLaunch implements Callable<Integer> {
* Tries to tail and fetch TAIL_SIZE_IN_BYTES of data from the error log.
* ErrorLog filename is not fixed and depends upon app, hence file name
* pattern is used.
+ *
* @param containerID
* @param ret
* @param containerLogDir
@@ -524,20 +532,46 @@ public class ContainerLaunch implements Callable<Integer> {
@SuppressWarnings("unchecked")
protected void handleContainerExitWithFailure(ContainerId containerID,
int ret, Path containerLogDir, StringBuilder diagnosticInfo) {
- LOG.warn(diagnosticInfo.toString());
+ LOG.warn("Container launch failed : " + diagnosticInfo.toString());
+
+ FileSystem fileSystem = null;
+ long tailSizeInBytes =
+ conf.getLong(YarnConfiguration.NM_CONTAINER_STDERR_BYTES,
+ YarnConfiguration.DEFAULT_NM_CONTAINER_STDERR_BYTES);
+
+ // Append container prelaunch stderr to diagnostics
+ try {
+ fileSystem = FileSystem.getLocal(conf).getRaw();
+ FileStatus preLaunchErrorFileStatus = fileSystem
+ .getFileStatus(new Path(containerLogDir, ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR));
+
+ Path errorFile = preLaunchErrorFileStatus.getPath();
+ long fileSize = preLaunchErrorFileStatus.getLen();
+
+ diagnosticInfo.append("Error file: ")
+ .append(ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR).append(".\n");
+ ;
+ byte[] tailBuffer = tailFile(errorFile, fileSize, tailSizeInBytes);
+ diagnosticInfo.append("Last ").append(tailSizeInBytes)
+ .append(" bytes of ").append(errorFile.getName()).append(" :\n")
+ .append(new String(tailBuffer, StandardCharsets.UTF_8));
+ } catch (IOException e) {
+ LOG.error("Failed to get tail of the container's prelaunch error log file", e);
+ }
+
+ // Append container stderr to diagnostics
String errorFileNamePattern =
conf.get(YarnConfiguration.NM_CONTAINER_STDERR_PATTERN,
YarnConfiguration.DEFAULT_NM_CONTAINER_STDERR_PATTERN);
- FSDataInputStream errorFileIS = null;
+
try {
- FileSystem fileSystem = FileSystem.getLocal(conf).getRaw();
+ if (fileSystem == null) {
+ fileSystem = FileSystem.getLocal(conf).getRaw();
+ }
FileStatus[] errorFileStatuses = fileSystem
.globStatus(new Path(containerLogDir, errorFileNamePattern));
if (errorFileStatuses != null && errorFileStatuses.length != 0) {
- long tailSizeInBytes =
- conf.getLong(YarnConfiguration.NM_CONTAINER_STDERR_BYTES,
- YarnConfiguration.DEFAULT_NM_CONTAINER_STDERR_BYTES);
Path errorFile = errorFileStatuses[0].getPath();
long fileSize = errorFileStatuses[0].getLen();
@@ -560,32 +594,40 @@ public class ContainerLaunch implements Callable<Integer> {
.append(StringUtils.join(", ", errorFileNames)).append(".\n");
}
- long startPosition =
- (fileSize < tailSizeInBytes) ? 0 : fileSize - tailSizeInBytes;
- int bufferSize =
- (int) ((fileSize < tailSizeInBytes) ? fileSize : tailSizeInBytes);
- byte[] tailBuffer = new byte[bufferSize];
- errorFileIS = fileSystem.open(errorFile);
- errorFileIS.readFully(startPosition, tailBuffer);
-
+ byte[] tailBuffer = tailFile(errorFile, fileSize, tailSizeInBytes);
String tailBufferMsg = new String(tailBuffer, StandardCharsets.UTF_8);
diagnosticInfo.append("Last ").append(tailSizeInBytes)
.append(" bytes of ").append(errorFile.getName()).append(" :\n")
.append(tailBufferMsg).append("\n")
.append(analysesErrorMsgOfContainerExitWithFailure(tailBufferMsg));
+
}
} catch (IOException e) {
LOG.error("Failed to get tail of the container's error log file", e);
- } finally {
- IOUtils.cleanupWithLogger(LOG, errorFileIS);
}
-
this.dispatcher.getEventHandler()
.handle(new ContainerExitEvent(containerID,
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret,
diagnosticInfo.toString()));
}
+ private byte[] tailFile(Path filePath, long fileSize, long tailSizeInBytes) throws IOException {
+ FSDataInputStream errorFileIS = null;
+ FileSystem fileSystem = FileSystem.getLocal(conf).getRaw();
+ try {
+ long startPosition =
+ (fileSize < tailSizeInBytes) ? 0 : fileSize - tailSizeInBytes;
+ int bufferSize =
+ (int) ((fileSize < tailSizeInBytes) ? fileSize : tailSizeInBytes);
+ byte[] tailBuffer = new byte[bufferSize];
+ errorFileIS = fileSystem.open(filePath);
+ errorFileIS.readFully(startPosition, tailBuffer);
+ return tailBuffer;
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, errorFileIS);
+ }
+ }
+
private String analysesErrorMsgOfContainerExitWithFailure(String errorMsg) {
StringBuilder analysis = new StringBuilder();
if (errorMsg.indexOf("Error: Could not find or load main class"
@@ -982,8 +1024,48 @@ public class ContainerLaunch implements Callable<Integer> {
public abstract void whitelistedEnv(String key, String value) throws IOException;
+ protected static final String ENV_PRELAUNCH_STDOUT = "PRELAUNCH_OUT";
+ protected static final String ENV_PRELAUNCH_STDERR = "PRELAUNCH_ERR";
+
+ private boolean redirectStdOut = false;
+ private boolean redirectStdErr = false;
+
+ /**
+ * Set stdout for the shell script
+ * @param stdoutDir stdout must be an absolute path
+ * @param stdOutFile stdout file name
+ * @throws IOException thrown when stdout path is not absolute
+ */
+ public final void stdout(Path stdoutDir, String stdOutFile) throws IOException {
+ if (!stdoutDir.isAbsolute()) {
+ throw new IOException("Stdout path must be absolute");
+ }
+ redirectStdOut = true;
+ setStdOut(new Path(stdoutDir, stdOutFile));
+ }
+
+ /**
+ * Set stderr for the shell script
+ * @param stderrDir stderr must be an absolute path
+ * @param stdErrFile stderr file name
+ * @throws IOException thrown when stderr path is not absolute
+ */
+ public final void stderr(Path stderrDir, String stdErrFile) throws IOException {
+ if (!stderrDir.isAbsolute()) {
+ throw new IOException("Stdout path must be absolute");
+ }
+ redirectStdErr = true;
+ setStdErr(new Path(stderrDir, stdErrFile));
+ }
+
+ protected abstract void setStdOut(Path stdout) throws IOException;
+
+ protected abstract void setStdErr(Path stdout) throws IOException;
+
public abstract void env(String key, String value) throws IOException;
+ public abstract void echo(String echoStr) throws IOException;
+
public final void symlink(Path src, Path dst) throws IOException {
if (!src.isAbsolute()) {
throw new IOException("Source must be absolute");
@@ -1028,13 +1110,21 @@ public class ContainerLaunch implements Callable<Integer> {
out.append(sb);
}
- protected final void line(String... command) {
+ protected final void buildCommand(String... command) {
for (String s : command) {
sb.append(s);
}
+ }
+
+ protected final void linebreak(String... command) {
sb.append(LINE_SEPARATOR);
}
+ protected final void line(String... command) {
+ buildCommand(command);
+ linebreak();
+ }
+
public void setExitOnFailure() {
// Dummy implementation
}
@@ -1042,19 +1132,27 @@ public class ContainerLaunch implements Callable<Integer> {
protected abstract void link(Path src, Path dst) throws IOException;
protected abstract void mkdir(Path path) throws IOException;
+
+ boolean doRedirectStdOut() {
+ return redirectStdOut;
+ }
+
+ boolean doRedirectStdErr() {
+ return redirectStdErr;
+ }
+
}
private static final class UnixShellScriptBuilder extends ShellScriptBuilder {
-
private void errorCheck() {
line("hadoop_shell_errorcode=$?");
- line("if [ $hadoop_shell_errorcode -ne 0 ]");
+ line("if [[ \"$hadoop_shell_errorcode\" -ne 0 ]]");
line("then");
line(" exit $hadoop_shell_errorcode");
line("fi");
}
- public UnixShellScriptBuilder(){
+ public UnixShellScriptBuilder() {
line("#!/bin/bash");
line();
}
@@ -1062,29 +1160,47 @@ public class ContainerLaunch implements Callable<Integer> {
@Override
public void command(List<String> command) {
line("exec /bin/bash -c \"", StringUtils.join(" ", command), "\"");
- errorCheck();
}
@Override
- public void whitelistedEnv(String key, String value) {
+ public void whitelistedEnv(String key, String value) throws IOException {
line("export ", key, "=${", key, ":-", "\"", value, "\"}");
}
@Override
- public void env(String key, String value) {
+ public void setStdOut(final Path stdout) throws IOException {
+ line("export ", ENV_PRELAUNCH_STDOUT, "=\"", stdout.toString(), "\"");
+ // tee is needed for DefaultContainerExecutor error propagation to stdout
+ // Close stdout of subprocess to prevent it from writing to the stdout file
+ line("exec >\"${" + ENV_PRELAUNCH_STDOUT + "}\"");
+ }
+
+ @Override
+ public void setStdErr(final Path stderr) throws IOException {
+ line("export ", ENV_PRELAUNCH_STDERR, "=\"", stderr.toString(), "\"");
+ // tee is needed for DefaultContainerExecutor error propagation to stderr
+ // Close stdout of subprocess to prevent it from writing to the stdout file
+ line("exec 2>\"${" + ENV_PRELAUNCH_STDERR + "}\"");
+ }
+
+ @Override
+ public void env(String key, String value) throws IOException {
line("export ", key, "=\"", value, "\"");
}
@Override
+ public void echo(final String echoStr) throws IOException {
+ line("echo \"" + echoStr + "\"");
+ }
+
+ @Override
protected void link(Path src, Path dst) throws IOException {
line("ln -sf \"", src.toUri().getPath(), "\" \"", dst.toString(), "\"");
- errorCheck();
}
@Override
- protected void mkdir(Path path) {
+ protected void mkdir(Path path) throws IOException {
line("mkdir -p ", path.toString());
- errorCheck();
}
@Override
@@ -1152,6 +1268,16 @@ public class ContainerLaunch implements Callable<Integer> {
errorCheck();
}
+ //Dummy implementation
+ @Override
+ protected void setStdOut(final Path stdout) throws IOException {
+ }
+
+ //Dummy implementation
+ @Override
+ protected void setStdErr(final Path stderr) throws IOException {
+ }
+
@Override
public void env(String key, String value) throws IOException {
lineWithLenCheck("@set ", key, "=", value);
@@ -1159,6 +1285,11 @@ public class ContainerLaunch implements Callable<Integer> {
}
@Override
+ public void echo(final String echoStr) throws IOException {
+ lineWithLenCheck("@echo \"", echoStr, "\"");
+ }
+
+ @Override
protected void link(Path src, Path dst) throws IOException {
File srcFile = new File(src.toUri().getPath());
String srcFileStr = srcFile.getPath();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/febeead5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 7176942..588fdcd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -32,6 +32,7 @@ import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -192,7 +193,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
shexc.execute();
assertEquals(shexc.getExitCode(), 0);
- assert(shexc.getOutput().contains("hello"));
+ //Capture output from prelaunch.out
+
+ List<String> output = Files.readAllLines(Paths.get(localLogDir.getAbsolutePath(), ContainerLaunch.CONTAINER_PRE_LAUNCH_STDOUT),
+ Charset.forName("UTF-8"));
+ assert(output.contains("hello"));
symLinkFile = new File(tmpDir, badSymlink);
}
@@ -358,7 +363,10 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
shexc.execute();
Assert.fail("Should catch exception");
} catch(ExitCodeException e){
- diagnostics = e.getMessage();
+ //Capture diagnostics from prelaunch.stderr
+ List<String> error = Files.readAllLines(Paths.get(localLogDir.getAbsolutePath(), ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR),
+ Charset.forName("UTF-8"));
+ diagnostics = StringUtils.join("\n", error);
}
Assert.assertTrue(diagnostics.contains(Shell.WINDOWS ?
"is not recognized as an internal or external command" :
@@ -1545,6 +1553,109 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
}
+ /**
+ * Test that script exists with non-zero exit code when command fails.
+ * @throws IOException
+ */
+ @Test
+ public void testShellScriptBuilderStdOutandErrRedirection() throws IOException {
+ ShellScriptBuilder builder = ShellScriptBuilder.create();
+
+ Path logDir = new Path(localLogDir.getAbsolutePath());
+ File stdout = new File(logDir.toString(), ContainerLaunch.CONTAINER_PRE_LAUNCH_STDOUT);
+ File stderr = new File(logDir.toString(), ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR);
+
+ builder.stdout(logDir, ContainerLaunch.CONTAINER_PRE_LAUNCH_STDOUT);
+ builder.stderr(logDir, ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR);
+
+ //should redirect to specified stdout path
+ String TEST_STDOUT_ECHO = "Test stdout redirection";
+ builder.echo(TEST_STDOUT_ECHO);
+ //should fail and redirect to stderr
+ builder.mkdir(new Path("/invalidSrcDir"));
+
+ builder.command(Arrays.asList(new String[] {"unknownCommand"}));
+
+ File shellFile = Shell.appendScriptExtension(tmpDir, "testShellScriptBuilderStdOutandErrRedirection");
+ PrintStream writer = new PrintStream(new FileOutputStream(shellFile));
+ builder.write(writer);
+ writer.close();
+ try {
+ FileUtil.setExecutable(shellFile, true);
+
+ Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor(
+ new String[]{shellFile.getAbsolutePath()}, tmpDir);
+ try {
+ shexc.execute();
+ fail("builder shell command was expected to throw");
+ }
+ catch(IOException e) {
+ // expected
+ System.out.println("Received an expected exception: " + e.getMessage());
+
+ Assert.assertEquals(true, stdout.exists());
+ BufferedReader stdoutReader = new BufferedReader(new FileReader(stdout));
+ // Get the pid of the process
+ String line = stdoutReader.readLine().trim();
+ Assert.assertEquals(TEST_STDOUT_ECHO, line);
+ // No more lines
+ Assert.assertEquals(null, stdoutReader.readLine());
+ stdoutReader.close();
+
+ Assert.assertEquals(true, stderr.exists());
+ Assert.assertTrue(stderr.length() > 0);
+ }
+ }
+ finally {
+ FileUtil.fullyDelete(shellFile);
+ FileUtil.fullyDelete(stdout);
+ FileUtil.fullyDelete(stderr);
+ }
+ }
+
+ /**
+ * Test that script exists with non-zero exit code when command fails.
+ * @throws IOException
+ */
+ @Test
+ public void testShellScriptBuilderWithNoRedirection() throws IOException {
+ ShellScriptBuilder builder = ShellScriptBuilder.create();
+
+ Path logDir = new Path(localLogDir.getAbsolutePath());
+ File stdout = new File(logDir.toString(), ContainerLaunch.CONTAINER_PRE_LAUNCH_STDOUT);
+ File stderr = new File(logDir.toString(), ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR);
+
+ //should redirect to specified stdout path
+ String TEST_STDOUT_ECHO = "Test stdout redirection";
+ builder.echo(TEST_STDOUT_ECHO);
+ //should fail and redirect to stderr
+ builder.mkdir(new Path("/invalidSrcDir"));
+
+ builder.command(Arrays.asList(new String[]{"unknownCommand"}));
+
+ File shellFile = Shell.appendScriptExtension(tmpDir, "testShellScriptBuilderStdOutandErrRedirection");
+ PrintStream writer = new PrintStream(new FileOutputStream(shellFile));
+ builder.write(writer);
+ writer.close();
+ try {
+ FileUtil.setExecutable(shellFile, true);
+
+ Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor(
+ new String[]{shellFile.getAbsolutePath()}, tmpDir);
+ try {
+ shexc.execute();
+ fail("builder shell command was expected to throw");
+ } catch (IOException e) {
+ // expected
+ System.out.println("Received an expected exception: " + e.getMessage());
+
+ Assert.assertEquals(false, stdout.exists());
+ Assert.assertEquals(false, stderr.exists());
+ }
+ } finally {
+ FileUtil.fullyDelete(shellFile);
+ }
+ }
/*
* ${foo.version} is substituted to suffix a specific version number
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/35] hadoop git commit: YARN-7250. Update Shared cache client api
to use URLs.
Posted by ae...@apache.org.
YARN-7250. Update Shared cache client api to use URLs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c114da5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c114da5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c114da5e
Branch: refs/heads/HDFS-7240
Commit: c114da5e64d14b1d9e614081c4171ea0391cb1aa
Parents: 6f789fe
Author: Chris Trezzo <ct...@apache.org>
Authored: Thu Sep 28 15:28:06 2017 -0700
Committer: Chris Trezzo <ct...@apache.org>
Committed: Thu Sep 28 15:28:06 2017 -0700
----------------------------------------------------------------------
.../yarn/client/api/SharedCacheClient.java | 22 ++++--------
.../client/api/impl/SharedCacheClientImpl.java | 36 +++++---------------
.../api/impl/TestSharedCacheClientImpl.java | 31 +++++------------
3 files changed, 23 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c114da5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
index 60c1bd98..a9c1a07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.client.api.impl.SharedCacheClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -58,34 +59,25 @@ public abstract class SharedCacheClient extends AbstractService {
*
* <p>
* The <code>SharedCacheManager</code> responds with whether or not the
- * resource exists in the cache. If the resource exists, a <code>Path</code>
- * to the resource in the shared cache is returned. If the resource does not
+ * resource exists in the cache. If the resource exists, a <code>URL</code> to
+ * the resource in the shared cache is returned. If the resource does not
* exist, null is returned instead.
* </p>
*
* <p>
- * Once a path has been returned for a resource, that path is safe to use for
+ * Once a URL has been returned for a resource, that URL is safe to use for
* the lifetime of the application that corresponds to the provided
* ApplicationId.
* </p>
*
- * <p>
- * Additionally, a name for the resource should be specified. A fragment will
- * be added to the path with the desired name if the desired name is different
- * than the name of the provided path from the shared cache. This ensures that
- * if the returned path is used to create a LocalResource, then the symlink
- * created during YARN localization will match the name specified.
- * </p>
- *
* @param applicationId ApplicationId of the application using the resource
* @param resourceKey the key (i.e. checksum) that identifies the resource
- * @param resourceName the desired name of the resource
- * @return Path to the resource, or null if it does not exist
+ * @return URL to the resource, or null if it does not exist
*/
@Public
@Unstable
- public abstract Path use(ApplicationId applicationId, String resourceKey,
- String resourceName) throws YarnException;
+ public abstract URL use(ApplicationId applicationId, String resourceKey)
+ throws YarnException;
/**
* <p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c114da5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
index b910c28..3191d36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.client.api.impl;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -38,6 +36,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequ
import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.client.api.SharedCacheClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -113,8 +112,8 @@ public class SharedCacheClientImpl extends SharedCacheClient {
}
@Override
- public Path use(ApplicationId applicationId, String resourceKey,
- String resourceName) throws YarnException {
+ public URL use(ApplicationId applicationId, String resourceKey)
+ throws YarnException {
Path resourcePath = null;
UseSharedCacheResourceRequest request = Records.newRecord(
UseSharedCacheResourceRequest.class);
@@ -132,31 +131,12 @@ public class SharedCacheClientImpl extends SharedCacheClient {
throw new YarnException(e);
}
if (resourcePath != null) {
- if (resourcePath.getName().equals(resourceName)) {
- // The preferred name is the same as the name of the item in the cache,
- // so we skip generating the fragment to save space in the MRconfig.
- return resourcePath;
- } else {
- // We are using the shared cache, and a preferred name has been
- // specified that is different than the name of the resource in the
- // shared cache. We need to set the fragment portion of the URI to
- // preserve the desired name.
- URI pathURI = resourcePath.toUri();
- try {
- // We assume that there is no existing fragment in the URI since the
- // shared cache manager does not use fragments.
- pathURI =
- new URI(pathURI.getScheme(), pathURI.getSchemeSpecificPart(),
- resourceName);
- resourcePath = new Path(pathURI);
- } catch (URISyntaxException e) {
- throw new YarnException(
- "Could not create a new URI due to syntax errors: "
- + pathURI.toString(), e);
- }
- }
+ URL pathURL = URL.fromPath(resourcePath);
+ return pathURL;
+ } else {
+ // The resource was not in the cache.
+ return null;
}
- return resourcePath;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c114da5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
index 16b32db..6a8edd2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
@@ -27,7 +27,6 @@ import static org.mockito.Mockito.when;
import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -40,6 +39,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceResponsePBImpl;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.After;
import org.junit.AfterClass;
@@ -114,36 +114,21 @@ public class TestSharedCacheClientImpl {
response.setPath(null);
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
response);
- Path newPath = client.use(mock(ApplicationId.class), "key", null);
- assertNull("The path is not null!", newPath);
+ URL newURL = client.use(mock(ApplicationId.class), "key");
+ assertNull("The path is not null!", newURL);
}
@Test
- public void testUseWithResourceName() throws Exception {
+ public void testUseCacheHit() throws Exception {
Path file = new Path("viewfs://test/path");
- URI useUri = new URI("viewfs://test/path#linkName");
- Path usePath = new Path(useUri);
+ URL useUrl = URL.fromPath(new Path("viewfs://test/path"));
UseSharedCacheResourceResponse response =
new UseSharedCacheResourceResponsePBImpl();
response.setPath(file.toString());
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
response);
- Path newPath = client.use(mock(ApplicationId.class), "key", "linkName");
- assertEquals("The paths are not equal!", usePath, newPath);
- }
-
- @Test
- public void testUseWithSameResourceName() throws Exception {
- Path file = new Path("viewfs://test/path");
- URI useUri = new URI("viewfs://test/path");
- Path usePath = new Path(useUri);
- UseSharedCacheResourceResponse response =
- new UseSharedCacheResourceResponsePBImpl();
- response.setPath(file.toString());
- when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
- response);
- Path newPath = client.use(mock(ApplicationId.class), "key", "path");
- assertEquals("The paths are not equal!", usePath, newPath);
+ URL newURL = client.use(mock(ApplicationId.class), "key");
+ assertEquals("The paths are not equal!", useUrl, newURL);
}
@Test(expected = YarnException.class)
@@ -151,7 +136,7 @@ public class TestSharedCacheClientImpl {
String message = "Mock IOExcepiton!";
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenThrow(
new IOException(message));
- client.use(mock(ApplicationId.class), "key", null);
+ client.use(mock(ApplicationId.class), "key");
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/35] hadoop git commit: YARN-3661. Basic Federation UI.
(Contributed by Inigo Goiri via curino)
Posted by ae...@apache.org.
YARN-3661. Basic Federation UI. (Contributed by Inigo Goiri via curino)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ceca9694
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ceca9694
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ceca9694
Branch: refs/heads/HDFS-7240
Commit: ceca9694f9a0c78d07cab2c382036f175183e67b
Parents: 5a68c2c
Author: Carlo Curino <cu...@apache.org>
Authored: Mon Oct 2 13:03:32 2017 -0700
Committer: Carlo Curino <cu...@apache.org>
Committed: Mon Oct 2 13:03:32 2017 -0700
----------------------------------------------------------------------
.../yarn/server/router/webapp/AboutBlock.java | 91 ++++++++++
.../yarn/server/router/webapp/AboutPage.java | 37 ++++
.../yarn/server/router/webapp/AppsBlock.java | 130 ++++++++++++++
.../yarn/server/router/webapp/AppsPage.java | 77 ++++++++
.../server/router/webapp/FederationBlock.java | 176 +++++++++++++++++++
.../server/router/webapp/FederationPage.java | 57 ++++++
.../yarn/server/router/webapp/NavBlock.java | 46 +++++
.../yarn/server/router/webapp/NodesBlock.java | 109 ++++++++++++
.../yarn/server/router/webapp/NodesPage.java | 60 +++++++
.../server/router/webapp/RouterController.java | 60 +++++++
.../yarn/server/router/webapp/RouterView.java | 52 ++++++
.../yarn/server/router/webapp/RouterWebApp.java | 6 +
.../src/site/markdown/Federation.md | 6 +-
13 files changed, 905 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java
new file mode 100644
index 0000000..cd588fc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java
@@ -0,0 +1,91 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+
+import com.google.inject.Inject;
+
+/**
+ * About block for the Router Web UI.
+ */
+public class AboutBlock extends HtmlBlock {
+
+ private static final long BYTES_IN_MB = 1024 * 1024;
+
+ private final Router router;
+
+ @Inject
+ AboutBlock(Router router, ViewContext ctx) {
+ super(ctx);
+ this.router = router;
+ }
+
+ @Override
+ protected void render(Block html) {
+ Configuration conf = this.router.getConfig();
+ String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
+
+ ClusterMetricsInfo metrics = RouterWebServiceUtil.genericForward(
+ webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET,
+ RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null);
+ boolean isEnabled = conf.getBoolean(
+ YarnConfiguration.FEDERATION_ENABLED,
+ YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
+ info("Cluster Status").
+ __("Federation Enabled", isEnabled).
+ __("Applications Submitted", "N/A").
+ __("Applications Pending", "N/A").
+ __("Applications Running", "N/A").
+ __("Applications Failed", "N/A").
+ __("Applications Killed", "N/A").
+ __("Applications Completed", "N/A").
+ __("Containers Allocated", metrics.getContainersAllocated()).
+ __("Containers Reserved", metrics.getReservedContainers()).
+ __("Containers Pending", metrics.getPendingContainers()).
+ __("Available Memory",
+ StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB)).
+ __("Allocated Memory",
+ StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB)).
+ __("Reserved Memory",
+ StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB)).
+ __("Total Memory",
+ StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB)).
+ __("Available VirtualCores", metrics.getAvailableVirtualCores()).
+ __("Allocated VirtualCores", metrics.getAllocatedVirtualCores()).
+ __("Reserved VirtualCores", metrics.getReservedVirtualCores()).
+ __("Total VirtualCores", metrics.getTotalVirtualCores()).
+ __("Active Nodes", metrics.getActiveNodes()).
+ __("Lost Nodes", metrics.getLostNodes()).
+ __("Available Nodes", metrics.getDecommissionedNodes()).
+ __("Unhealthy Nodes", metrics.getUnhealthyNodes()).
+ __("Rebooted Nodes", metrics.getRebootedNodes()).
+ __("Total Nodes", metrics.getTotalNodes());
+
+ html.__(InfoBlock.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutPage.java
new file mode 100644
index 0000000..3c9f00d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutPage.java
@@ -0,0 +1,37 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+
+/**
+ * About page for the Router Web UI.
+ */
+public class AboutPage extends RouterView {
+
+ @Override
+ protected void preHead(Page.HTML<__> html) {
+ commonPreHead(html);
+ }
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return AboutBlock.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
new file mode 100644
index 0000000..56fa73e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+import static org.apache.commons.lang.StringEscapeUtils.escapeJavaScript;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+import com.google.inject.Inject;
+
+/**
+ * Applications block for the Router Web UI.
+ */
+public class AppsBlock extends HtmlBlock {
+ private final Router router;
+
+ @Inject
+ AppsBlock(Router router, ViewContext ctx) {
+ super(ctx);
+ this.router = router;
+ }
+
+ @Override
+ protected void render(Block html) {
+ // Get the applications from the Resource Managers
+ Configuration conf = this.router.getConfig();
+ String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
+ AppsInfo apps = RouterWebServiceUtil.genericForward(webAppAddress, null,
+ AppsInfo.class, HTTPMethods.GET,
+ RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null);
+
+ setTitle("Applications");
+
+ TBODY<TABLE<Hamlet>> tbody = html.table("#apps").thead()
+ .tr()
+ .th(".id", "ID")
+ .th(".user", "User")
+ .th(".name", "Name")
+ .th(".type", "Application Type")
+ .th(".queue", "Queue")
+ .th(".priority", "Application Priority")
+ .th(".starttime", "StartTime")
+ .th(".finishtime", "FinishTime")
+ .th(".state", "State")
+ .th(".finalstatus", "FinalStatus")
+ .th(".progress", "Progress")
+ .th(".ui", "Tracking UI")
+ .__().__().tbody();
+
+ // Render the applications
+ StringBuilder appsTableData = new StringBuilder("[\n");
+ for (AppInfo app : apps.getApps()) {
+ try {
+
+ String percent = String.format("%.1f", app.getProgress() * 100.0F);
+ String trackingURL =
+ app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();
+ // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
+ appsTableData.append("[\"")
+ .append("<a href='").append(trackingURL).append("'>")
+ .append(app.getAppId()).append("</a>\",\"")
+ .append(escape(app.getUser())).append("\",\"")
+ .append(escape(app.getName())).append("\",\"")
+ .append(escape(app.getApplicationType())).append("\",\"")
+ .append(escape(app.getQueue())).append("\",\"")
+ .append(String.valueOf(app.getPriority())).append("\",\"")
+ .append(app.getStartTime()).append("\",\"")
+ .append(app.getFinishTime()).append("\",\"")
+ .append(app.getState()).append("\",\"")
+ .append(app.getFinalStatus()).append("\",\"")
+ // Progress bar
+ .append("<br title='").append(percent).append("'> <div class='")
+ .append(C_PROGRESSBAR).append("' title='")
+ .append(join(percent, '%')).append("'> ").append("<div class='")
+ .append(C_PROGRESSBAR_VALUE).append("' style='")
+ .append(join("width:", percent, '%')).append("'> </div> </div>")
+ // History link
+ .append("\",\"<a href='").append(trackingURL).append("'>")
+ .append("History").append("</a>");
+ appsTableData.append("\"],\n");
+
+ } catch (Exception e) {
+ LOG.info(
+ "Cannot add application {}: {}", app.getAppId(), e.getMessage());
+ }
+ }
+ if (appsTableData.charAt(appsTableData.length() - 2) == ',') {
+ appsTableData.delete(appsTableData.length() - 2,
+ appsTableData.length() - 1);
+ }
+ appsTableData.append("]");
+ html.script().$type("text/javascript")
+ .__("var appsTableData=" + appsTableData).__();
+
+ tbody.__().__();
+ }
+
+ private static String escape(String str) {
+ return escapeJavaScript(escapeHtml(str));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
new file mode 100644
index 0000000..12d0b5b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+
+class AppsPage extends RouterView {
+
+ @Override
+ protected void preHead(Page.HTML<__> html) {
+ commonPreHead(html);
+ set(DATATABLES_ID, "apps");
+ set(initID(DATATABLES, "apps"), appsTableInit());
+ setTableStyles(html, "apps", ".queue {width:6em}", ".ui {width:8em}");
+
+ // Set the correct title.
+ String reqState = $(APP_STATE);
+ reqState = (reqState == null || reqState.isEmpty() ? "All" : reqState);
+ setTitle(sjoin(reqState, "Applications"));
+ }
+
+ private String appsTableInit() {
+ // id, user, name, queue, starttime, finishtime, state, status, progress, ui
+ return tableInit()
+ .append(", 'aaData': appsTableData")
+ .append(", bDeferRender: true")
+ .append(", bProcessing: true")
+
+ .append("\n, aoColumnDefs: ")
+ .append(getAppsTableColumnDefs())
+
+ // Sort by id upon page load
+ .append(", aaSorting: [[0, 'desc']]}").toString();
+ }
+
+ protected String getAppsTableColumnDefs() {
+ StringBuilder sb = new StringBuilder();
+ return sb
+ .append("[\n")
+ .append("{'sType':'string', 'aTargets': [0]")
+ .append(", 'mRender': parseHadoopID }")
+
+ .append("\n, {'sType':'numeric', 'aTargets': [6, 7]")
+ .append(", 'mRender': renderHadoopDate }")
+
+ .append("\n, {'sType':'numeric', bSearchable:false, 'aTargets': [10]")
+ .append(", 'mRender': parseHadoopProgress }]").toString();
+ }
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return AppsBlock.class;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
new file mode 100644
index 0000000..d84c705
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+import com.google.inject.Inject;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONUnmarshaller;
+
+class FederationBlock extends HtmlBlock {
+
+ private static final long BYTES_IN_MB = 1024 * 1024;
+
+ private final Router router;
+
+ @Inject
+ FederationBlock(ViewContext ctx, Router router) {
+ super(ctx);
+ this.router = router;
+ }
+
+ @Override
+ public void render(Block html) {
+ Configuration conf = this.router.getConfig();
+ boolean isEnabled = conf.getBoolean(
+ YarnConfiguration.FEDERATION_ENABLED,
+ YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
+ if (isEnabled) {
+ setTitle("Federation");
+
+ // Table header
+ TBODY<TABLE<Hamlet>> tbody = html.table("#rms").thead().tr()
+ .th(".id", "SubCluster")
+ .th(".submittedA", "Applications Submitted*")
+ .th(".pendingA", "Applications Pending*")
+ .th(".runningA", "Applications Running*")
+ .th(".failedA", "Applications Failed*")
+ .th(".killedA", "Applications Killed*")
+ .th(".completedA", "Applications Completed*")
+ .th(".contAllocated", "Containers Allocated")
+ .th(".contReserved", "Containers Reserved")
+ .th(".contPending", "Containers Pending")
+ .th(".availableM", "Available Memory")
+ .th(".allocatedM", "Allocated Memory")
+ .th(".reservedM", "Reserved Memory")
+ .th(".totalM", "Total Memory")
+ .th(".availableVC", "Available VirtualCores")
+ .th(".allocatedVC", "Allocated VirtualCores")
+ .th(".reservedVC", "Reserved VirtualCores")
+ .th(".totalVC", "Total VirtualCores")
+ .th(".activeN", "Active Nodes")
+ .th(".lostN", "Lost Nodes")
+ .th(".availableN", "Available Nodes")
+ .th(".unhealtyN", "Unhealthy Nodes")
+ .th(".rebootedN", "Rebooted Nodes")
+ .th(".totalN", "Total Nodes")
+ .__().__().tbody();
+
+ try {
+ // Binding to the FederationStateStore
+ FederationStateStoreFacade facade =
+ FederationStateStoreFacade.getInstance();
+ Map<SubClusterId, SubClusterInfo> subClustersInfo =
+ facade.getSubClusters(true);
+
+ // Sort the SubClusters
+ List<SubClusterInfo> subclusters = new ArrayList<>();
+ subclusters.addAll(subClustersInfo.values());
+ Comparator<? super SubClusterInfo> cmp =
+ new Comparator<SubClusterInfo>() {
+ @Override
+ public int compare(SubClusterInfo o1, SubClusterInfo o2) {
+ return o1.getSubClusterId().compareTo(o2.getSubClusterId());
+ }
+ };
+ Collections.sort(subclusters, cmp);
+
+ for (SubClusterInfo subcluster : subclusters) {
+ SubClusterId subClusterId = subcluster.getSubClusterId();
+ String webAppAddress = subcluster.getRMWebServiceAddress();
+ String capability = subcluster.getCapability();
+ ClusterMetricsInfo subClusterInfo = getClusterMetricsInfo(capability);
+
+ // Building row per SubCluster
+ tbody.tr().td().a("//" + webAppAddress, subClusterId.toString()).__()
+ .td(Integer.toString(subClusterInfo.getAppsSubmitted()))
+ .td(Integer.toString(subClusterInfo.getAppsPending()))
+ .td(Integer.toString(subClusterInfo.getAppsRunning()))
+ .td(Integer.toString(subClusterInfo.getAppsFailed()))
+ .td(Integer.toString(subClusterInfo.getAppsKilled()))
+ .td(Integer.toString(subClusterInfo.getAppsCompleted()))
+ .td(Integer.toString(subClusterInfo.getContainersAllocated()))
+ .td(Integer.toString(subClusterInfo.getReservedContainers()))
+ .td(Integer.toString(subClusterInfo.getPendingContainers()))
+ .td(StringUtils.byteDesc(
+ subClusterInfo.getAvailableMB() * BYTES_IN_MB))
+ .td(StringUtils.byteDesc(
+ subClusterInfo.getAllocatedMB() * BYTES_IN_MB))
+ .td(StringUtils.byteDesc(
+ subClusterInfo.getReservedMB() * BYTES_IN_MB))
+ .td(StringUtils.byteDesc(
+ subClusterInfo.getTotalMB() * BYTES_IN_MB))
+ .td(Long.toString(subClusterInfo.getAvailableVirtualCores()))
+ .td(Long.toString(subClusterInfo.getAllocatedVirtualCores()))
+ .td(Long.toString(subClusterInfo.getReservedVirtualCores()))
+ .td(Long.toString(subClusterInfo.getTotalVirtualCores()))
+ .td(Integer.toString(subClusterInfo.getActiveNodes()))
+ .td(Integer.toString(subClusterInfo.getLostNodes()))
+ .td(Integer.toString(subClusterInfo.getDecommissionedNodes()))
+ .td(Integer.toString(subClusterInfo.getUnhealthyNodes()))
+ .td(Integer.toString(subClusterInfo.getRebootedNodes()))
+ .td(Integer.toString(subClusterInfo.getTotalNodes())).__();
+ }
+ } catch (YarnException e) {
+ LOG.error("Cannot render ResourceManager", e);
+ }
+
+ tbody.__().__().div()
+ .p().__("*The application counts are local per subcluster").__().__();
+ } else {
+ setTitle("Federation is not Enabled!");
+ }
+ }
+
+ private static ClusterMetricsInfo getClusterMetricsInfo(String capability) {
+ ClusterMetricsInfo clusterMetrics = null;
+ try {
+ JSONJAXBContext jc = new JSONJAXBContext(
+ JSONConfiguration.mapped().rootUnwrapping(false).build(),
+ ClusterMetricsInfo.class);
+ JSONUnmarshaller unmarshaller = jc.createJSONUnmarshaller();
+ clusterMetrics = unmarshaller.unmarshalFromJSON(
+ new StringReader(capability), ClusterMetricsInfo.class);
+ } catch (Exception e) {
+ LOG.error("Cannot parse SubCluster info", e);
+ }
+ return clusterMetrics;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationPage.java
new file mode 100644
index 0000000..ddd2866
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationPage.java
@@ -0,0 +1,57 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+
+/**
+ * Renders a block for the applications with metrics information.
+ */
+class FederationPage extends RouterView {
+
+ @Override
+ protected void preHead(Page.HTML<__> html) {
+ commonPreHead(html);
+ setTitle("Federation");
+ set(DATATABLES_ID, "rms");
+ set(initID(DATATABLES, "rms"), rmsTableInit());
+ setTableStyles(html, "rms", ".healthStatus {width:10em}",
+ ".healthReport {width:10em}");
+ }
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return FederationBlock.class;
+ }
+
+ private String rmsTableInit() {
+ StringBuilder b = tableInit().append(", aoColumnDefs: [");
+ b.append("{'bSearchable': false, 'aTargets': [ 7 ]}");
+ b.append(", {'sType': 'title-numeric', 'bSearchable': false, "
+ + "'aTargets': [ 8, 9 ] }");
+ b.append(", {'sType': 'title-numeric', 'aTargets': [ 5 ]}");
+ b.append("]}");
+ return b.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
new file mode 100644
index 0000000..9c39bb7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
@@ -0,0 +1,46 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+/**
+ * Navigation block for the Router Web UI.
+ */
+public class NavBlock extends HtmlBlock {
+
+ @Override
+ public void render(Block html) {
+ html.
+ div("#nav").
+ h3("Cluster").
+ ul().
+ li().a(url(""), "About").__().
+ li().a(url("federation"), "Federation").__().
+ li().a(url("nodes"), "Nodes").__().
+ li().a(url("apps"), "Applications").__().
+ __().
+ h3("Tools").
+ ul().
+ li().a("/conf", "Configuration").__().
+ li().a("/logs", "Local logs").__().
+ li().a("/stacks", "Server stacks").__().
+ li().a("/jmx?qry=Hadoop:*", "Server metrics").__().__().__();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
new file mode 100644
index 0000000..720302e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TR;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+import com.google.inject.Inject;
+
+/**
+ * Nodes block for the Router Web UI.
+ */
+public class NodesBlock extends HtmlBlock {
+
+ private static final long BYTES_IN_MB = 1024 * 1024;
+
+ private final Router router;
+
+ @Inject
+ NodesBlock(Router router, ViewContext ctx) {
+ super(ctx);
+ this.router = router;
+ }
+
+ @Override
+ protected void render(Block html) {
+ // Get the node info from the federation
+ Configuration conf = this.router.getConfig();
+ String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
+ NodesInfo nodes = RouterWebServiceUtil.genericForward(webAppAddress, null,
+ NodesInfo.class, HTTPMethods.GET,
+ RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null);
+
+ setTitle("Nodes");
+
+ TBODY<TABLE<Hamlet>> tbody = html.table("#nodes").thead().tr()
+ .th(".nodelabels", "Node Labels")
+ .th(".rack", "Rack")
+ .th(".state", "Node State")
+ .th(".nodeaddress", "Node Address")
+ .th(".nodehttpaddress", "Node HTTP Address")
+ .th(".lastHealthUpdate", "Last health-update")
+ .th(".healthReport", "Health-report")
+ .th(".containers", "Containers")
+ .th(".mem", "Mem Used")
+ .th(".mem", "Mem Avail")
+ .th(".vcores", "VCores Used")
+ .th(".vcores", "VCores Avail")
+ .th(".nodeManagerVersion", "Version")
+ .__().__().tbody();
+
+ // Add nodes to the web UI
+ for (NodeInfo info : nodes.getNodes()) {
+ int usedMemory = (int) info.getUsedMemory();
+ int availableMemory = (int) info.getAvailableMemory();
+ TR<TBODY<TABLE<Hamlet>>> row = tbody.tr();
+ row.td().__(StringUtils.join(",", info.getNodeLabels())).__();
+ row.td().__(info.getRack()).__();
+ row.td().__(info.getState()).__();
+ row.td().__(info.getNodeId()).__();
+ boolean isInactive = false;
+ if (isInactive) {
+ row.td().__("N/A").__();
+ } else {
+ String httpAddress = info.getNodeHTTPAddress();
+ row.td().a("//" + httpAddress, httpAddress).__();
+ }
+ row.td().br().$title(String.valueOf(info.getLastHealthUpdate())).__()
+ .__(Times.format(info.getLastHealthUpdate())).__()
+ .td(info.getHealthReport())
+ .td(String.valueOf(info.getNumContainers())).td().br()
+ .$title(String.valueOf(usedMemory)).__()
+ .__(StringUtils.byteDesc(usedMemory * BYTES_IN_MB)).__().td().br()
+ .$title(String.valueOf(availableMemory)).__()
+ .__(StringUtils.byteDesc(availableMemory * BYTES_IN_MB)).__()
+ .td(String.valueOf(info.getUsedVirtualCores()))
+ .td(String.valueOf(info.getAvailableVirtualCores()))
+ .td(info.getVersion()).__();
+ }
+ tbody.__().__();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java
new file mode 100644
index 0000000..0351cba
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+
+class NodesPage extends RouterView {
+
+ @Override
+ protected void preHead(Page.HTML<__> html) {
+ commonPreHead(html);
+ String type = $(NODE_STATE);
+ String title = "Nodes of the cluster";
+ if (type != null && !type.isEmpty()) {
+ title = title + " (" + type + ")";
+ }
+ setTitle(title);
+ set(DATATABLES_ID, "nodes");
+ set(initID(DATATABLES, "nodes"), nodesTableInit());
+ setTableStyles(html, "nodes", ".healthStatus {width:10em}",
+ ".healthReport {width:10em}");
+ }
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return NodesBlock.class;
+ }
+
+ private String nodesTableInit() {
+ StringBuilder b = tableInit().append(", aoColumnDefs: [");
+ b.append("{'bSearchable': false, 'aTargets': [ 7 ]}");
+ b.append(", {'sType': 'title-numeric', 'bSearchable': false, "
+ + "'aTargets': [ 2, 3, 4, 5, 6 ] }");
+ b.append(", {'sType': 'title-numeric', 'aTargets': [ 5 ]}");
+ b.append("]}");
+ return b.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java
new file mode 100644
index 0000000..e8cd0de
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java
@@ -0,0 +1,60 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.yarn.webapp.Controller;
+
+import com.google.inject.Inject;
+
+/**
+ * Controller for the Router Web UI.
+ */
+public class RouterController extends Controller {
+
+ @Inject
+ RouterController(RequestContext ctx) {
+ super(ctx);
+ }
+
+ @Override
+ public void index() {
+ setTitle("Router");
+ render(AboutPage.class);
+ }
+
+ public void about() {
+ setTitle("About the Cluster");
+ render(AboutPage.class);
+ }
+
+ public void federation() {
+ setTitle("Federation");
+ render(FederationPage.class);
+ }
+
+ public void apps() {
+ setTitle("Applications");
+ render(AppsPage.class);
+ }
+
+ public void nodes() {
+ setTitle("Nodes");
+ render(NodesPage.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterView.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterView.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterView.java
new file mode 100644
index 0000000..b377f7d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterView.java
@@ -0,0 +1,52 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
+
+/**
+ * View for the Router Web UI.
+ */
+public class RouterView extends TwoColumnLayout {
+
+ @Override
+ protected void preHead(Page.HTML<__> html) {
+ commonPreHead(html);
+
+ setTitle("Router");
+ }
+
+ protected void commonPreHead(Page.HTML<__> html) {
+ set(ACCORDION_ID, "nav");
+ set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
+ }
+
+ @Override
+ protected Class<? extends SubView> nav() {
+ return NavBlock.class;
+ }
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return AboutBlock.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
index 5436bad..ba07a1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
@@ -44,5 +44,11 @@ public class RouterWebApp extends WebApp implements YarnWebParams {
if (router != null) {
bind(Router.class).toInstance(router);
}
+ route("/", RouterController.class);
+ route("/cluster", RouterController.class, "about");
+ route("/about", RouterController.class, "about");
+ route("/apps", RouterController.class, "apps");
+ route("/nodes", RouterController.class, "nodes");
+ route("/federation", RouterController.class, "federation");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
index 8a6c137..ef0f713 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
@@ -59,6 +59,7 @@ URL and redirects the application submission request to the appropriate sub-clus
The Router exposes the ApplicationClientProtocol to the outside world, transparently hiding the presence of multiple RMs. To achieve this the Router also persists the mapping
between the application and its home sub-cluster into the State Store. This allows Routers to be soft-state while supporting user requests cheaply, as any Router can recover
this application to home sub-cluster mapping and direct requests to the right RM without broadcasting them. For performance caching and session stickiness might be advisable.
+The state of the federation (including applications and nodes) is exposed through the Web UI.
###AMRMProxy
The AMRMProxy is a key component to allow the application to scale and run across sub-clusters. The AMRMProxy runs on all the NM machines and acts as a proxy to the
@@ -246,9 +247,9 @@ Optional:
|:---- |:---- |
|`yarn.router.hostname` | `0.0.0.0` | Router host name.
|`yarn.router.clientrm.address` | `0.0.0.0:8050` | Router client address. |
-|`yarn.router.webapp.address` | `0.0.0.0:80` | Webapp address at the router. |
+|`yarn.router.webapp.address` | `0.0.0.0:8089` | Webapp address at the router. |
|`yarn.router.admin.address` | `0.0.0.0:8052` | Admin address at the router. |
-|`yarn.router.webapp.https.address` | `0.0.0.0:443` | Secure webapp address at the router. |
+|`yarn.router.webapp.https.address` | `0.0.0.0:8091` | Secure webapp address at the router. |
|`yarn.router.submit.retry` | `3` | The number of retries in the router before we give up. |
|`yarn.federation.statestore.max-connections` | `10` | This is the maximum number of parallel connections each Router makes to the state-store. |
|`yarn.federation.cache-ttl.secs` | `60` | The Router caches informations, and this is the time to leave before the cache is invalidated. |
@@ -306,4 +307,5 @@ The output from this particular example job should be something like:
Job Finished in 30.586 seconds
Estimated value of Pi is 3.14250000......
+The state of the job can also be tracked on the Router Web UI at `routerhost:8089`.
Note that no change in the code or recompilation of the input jar was required to use federation. Also, the output of this job is the exact same as it would be when run without federation. Also, in order to get the full benefit of federation, use a large enough number of mappers such that more than one cluster is required. That number happens to be 16 in the case of the above example.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/35] hadoop git commit: HADOOP-14651. Update okhttp version to
2.7.5. Contributed by Ray Chiang and John Zhuge.
Posted by ae...@apache.org.
HADOOP-14651. Update okhttp version to 2.7.5. Contributed by Ray Chiang and John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8aca46e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8aca46e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8aca46e5
Branch: refs/heads/HDFS-7240
Commit: 8aca46e53f5962217851dc2103211f99a4ef7258
Parents: f40dbc1
Author: John Zhuge <jz...@cloudera.com>
Authored: Wed Sep 13 20:36:31 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Fri Sep 29 12:15:06 2017 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 3 ++-
hadoop-tools/hadoop-azure-datalake/pom.xml | 7 ++++---
2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aca46e5/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 915bd4e..66d2eb4 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -102,6 +102,7 @@
<ehcache.version>3.3.1</ehcache.version>
<hikari.version>2.4.12</hikari.version>
<mssql.version>6.2.1.jre7</mssql.version>
+ <okhttp.version>2.7.5</okhttp.version>
<!-- define the Java language version used by the compiler -->
<javac.version>1.8</javac.version>
@@ -147,7 +148,7 @@
<dependency>
<groupId>com.squareup.okhttp</groupId>
<artifactId>okhttp</artifactId>
- <version>2.4.0</version>
+ <version>${okhttp.version}</version>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aca46e5/hadoop-tools/hadoop-azure-datalake/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
index e32f9fc..660bfb9 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -29,7 +29,7 @@
</description>
<packaging>jar</packaging>
<properties>
- <okHttpVersion>2.4.0</okHttpVersion>
+ <okHttpVersion>${okhttp.version}</okHttpVersion>
<minimalJsonVersion>0.9.1</minimalJsonVersion>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
@@ -120,7 +120,8 @@
<dependency>
<groupId>com.squareup.okhttp</groupId>
<artifactId>okhttp</artifactId>
- <version>2.4.0</version>
+ <version>${okhttp.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -142,7 +143,7 @@
<dependency>
<groupId>com.squareup.okhttp</groupId>
<artifactId>mockwebserver</artifactId>
- <version>2.4.0</version>
+ <version>${okhttp.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/35] hadoop git commit: YARN-2037. Add work preserving restart
support for Unmanaged AMs. (Botong Huang via Subru).
Posted by ae...@apache.org.
YARN-2037. Add work preserving restart support for Unmanaged AMs. (Botong Huang via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4d2fd1a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4d2fd1a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4d2fd1a
Branch: refs/heads/HDFS-7240
Commit: d4d2fd1acd2fdddf04f45e67897804eea30d79a1
Parents: 015abcd
Author: Subru Krishnan <su...@apache.org>
Authored: Mon Oct 2 18:14:44 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Mon Oct 2 18:14:44 2017 -0700
----------------------------------------------------------------------
.../yarn/api/ApplicationMasterProtocol.java | 21 ++-
.../records/ApplicationSubmissionContext.java | 17 +-
.../ApplicationMasterService.java | 26 +--
.../resourcemanager/DefaultAMSProcessor.java | 5 +
.../rmapp/attempt/RMAppAttemptImpl.java | 6 +-
.../scheduler/AbstractYarnScheduler.java | 13 +-
.../TestWorkPreservingUnmanagedAM.java | 159 +++++++++++++++++++
7 files changed, 214 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
index 4d78961..eb40fc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
@@ -55,27 +55,32 @@ public interface ApplicationMasterProtocol {
* The interface used by a new <code>ApplicationMaster</code> to register with
* the <code>ResourceManager</code>.
* </p>
- *
+ *
* <p>
* The <code>ApplicationMaster</code> needs to provide details such as RPC
* Port, HTTP tracking url etc. as specified in
* {@link RegisterApplicationMasterRequest}.
* </p>
- *
+ *
* <p>
* The <code>ResourceManager</code> responds with critical details such as
* maximum resource capabilities in the cluster as specified in
* {@link RegisterApplicationMasterResponse}.
* </p>
- *
- * @param request
- * registration request
+ *
+ * <p>
+ * Re-register is only allowed for <code>Unmanaged Application Master</code>
+ * (UAM) HA, with
+ * {@link org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#getKeepContainersAcrossApplicationAttempts()}
+ * set to true.
+ * </p>
+ *
+ * @param request registration request
* @return registration respose
* @throws YarnException
* @throws IOException
- * @throws InvalidApplicationMasterRequestException
- * The exception is thrown when an ApplicationMaster tries to
- * register more then once.
+ * @throws InvalidApplicationMasterRequestException The exception is thrown
+ * when an ApplicationMaster tries to register more then once.
* @see RegisterApplicationMasterRequest
* @see RegisterApplicationMasterResponse
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index 4f1d147..a6bbca7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -395,15 +395,18 @@ public abstract class ApplicationSubmissionContext {
* Set the flag which indicates whether to keep containers across application
* attempts.
* <p>
- * If the flag is true, running containers will not be killed when application
- * attempt fails and these containers will be retrieved by the new application
- * attempt on registration via
+ * For managed AM, if the flag is true, running containers will not be killed
+ * when application attempt fails and these containers will be retrieved by
+ * the new application attempt on registration via
* {@link ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)}.
* </p>
- *
- * @param keepContainers
- * the flag which indicates whether to keep containers across
- * application attempts.
+ * <p>
+ * For unmanaged AM, if the flag is true, RM allows re-register and returns
+ * the running containers in the same attempt back to the UAM for HA.
+ * </p>
+ *
+ * @param keepContainers the flag which indicates whether to keep containers
+ * across application attempts.
*/
@Public
@Stable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 479aa43..90c42be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterReque
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
@@ -65,8 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.security
- .AMRMTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
import org.apache.hadoop.yarn.server.utils.AMRMClientUtils;
@@ -213,14 +213,20 @@ public class ApplicationMasterService extends AbstractService implements
synchronized (lock) {
AllocateResponse lastResponse = lock.getAllocateResponse();
if (hasApplicationMasterRegistered(applicationAttemptId)) {
- String message = AMRMClientUtils.APP_ALREADY_REGISTERED_MESSAGE + appID;
- LOG.warn(message);
- RMAuditLogger.logFailure(
- this.rmContext.getRMApps()
- .get(appID).getUser(),
- AuditConstants.REGISTER_AM, "", "ApplicationMasterService", message,
- appID, applicationAttemptId);
- throw new InvalidApplicationMasterRequestException(message);
+ // allow UAM re-register if work preservation is enabled
+ ApplicationSubmissionContext appContext =
+ rmContext.getRMApps().get(appID).getApplicationSubmissionContext();
+ if (!(appContext.getUnmanagedAM()
+ && appContext.getKeepContainersAcrossApplicationAttempts())) {
+ String message =
+ AMRMClientUtils.APP_ALREADY_REGISTERED_MESSAGE + appID;
+ LOG.warn(message);
+ RMAuditLogger.logFailure(
+ this.rmContext.getRMApps().get(appID).getUser(),
+ AuditConstants.REGISTER_AM, "", "ApplicationMasterService",
+ message, appID, applicationAttemptId);
+ throw new InvalidApplicationMasterRequestException(message);
+ }
}
this.amLivelinessMonitor.receivedPing(applicationAttemptId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index 5632efe..3eef270 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -148,6 +148,11 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
.getTransferredContainers(applicationAttemptId);
if (!transferredContainers.isEmpty()) {
response.setContainersFromPreviousAttempts(transferredContainers);
+ // Clear the node set remembered by the secret manager. Necessary
+ // for UAM restart because we use the same attemptId.
+ rmContext.getNMTokenSecretManager()
+ .clearNodeSetForAttempt(applicationAttemptId);
+
List<NMToken> nmTokens = new ArrayList<NMToken>();
for (Container container : transferredContainers) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index fa29e00..1b4d613 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -363,7 +363,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
// Transitions from RUNNING State
.addTransition(RMAppAttemptState.RUNNING, RMAppAttemptState.RUNNING,
- RMAppAttemptEventType.LAUNCHED)
+ EnumSet.of(
+ RMAppAttemptEventType.LAUNCHED,
+ // Valid only for UAM restart
+ RMAppAttemptEventType.REGISTERED))
.addTransition(RMAppAttemptState.RUNNING, RMAppAttemptState.FINAL_SAVING,
RMAppAttemptEventType.UNREGISTERED, new AMUnregisteredTransition())
.addTransition(RMAppAttemptState.RUNNING, RMAppAttemptState.RUNNING,
@@ -1236,7 +1239,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
if (appAttempt.submissionContext
.getKeepContainersAcrossApplicationAttempts()
- && !appAttempt.submissionContext.getUnmanagedAM()
&& rmApp.getCurrentAppAttempt() != appAttempt) {
appAttempt.transferStateFromAttempt(rmApp.getCurrentAppAttempt());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 4896ab0..9d6037e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -243,17 +243,18 @@ public abstract class AbstractYarnScheduler
ApplicationId appId = currentAttempt.getApplicationId();
SchedulerApplication<T> app = applications.get(appId);
List<Container> containerList = new ArrayList<Container>();
- RMApp appImpl = this.rmContext.getRMApps().get(appId);
- if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) {
- return containerList;
- }
if (app == null) {
return containerList;
}
Collection<RMContainer> liveContainers =
app.getCurrentAppAttempt().getLiveContainers();
- ContainerId amContainerId = rmContext.getRMApps().get(appId)
- .getCurrentAppAttempt().getMasterContainer().getId();
+ ContainerId amContainerId = null;
+ // For UAM, amContainer would be null
+ if (rmContext.getRMApps().get(appId).getCurrentAppAttempt()
+ .getMasterContainer() != null) {
+ amContainerId = rmContext.getRMApps().get(appId).getCurrentAppAttempt()
+ .getMasterContainer().getId();
+ }
for (RMContainer rmContainer : liveContainers) {
if (!rmContainer.getContainerId().equals(amContainerId)) {
containerList.add(rmContainer.getContainer());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java
new file mode 100644
index 0000000..80f7319
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test UAM handling in RM.
+ */
+public class TestWorkPreservingUnmanagedAM
+ extends ParameterizedSchedulerTestBase {
+
+ private YarnConfiguration conf;
+
+ public TestWorkPreservingUnmanagedAM(SchedulerType type) throws IOException {
+ super(type);
+ }
+
+ @Before
+ public void setup() {
+ Logger rootLogger = LogManager.getRootLogger();
+ rootLogger.setLevel(Level.DEBUG);
+ conf = getConf();
+ UserGroupInformation.setConfiguration(conf);
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ }
+
+ /**
+ * Test UAM work preserving restart. When the keepContainersAcrossAttempt flag
+ * is on, we allow UAM to directly register again and move on without getting
+ * the applicationAlreadyRegistered exception.
+ */
+ protected void testUAMRestart(boolean keepContainers) throws Exception {
+ // start RM
+ MockRM rm = new MockRM();
+ rm.start();
+ MockNM nm =
+ new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
+ nm.registerNode();
+
+ // create app and launch the UAM
+ boolean unamanged = true;
+ int maxAttempts = 1;
+ boolean waitForAccepted = true;
+ RMApp app = rm.submitApp(200, "",
+ UserGroupInformation.getCurrentUser().getShortUserName(), null,
+ unamanged, null, maxAttempts, null, null, waitForAccepted,
+ keepContainers);
+
+ MockAM am = MockRM.launchUAM(app, rm, nm);
+
+ // Register for the first time
+ am.registerAppAttempt();
+
+ // Allocate two containers to UAM
+ int numContainers = 3;
+ List<Container> conts = am.allocate("127.0.0.1", 1000, numContainers,
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ while (conts.size() < numContainers) {
+ nm.nodeHeartbeat(true);
+ conts.addAll(am.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers());
+ Thread.sleep(100);
+ }
+
+ // Release one container
+ List<ContainerId> releaseList =
+ Collections.singletonList(conts.get(0).getId());
+ List<ContainerStatus> finishedConts =
+ am.allocate(new ArrayList<ResourceRequest>(), releaseList)
+ .getCompletedContainersStatuses();
+ while (finishedConts.size() < releaseList.size()) {
+ nm.nodeHeartbeat(true);
+ finishedConts
+ .addAll(am
+ .allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>())
+ .getCompletedContainersStatuses());
+ Thread.sleep(100);
+ }
+
+ // Register for the second time
+ RegisterApplicationMasterResponse response = null;
+ try {
+ response = am.registerAppAttempt(false);
+ } catch (InvalidApplicationMasterRequestException e) {
+ Assert.assertEquals(false, keepContainers);
+ return;
+ }
+ Assert.assertEquals("RM should not allow second register"
+ + " for UAM without keep container flag ", true, keepContainers);
+
+ // Expecting the two running containers previously
+ Assert.assertEquals(2, response.getContainersFromPreviousAttempts().size());
+ Assert.assertEquals(1, response.getNMTokensFromPreviousAttempts().size());
+
+ // Allocate one more containers to UAM, just to be safe
+ numContainers = 1;
+ am.allocate("127.0.0.1", 1000, numContainers, new ArrayList<ContainerId>());
+ nm.nodeHeartbeat(true);
+ conts = am.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ while (conts.size() < numContainers) {
+ nm.nodeHeartbeat(true);
+ conts.addAll(am.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers());
+ Thread.sleep(100);
+ }
+
+ rm.stop();
+ }
+
+ @Test(timeout = 600000)
+ public void testUAMRestartKeepContainers() throws Exception {
+ testUAMRestart(true);
+ }
+
+ @Test(timeout = 600000)
+ public void testUAMRestartNoKeepContainers() throws Exception {
+ testUAMRestart(false);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/35] hadoop git commit: YARN-6623. Add support to turn off
launching privileged containers in the container-executor. (Varun Vasudev via
wangda)
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
new file mode 100644
index 0000000..c627ca8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -0,0 +1,1122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+#include <fstream>
+#include "errno.h"
+
+extern "C" {
+#include "utils/docker-util.c"
+}
+
+namespace ContainerExecutor {
+
+ class TestDockerUtil : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+ docker_command_file = "docker-command.cmd";
+ container_executor_cfg_file = "container-executor.cfg";
+ container_executor_cfg.size = 0;
+ container_executor_cfg.sections = NULL;
+ }
+
+ virtual void TearDown() {
+ remove(docker_command_file.c_str());
+ remove(container_executor_cfg_file.c_str());
+ delete_ce_file();
+ }
+
+ struct configuration container_executor_cfg;
+ std::string docker_command_file;
+ std::string container_executor_cfg_file;
+
+
+ void write_file(const std::string fname, const std::string contents) {
+ std::ofstream command_file;
+ command_file.open(fname.c_str());
+ command_file << contents;
+ command_file.close();
+ }
+
+ int create_ce_file() {
+ int ret = 0;
+ const char *fname = HADOOP_CONF_DIR "/" CONF_FILENAME;
+ if (strcmp("../etc/hadoop/container-executor.cfg", fname) == 0) {
+ ret = mkdir("../etc", 0755);
+ if (ret == 0 || errno == EEXIST) {
+ ret = mkdir("../etc/hadoop", 0755);
+ if (ret == 0 || errno == EEXIST) {
+ write_file("../etc/hadoop/container-executor.cfg", "");
+ return 0;
+ } else {
+ std::cerr << "Could not create ../etc/hadoop, " << strerror(errno) << std::endl;
+ }
+ } else {
+ std::cerr << "Could not create ../etc, " << strerror(errno) << std::endl;
+ }
+ }
+ std::cerr << "Could not create " << fname << std::endl;
+ return 1;
+ }
+
+ void delete_ce_file() {
+ const char *fname = HADOOP_CONF_DIR "/" CONF_FILENAME;
+ if (strcmp("../etc/hadoop/container-executor.cfg", fname) == 0) {
+ struct stat buffer;
+ if (stat(fname, &buffer) == 0) {
+ remove("../etc/hadoop/container-executor.cfg");
+ rmdir("../etc/hadoop");
+ rmdir("../etc");
+ }
+ }
+ }
+
+ void write_container_executor_cfg(const std::string contents) {
+ write_file(container_executor_cfg_file, contents);
+ }
+
+ void write_command_file(const std::string contents) {
+ write_file(docker_command_file, contents);
+ }
+
+ void run_docker_command_test(const std::vector<std::pair<std::string, std::string> > &file_cmd_vec,
+ const std::vector<std::pair<std::string, int> > &bad_file_cmd_vec,
+ int (*docker_func)(const char *, const struct configuration *, char *, const size_t)) {
+ char tmp[8192];
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(tmp, 0, 8192);
+ write_command_file(itr->first);
+ int ret = (*docker_func)(docker_command_file.c_str(), &container_executor_cfg, tmp, 8192);
+ ASSERT_EQ(0, ret) << "error message: " << get_docker_error_message(ret) << " for input " << itr->first;
+ ASSERT_STREQ(itr->second.c_str(), tmp);
+ }
+
+ std::vector<std::pair<std::string, int> >::const_iterator itr2;
+ for (itr2 = bad_file_cmd_vec.begin(); itr2 != bad_file_cmd_vec.end(); ++itr2) {
+ memset(tmp, 0, 8192);
+ write_command_file(itr2->first);
+ int ret = (*docker_func)(docker_command_file.c_str(), &container_executor_cfg, tmp, 8192);
+ ASSERT_EQ(itr2->second, ret) << " for " << itr2->first << std::endl;
+ ASSERT_EQ(0, strlen(tmp));
+ }
+ int ret = (*docker_func)("unknown-file", &container_executor_cfg, tmp, 8192);
+ ASSERT_EQ(static_cast<int>(INVALID_COMMAND_FILE), ret);
+ }
+
+ void run_docker_run_helper_function(const std::vector<std::pair<std::string, std::string> > &file_cmd_vec,
+ int (*helper_func)(const struct configuration *, char *, const size_t)) {
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ for(itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ struct configuration cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ int ret = read_config(docker_command_file.c_str(), &cfg);
+ if(ret == 0) {
+ ret = (*helper_func)(&cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ }
+ }
+ };
+
+ TEST_F(TestDockerUtil, test_docker_inspect) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=inspect\n format={{.State.Status}}\n name=container_e1_12312_11111_02_000001",
+ "inspect --format={{.State.Status}} container_e1_12312_11111_02_000001"));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=inspect\n"
+ " format={{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}\n"
+ " name=container_e1_12312_11111_02_000001",
+ "inspect --format={{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}} container_e1_12312_11111_02_000001"));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n format='{{.State.Status}}'",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(
+ std::make_pair<std::string, int>("docker-command=inspect\n format='{{.State.Status}}'",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n format={{.State.Status}}\n name=",
+ static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n format={{.State.Status}}",
+ static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n format=\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INVALID_DOCKER_INSPECT_FORMAT)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INVALID_DOCKER_INSPECT_FORMAT)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n format={{.IPAddress}}\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INVALID_DOCKER_INSPECT_FORMAT)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_inspect_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_load) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=load\n image=image-id",
+ "load --i='image-id' "));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=image-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=load\n image=image-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=load\n image=", static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>("[docker-command-execution]\n docker-command=load",
+ static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_load_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_validate_image_name) {
+ const char *good_input[] = {
+ "ubuntu",
+ "ubuntu:latest",
+ "ubuntu:14.04",
+ "ubuntu:LATEST",
+ "registry.com:5000/user/ubuntu",
+ "registry.com:5000/user/ubuntu:latest",
+ "registry.com:5000/user/ubuntu:0.1.2.3",
+ "registry.com/user/ubuntu",
+ "registry.com/user/ubuntu:latest",
+ "registry.com/user/ubuntu:0.1.2.3",
+ "registry.com/user/ubuntu:test-image",
+ "registry.com/user/ubuntu:test_image",
+ "registry.com/ubuntu",
+ "user/ubuntu",
+ "user/ubuntu:0.1.2.3",
+ "user/ubuntu:latest",
+ "user/ubuntu:test_image",
+ "user/ubuntu.test:test_image",
+ "user/ubuntu-test:test-image",
+ "registry.com/ubuntu/ubuntu/ubuntu"
+ };
+
+ const char *bad_input[] = {
+ "UBUNTU",
+ "registry.com|5000/user/ubuntu",
+ "registry.com | 5000/user/ubuntu",
+ "ubuntu' || touch /tmp/file #",
+ "ubuntu || touch /tmp/file #",
+ "''''''''",
+ "bad_host_name:5000/user/ubuntu",
+ "registry.com:foo/ubuntu/ubuntu/ubuntu",
+ "registry.com/ubuntu:foo/ubuntu/ubuntu"
+ };
+
+ int good_input_size = sizeof(good_input) / sizeof(char *);
+ int i = 0;
+ for (i = 0; i < good_input_size; i++) {
+ int op = validate_docker_image_name(good_input[i]);
+ ASSERT_EQ(0, op);
+ }
+
+ int bad_input_size = sizeof(bad_input) / sizeof(char *);
+ int j = 0;
+ for (j = 0; j < bad_input_size; j++) {
+ int op = validate_docker_image_name(bad_input[j]);
+ ASSERT_EQ(1, op);
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_docker_pull) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=pull\n image=image-id",
+ "pull 'image-id' "));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=image-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=pull\n image=image-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=pull\n image=", static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>("[docker-command-execution]\n docker-command=pull",
+ static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_pull_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_rm) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(
+ std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=rm\n name=container_e1_12312_11111_02_000001",
+ "rm container_e1_12312_11111_02_000001"));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=rm\n name=ctr-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=rm\n name=", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=rm", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_rm_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_stop) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=stop\n name=container_e1_12312_11111_02_000001",
+ "stop container_e1_12312_11111_02_000001"));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=stop\n name=container_e1_12312_11111_02_000001\ntime=25",
+ "stop --time=25 container_e1_12312_11111_02_000001"));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=stop\n name=ctr-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=stop\n name=", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=stop", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=stop\n name=container_e1_12312_11111_02_000001\n time=abcd",
+ static_cast<int>(INVALID_DOCKER_STOP_COMMAND)));
+
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_stop_command);
+ }
+
+ TEST_F(TestDockerUtil, test_detach_container) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n detach=true", "-d "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, detach_container);
+ }
+
+ TEST_F(TestDockerUtil, test_rm_container) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rm=true", "--rm "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, rm_container_on_exit);
+ }
+
+ TEST_F(TestDockerUtil, test_set_container_workdir) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n workdir=/tmp/test", "--workdir='/tmp/test' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, set_container_workdir);
+ }
+
+ TEST_F(TestDockerUtil, test_set_cgroup_parent) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n cgroup-parent=/sys/fs/cgroup/yarn",
+ "--cgroup-parent='/sys/fs/cgroup/yarn' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, set_cgroup_parent);
+ }
+
+ TEST_F(TestDockerUtil, test_set_hostname) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n hostname=ctr-id", "--hostname='ctr-id' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, set_hostname);
+ }
+
+ TEST_F(TestDockerUtil, test_set_group_add) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n group-add=1000,1001", "--group-add '1000' --group-add '1001' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, set_group_add);
+ }
+
+ TEST_F(TestDockerUtil, test_set_network) {
+ struct configuration container_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.networks=sdn1,bridge";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n net=bridge", "--net='bridge' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n net=sdn1", "--net='sdn1' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ if (ret != 0) {
+ FAIL();
+ }
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ struct configuration cmd_cfg;
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_network(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ struct configuration cmd_cfg_1;
+ write_command_file("[docker-command-execution]\n docker-command=run\n net=sdn2");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg_1);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_network(&cmd_cfg_1, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_NETWORK, ret);
+ ASSERT_EQ(0, strlen(buff));
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_network(&cmd_cfg_1, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_NETWORK, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+ TEST_F(TestDockerUtil, test_check_mount_permitted) {
+ const char *permitted_mounts[] = {"/usr/", "/bin/ls", NULL};
+ std::vector<std::pair<std::string, int> > test_data;
+ test_data.push_back(std::make_pair<std::string, int>("/usr", 1));
+ test_data.push_back(std::make_pair<std::string, int>("/usr/", 1));
+ test_data.push_back(std::make_pair<std::string, int>("/bin/ls", 1));
+ test_data.push_back(std::make_pair<std::string, int>("//bin/", 0));
+ test_data.push_back(std::make_pair<std::string, int>("/tmp/random-file", -1));
+
+ std::vector<std::pair<std::string, int> >::const_iterator itr;
+ for (itr = test_data.begin(); itr != test_data.end(); ++itr) {
+ int ret = check_mount_permitted(permitted_mounts, itr->first.c_str());
+ ASSERT_EQ(itr->second, ret) << "for input " << itr->first;
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_normalize_mounts) {
+ const int entries = 4;
+ const char *permitted_mounts[] = {"/home", "/usr", "/bin/ls", NULL};
+ const char *expected[] = {"/home/", "/usr/", "/bin/ls", NULL};
+ char **ptr = static_cast<char **>(malloc(entries * sizeof(char *)));
+ for (int i = 0; i < entries; ++i) {
+ if (permitted_mounts[i] != NULL) {
+ ptr[i] = strdup(permitted_mounts[i]);
+ } else {
+ ptr[i] = NULL;
+ }
+ }
+ normalize_mounts(ptr);
+ for (int i = 0; i < entries; ++i) {
+ ASSERT_STREQ(expected[i], ptr[i]);
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_set_privileged) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents[] = {"[docker]\n docker.privileged-containers.enabled=1",
+ "[docker]\n docker.privileged-containers.enabled=0",
+ "[docker]\n"};
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n privileged=true", "--privileged "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n privileged=false", ""));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+ write_container_executor_cfg(container_executor_cfg_contents[0]);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ if (ret != 0) {
+ FAIL();
+ }
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+
+ // check default case and when it's turned off
+ for (int i = 1; i < 3; ++i) {
+ write_container_executor_cfg(container_executor_cfg_contents[i]);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ file_cmd_vec.clear();
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n privileged=false", ""));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ write_command_file("[docker-command-execution]\n docker-command=run\n privileged=true");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(PRIVILEGED_CONTAINERS_DISABLED, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_set_capabilities) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.capabilities=CHROOT,MKNOD";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n cap-add=CHROOT,MKNOD",
+ "--cap-drop='ALL' --cap-add='CHROOT' --cap-add='MKNOD' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n cap-add=CHROOT", "--cap-drop='ALL' --cap-add='CHROOT' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", "--cap-drop='ALL' "));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ if (ret != 0) {
+ FAIL();
+ }
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_capabilities(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ write_command_file("[docker-command-execution]\n docker-command=run\n cap-add=SETGID");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_capabilities(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret);
+ ASSERT_EQ(0, strlen(buff));
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_capabilities(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+ TEST_F(TestDockerUtil, test_set_devices) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.devices=/dev/test-device,/dev/device2";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n devices=/dev/test-device:/dev/test-device",
+ "--device='/dev/test-device:/dev/test-device' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n devices=/dev/device2:/dev/device2",
+ "--device='/dev/device2:/dev/device2' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n "
+ "devices=/dev/test-device:/dev/test-device,/dev/device2:/dev/device2",
+ "--device='/dev/test-device:/dev/test-device' --device='/dev/device2:/dev/device2' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n", ""));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ if (ret != 0) {
+ FAIL();
+ }
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ write_command_file("[docker-command-execution]\n docker-command=run\n devices=/dev/device3:/dev/device3");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
+ ASSERT_EQ(0, strlen(buff));
+
+ write_command_file("[docker-command-execution]\n docker-command=run\n devices=/dev/device1");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
+ ASSERT_EQ(0, strlen(buff));
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+
+ TEST_F(TestDockerUtil, test_add_rw_mounts) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/usr,/var,/bin/ls,..\n "
+ "docker.allowed.ro-mounts=/bin/cat";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/var:/var", "-v '/var:/var' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/var/:/var/", "-v '/var/:/var/' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr:/usr", "-v '/usr:/usr' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/:/usr", "-v '/usr/:/usr' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/bin/ls:/bin/ls", "-v '/bin/ls:/bin/ls' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/bin:/mydisk1,/var/log/:/mydisk2",
+ "-v '/usr/bin:/mydisk1' -v '/var/log/:/mydisk2' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n", ""));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+
+ std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/home:/home",
+ static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/bin/cat:/bin/cat",
+ static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/blah:/blah",
+ static_cast<int>(INVALID_DOCKER_MOUNT)));
+
+ std::vector<std::pair<std::string, int> >::const_iterator itr2;
+
+ for (itr2 = bad_file_cmds_vec.begin(); itr2 != bad_file_cmds_vec.end(); ++itr2) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr2->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(itr2->second, ret);
+ ASSERT_STREQ("", buff);
+ }
+
+ // verify that you can't mount any directory in the container-executor.cfg path
+ char *ce_path = realpath("../etc/hadoop/container-executor.cfg", NULL);
+ while (strlen(ce_path) != 0) {
+ std::string cmd_file_contents = "[docker-command-execution]\n docker-command=run\n rw-mounts=";
+ cmd_file_contents.append(ce_path).append(":").append("/etc/hadoop");
+ memset(buff, 0, buff_len);
+ write_command_file(cmd_file_contents);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_RW_MOUNT, ret) << " for input " << cmd_file_contents;
+ ASSERT_STREQ("", buff);
+ char *tmp = strrchr(ce_path, '/');
+ if (tmp != NULL) {
+ *tmp = '\0';
+ }
+ }
+ free(ce_path);
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_RW_MOUNT, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+ TEST_F(TestDockerUtil, test_add_ro_mounts) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/usr,/var,/bin/ls\n "
+ "docker.allowed.ro-mounts=/bin/cat,/bin/ln";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/var:/var", "-v '/var:/var:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/var/:/var/", "-v '/var/:/var/:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr:/usr", "-v '/usr:/usr:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr/:/usr", "-v '/usr/:/usr:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/bin/ls:/bin/ls", "-v '/bin/ls:/bin/ls:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/bin/ln:/bin/ln", "-v '/bin/ln:/bin/ln:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/bin/cat:/bin/cat",
+ "-v '/bin/cat:/bin/cat:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr/bin:/mydisk1,/bin/cat:/bin/cat",
+ "-v '/usr/bin:/mydisk1:ro' -v '/bin/cat:/bin/cat:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n", ""));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = add_ro_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+
+ std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/home:/home",
+ static_cast<int>(INVALID_DOCKER_RO_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/blah:/blah",
+ static_cast<int>(INVALID_DOCKER_MOUNT)));
+
+ std::vector<std::pair<std::string, int> >::const_iterator itr2;
+
+ for (itr2 = bad_file_cmds_vec.begin(); itr2 != bad_file_cmds_vec.end(); ++itr2) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr2->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = add_ro_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(itr2->second, ret);
+ ASSERT_STREQ("", buff);
+ }
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ write_command_file("[docker-command-execution]\n docker-command=run\n ro-mounts=/home:/home");
+ strcpy(buff, "test string");
+ ret = add_ro_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_RO_MOUNT, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+ TEST_F(TestDockerUtil, test_docker_run_privileged) {
+
+ std::string container_executor_contents = "[docker]\n docker.allowed.ro-mounts=/var,/etc,/bin/ls\n"
+ " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n "
+ " docker.privileged-containers.enabled=1\n docker.allowed.capabilities=CHOWN,SETUID\n"
+ " docker.allowed.devices=/dev/test";
+ write_file(container_executor_cfg_file, container_executor_contents);
+ int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
+ " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' "
+ "'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
+ "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'"
+ " 'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' "
+ "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' "
+ "'bash' 'test_script.sh' 'arg1' 'arg2' "));
+
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' "
+ "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --group-add '1000' --group-add '1001' "
+ "--device='/dev/test:/dev/test' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=docker-image\n user=test",
+ static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n user=test\n",
+ static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n",
+ static_cast<int>(INVALID_DOCKER_USER_NAME)));
+
+ // invalid rw mount
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/var/log:/var/log\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
+
+ // invalid ro mount
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/bin:/bin,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_RO_MOUNT)));
+
+ // invalid capability
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID,SETGID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_CAPABILITY)));
+
+ // invalid device
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/dev1:/dev/dev1\n privileged=true\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_DEVICE)));
+
+ // invalid network
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n privileged=true\n net=host\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_NETWORK)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_run_no_privileged) {
+
+ std::string container_executor_contents[] = {"[docker]\n docker.allowed.ro-mounts=/var,/etc,/bin/ls\n"
+ " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n "
+ " docker.allowed.capabilities=CHOWN,SETUID\n"
+ " docker.allowed.devices=/dev/test",
+ "[docker]\n docker.allowed.ro-mounts=/var,/etc,/bin/ls\n"
+ " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n "
+ " docker.allowed.capabilities=CHOWN,SETUID\n"
+ " privileged=0\n"
+ " docker.allowed.devices=/dev/test"};
+ for (int i = 0; i < 2; ++i) {
+ write_file(container_executor_cfg_file, container_executor_contents[i]);
+ int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n"
+ " user=test\n launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
+ " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' "
+ "'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
+ "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'"
+ " 'test_script.sh' 'arg1' 'arg2' "));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(PRIVILEGED_CONTAINERS_DISABLED)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_docker_config_param) {
+ std::vector<std::pair<std::string, std::string> > input_output_map;
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=inspect\n docker-config=/my-config\n"
+ " format={{.State.Status}}\n name=container_e1_12312_11111_02_000001",
+ "--config='/my-config' inspect --format={{.State.Status}} container_e1_12312_11111_02_000001"));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=load\n docker-config=/my-config\n image=image-id",
+ "--config='/my-config' load --i='image-id' "));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=pull\n docker-config=/my-config\n image=image-id",
+ "--config='/my-config' pull 'image-id' "));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=rm\n docker-config=/my-config\n name=container_e1_12312_11111_02_000001",
+ "--config='/my-config' rm container_e1_12312_11111_02_000001"));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=stop\n docker-config=/my-config\n name=container_e1_12312_11111_02_000001",
+ "--config='/my-config' stop container_e1_12312_11111_02_000001"));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n docker-config=/my-config\n name=container_e1_12312_11111_02_000001\n"
+ " image=docker-image\n user=test",
+ "--config='/my-config' run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ char buffer[4096];
+ struct configuration cfg = {0, NULL};
+ for (itr = input_output_map.begin(); itr != input_output_map.end(); ++itr) {
+ memset(buffer, 0, 4096);
+ write_command_file(itr->first);
+ int ret = get_docker_command(docker_command_file.c_str(), &cfg, buffer, 4096);
+ ASSERT_EQ(0, ret) << "for input " << itr->first;
+ ASSERT_STREQ(itr->second.c_str(), buffer);
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_docker_module_enabled) {
+
+ std::vector<std::pair<std::string, int> > input_out_vec;
+ input_out_vec.push_back(std::make_pair<std::string, int>("[docker]\n module.enabled=true", 1));
+ input_out_vec.push_back(std::make_pair<std::string, int>("[docker]\n module.enabled=false", 0));
+ input_out_vec.push_back(std::make_pair<std::string, int>("[docker]\n module.enabled=1", 0));
+ input_out_vec.push_back(std::make_pair<std::string, int>("[docker]\n", 0));
+
+ for (size_t i = 0; i < input_out_vec.size(); ++i) {
+ write_file(container_executor_cfg_file, input_out_vec[i].first);
+ int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = docker_module_enabled(&container_executor_cfg);
+ ASSERT_EQ(input_out_vec[i].second, ret) << " incorrect output for "
+ << input_out_vec[i].first;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index e548790..fbfee54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -295,32 +295,37 @@ public class TestDockerContainerRuntime {
List<String> args = op.getArguments();
String dockerCommandFile = args.get(11);
- //This is the expected docker invocation for this case
- StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ")
- .append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=host ")
- .append("--hostname=" + defaultHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ")
- .append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ")
- .append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ").append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- String expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
-
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals(expectedCommand, dockerCommands.get(0));
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
}
@Test
@@ -347,10 +352,13 @@ public class TestDockerContainerRuntime {
String uid = "";
String gid = "";
+ String[] groups = {};
Shell.ShellCommandExecutor shexec1 = new Shell.ShellCommandExecutor(
new String[]{"id", "-u", runAsUser});
Shell.ShellCommandExecutor shexec2 = new Shell.ShellCommandExecutor(
new String[]{"id", "-g", runAsUser});
+ Shell.ShellCommandExecutor shexec3 = new Shell.ShellCommandExecutor(
+ new String[]{"id", "-G", runAsUser});
try {
shexec1.execute();
// get rid of newline at the end
@@ -365,37 +373,48 @@ public class TestDockerContainerRuntime {
} catch (Exception e) {
LOG.info("Could not run id -g command: " + e);
}
+ try {
+ shexec3.execute();
+ groups = shexec3.getOutput().replace("\n", " ").split(" ");
+ } catch (Exception e) {
+ LOG.info("Could not run id -G command: " + e);
+ }
uidGidPair = uid + ":" + gid;
- //This is the expected docker invocation for this case
- StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ")
- .append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=host ")
- .append("--hostname=" + defaultHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ")
- .append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ")
- .append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ")
- .append("(--group-add \\d+ )*")
- .append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- String expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, uidGidPair,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
-
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
- //Assert.assertEquals(expectedCommand, dockerCommands.get(0));
- Assert.assertTrue(dockerCommands.get(0).matches(expectedCommand));
+ Assert.assertEquals(14, dockerCommands.size());
+ int counter = 0;
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id",
+ dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
}
@Test
@@ -481,29 +500,38 @@ public class TestDockerContainerRuntime {
String dockerCommandFile = args.get(11);
//This is the expected docker invocation for this case
- StringBuffer expectedCommandTemplate =
- new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=" + allowedNetwork + " ")
- .append("--hostname=" + expectedHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ").append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- String expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
-
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
-
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals(expectedCommand, dockerCommands.get(0));
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=test.hostname",
+ dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" net=" + allowedNetwork, dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
}
@Test
@@ -537,30 +565,37 @@ public class TestDockerContainerRuntime {
//This is the expected docker invocation for this case. customNetwork1
// ("sdn1") is the expected network to be used in this case
- StringBuffer expectedCommandTemplate =
- new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=" + customNetwork1 + " ")
- .append("--hostname=" + defaultHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ").append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- String expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
-
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals(expectedCommand, dockerCommands.get(0));
-
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=sdn1", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
//now set an explicit (non-default) allowedNetwork and ensure that it is
// used.
@@ -575,28 +610,37 @@ public class TestDockerContainerRuntime {
//This is the expected docker invocation for this case. customNetwork2
// ("sdn2") is the expected network to be used in this case
- expectedCommandTemplate =
- new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=" + customNetwork2 + " ")
- .append("--hostname=" + defaultHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ").append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
+ counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=sdn2", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals(expectedCommand, dockerCommands.get(0));
//disallowed network should trigger a launch failure
@@ -630,7 +674,8 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
+ int expected = 13;
+ Assert.assertEquals(expected, dockerCommands.size());
String command = dockerCommands.get(0);
@@ -738,13 +783,35 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
-
- String command = dockerCommands.get(0);
-
- //submitting user is whitelisted. ensure --privileged is in the invocation
- Assert.assertTrue("Did not find expected '--privileged' in docker run args "
- + ": " + command, command.contains("--privileged"));
+ int expected = 14;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(" privileged=true", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
}
@Test
@@ -833,15 +900,33 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
-
- String command = dockerCommands.get(0);
-
- Assert.assertTrue("Did not find expected " +
- "/test_local_dir/test_resource_file:test_mount mount in docker " +
- "run args : " + command,
- command.contains(" -v /test_local_dir/test_resource_file:test_mount" +
- ":ro "));
+ Assert.assertEquals(14, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(1));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2));
+ Assert.assertEquals(" detach=true", dockerCommands.get(3));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(4));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5));
+ Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(7));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(8));
+ Assert.assertEquals(" net=host", dockerCommands.get(9));
+ Assert.assertEquals(
+ " ro-mounts=/test_local_dir/test_resource_file:test_mount",
+ dockerCommands.get(10));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(11));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(12));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(13));
}
@Test
@@ -885,20 +970,35 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
-
- String command = dockerCommands.get(0);
+ Assert.assertEquals(14, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(1));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2));
+ Assert.assertEquals(" detach=true", dockerCommands.get(3));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(4));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5));
+ Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(7));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(8));
+ Assert.assertEquals(" net=host", dockerCommands.get(9));
+ Assert.assertEquals(
+ " ro-mounts=/test_local_dir/test_resource_file:test_mount1,"
+ + "/test_local_dir/test_resource_file:test_mount2",
+ dockerCommands.get(10));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(11));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(12));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(13));
- Assert.assertTrue("Did not find expected " +
- "/test_local_dir/test_resource_file:test_mount1 mount in docker " +
- "run args : " + command,
- command.contains(" -v /test_local_dir/test_resource_file:test_mount1" +
- ":ro "));
- Assert.assertTrue("Did not find expected " +
- "/test_local_dir/test_resource_file:test_mount2 mount in docker " +
- "run args : " + command,
- command.contains(" -v /test_local_dir/test_resource_file:test_mount2" +
- ":ro "));
}
@Test
@@ -930,8 +1030,10 @@ public class TestDockerContainerRuntime {
IOException {
List<String> dockerCommands = getDockerCommandsForSignal(
ContainerExecutor.Signal.TERM);
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals("stop container_id", dockerCommands.get(0));
+ Assert.assertEquals(3, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" docker-command=stop", dockerCommands.get(1));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(2));
}
@Test
@@ -940,8 +1042,10 @@ public class TestDockerContainerRuntime {
IOException {
List<String> dockerCommands = getDockerCommandsForSignal(
ContainerExecutor.Signal.KILL);
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals("stop container_id", dockerCommands.get(0));
+ Assert.assertEquals(3, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" docker-command=stop", dockerCommands.get(1));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(2));
}
@Test
@@ -950,8 +1054,10 @@ public class TestDockerContainerRuntime {
IOException {
List<String> dockerCommands = getDockerCommandsForSignal(
ContainerExecutor.Signal.QUIT);
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals("stop container_id", dockerCommands.get(0));
+ Assert.assertEquals(3, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" docker-command=stop", dockerCommands.get(1));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(2));
}
private List<String> getDockerCommandsForSignal(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/35] hadoop git commit: HDFS-12552. Use slf4j instead of log4j in
FSNamesystem. Contributed by Ajay Kumar.
Posted by ae...@apache.org.
HDFS-12552. Use slf4j instead of log4j in FSNamesystem. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a68c2ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a68c2ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a68c2ce
Branch: refs/heads/HDFS-7240
Commit: 5a68c2cef31031522f2532064965f482017dfefd
Parents: 563dcdf
Author: Chen Liang <cl...@apache.org>
Authored: Mon Oct 2 11:26:53 2017 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Mon Oct 2 11:26:53 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 9 ++++++---
.../org/apache/hadoop/hdfs/TestFileConcurrentReader.java | 8 ++++----
.../java/org/apache/hadoop/hdfs/TestFileCreation.java | 8 ++++----
.../org/apache/hadoop/hdfs/TestFileCreationClient.java | 10 +++++-----
.../test/java/org/apache/hadoop/hdfs/TestFileStatus.java | 6 +++---
.../java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java | 8 ++++----
.../java/org/apache/hadoop/hdfs/TestPersistBlocks.java | 6 +++---
.../java/org/apache/hadoop/hdfs/TestReadWhileWriting.java | 6 +++---
.../blockmanagement/TestBlocksWithNotEnoughRacks.java | 6 +++---
.../hadoop/hdfs/server/namenode/TestFSNamesystemLock.java | 2 +-
10 files changed, 36 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 4094694..dc50808 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -310,6 +310,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.LoggerFactory;
/**
* FSNamesystem is a container of both transient
@@ -342,7 +343,9 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
@Metrics(context="dfs")
public class FSNamesystem implements Namesystem, FSNamesystemMBean,
NameNodeMXBean, ReplicatedBlocksMBean, ECBlockGroupsMBean {
- public static final Log LOG = LogFactory.getLog(FSNamesystem.class);
+
+ public static final org.slf4j.Logger LOG = LoggerFactory
+ .getLogger(FSNamesystem.class.getName());
private final MetricsRegistry registry = new MetricsRegistry("FSNamesystem");
@Metric final MutableRatesWithAggregation detailedLockHoldTimeMetrics =
registry.newRatesWithAggregation("detailedLockHoldTimeMetrics");
@@ -1671,8 +1674,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
stopStandbyServices();
} catch (IOException ie) {
} finally {
- IOUtils.cleanup(LOG, dir);
- IOUtils.cleanup(LOG, fsImage);
+ IOUtils.cleanupWithLogger(LOG, dir);
+ IOUtils.cleanupWithLogger(LOG, fsImage);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java
index e0a0d3b..7557197 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java
@@ -38,12 +38,12 @@ import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.StringUtils;
-import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
+import org.slf4j.event.Level;
/**
@@ -62,9 +62,9 @@ public class TestFileConcurrentReader {
Logger.getLogger(TestFileConcurrentReader.class);
{
- GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(DFSClient.LOG, Level.TRACE);
}
static final long seed = 0xDEADBEEFL;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
index e191414..a738343 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
@@ -86,9 +86,9 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
-import org.apache.log4j.Level;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.event.Level;
/**
* This class tests various cases during file creation.
@@ -97,9 +97,9 @@ public class TestFileCreation {
static final String DIR = "/" + TestFileCreation.class.getSimpleName() + "/";
{
- GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(DFSClient.LOG, Level.TRACE);
}
private static final String RPC_DETAILED_METRICS =
"RpcDetailedActivityForPort";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationClient.java
index db291a0..986bb56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationClient.java
@@ -30,8 +30,8 @@ import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
import org.junit.Test;
+import org.slf4j.event.Level;
/**
* This class tests client lease recovery.
@@ -40,10 +40,10 @@ public class TestFileCreationClient {
static final String DIR = "/" + TestFileCreationClient.class.getSimpleName() + "/";
{
- GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(InterDatanodeProtocol.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(DataNode.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(InterDatanodeProtocol.LOG, Level.TRACE);
}
/** Test lease recovery Triggered by DFSClient. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
index 31007dd..a5f8911 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
@@ -38,18 +38,18 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.event.Level;
/**
* This class tests the FileStatus API.
*/
public class TestFileStatus {
{
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FileSystem.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(FileSystem.LOG, Level.TRACE);
}
static final long seed = 0xDEADBEEFL;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
index a807bbb..a96d8b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
@@ -52,21 +52,21 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import org.slf4j.event.Level;
public class TestLeaseRecovery2 {
public static final Log LOG = LogFactory.getLog(TestLeaseRecovery2.class);
{
- GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(DataNode.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
}
static final private long BLOCK_SIZE = 1024;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java
index 54eb934..4a99493 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java
@@ -43,8 +43,8 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
-import org.apache.log4j.Level;
import org.junit.Test;
+import org.slf4j.event.Level;
/**
* A JUnit test for checking if restarting DFS preserves the
@@ -52,8 +52,8 @@ import org.junit.Test;
*/
public class TestPersistBlocks {
static {
- GenericTestUtils.setLogLevel(FSImage.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSImage.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
}
private static final int BLOCK_SIZE = 4096;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadWhileWriting.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadWhileWriting.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadWhileWriting.java
index 2b55d25..8de1799 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadWhileWriting.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadWhileWriting.java
@@ -32,15 +32,15 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.event.Level;
/** Test reading from hdfs while a file is being written. */
public class TestReadWhileWriting {
{
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(DFSClient.LOG, Level.TRACE);
}
private static final String DIR = "/"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
index 0298cbc..2bf6045 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
@@ -40,14 +40,14 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.util.HostsFileWriter;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
import org.junit.Test;
+import org.slf4j.event.Level;
public class TestBlocksWithNotEnoughRacks {
public static final Log LOG = LogFactory.getLog(TestBlocksWithNotEnoughRacks.class);
static {
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(LOG, Level.TRACE);
}
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
index 94e0b33..2daf5c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.test.MetricsAsserts;
import org.apache.hadoop.util.FakeTimer;
-import org.apache.log4j.Level;
import org.junit.Test;
import java.io.IOException;
@@ -39,6 +38,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.regex.Pattern;
import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
import static org.junit.Assert.*;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FSLOCK_FAIR_KEY;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/35] hadoop git commit: HADOOP-14902. LoadGenerator#genFile write
close timing is incorrectly calculated. Contributed by Hanisha Koneru
Posted by ae...@apache.org.
HADOOP-14902. LoadGenerator#genFile write close timing is incorrectly calculated. Contributed by Hanisha Koneru
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f789fe0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f789fe0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f789fe0
Branch: refs/heads/HDFS-7240
Commit: 6f789fe05766a61b12ca10df3f26ee354eac84aa
Parents: ca669f9
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Sep 28 16:38:30 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Sep 28 16:38:30 2017 -0500
----------------------------------------------------------------------
.../apache/hadoop/fs/loadGenerator/LoadGenerator.java | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f789fe0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
index 0bb1b46..b74e75d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
@@ -322,6 +322,7 @@ public class LoadGenerator extends Configured implements Tool {
private void genFile(Path file, long fileSize) throws IOException {
long startTimestamp = Time.monotonicNow();
FSDataOutputStream out = null;
+ boolean isOutClosed = false;
try {
out = fc.create(file,
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
@@ -337,11 +338,15 @@ public class LoadGenerator extends Configured implements Tool {
i -= s;
}
- startTimestamp = Time.monotonicNow();
- executionTime[WRITE_CLOSE] += (Time.monotonicNow() - startTimestamp);
+ startTime = Time.monotonicNow();
+ out.close();
+ executionTime[WRITE_CLOSE] += (Time.monotonicNow() - startTime);
numOfOps[WRITE_CLOSE]++;
+ isOutClosed = true;
} finally {
- IOUtils.cleanupWithLogger(LOG, out);
+ if (!isOutClosed && out != null) {
+ out.close();
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/35] hadoop git commit: YARN-6962. Add support for
updateContainers when allocating using FederationInterceptor. (Botong Huang
via Subru).
Posted by ae...@apache.org.
YARN-6962. Add support for updateContainers when allocating using FederationInterceptor. (Botong Huang via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca669f9f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca669f9f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca669f9f
Branch: refs/heads/HDFS-7240
Commit: ca669f9f8bc7abe5b7d4648c589aa1756bd336d1
Parents: 85d81ae5
Author: Subru Krishnan <su...@apache.org>
Authored: Thu Sep 28 13:04:03 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Thu Sep 28 13:04:03 2017 -0700
----------------------------------------------------------------------
.../amrmproxy/FederationInterceptor.java | 86 +++++++++++++-------
.../amrmproxy/TestFederationInterceptor.java | 54 ++++++++++++
2 files changed, 111 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca669f9f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
index 28724aa..33cfca3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
@@ -540,30 +540,33 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
}
}
- if (request.getResourceBlacklistRequest() != null && !isNullOrEmpty(
- request.getResourceBlacklistRequest().getBlacklistAdditions())) {
- for (String resourceName : request.getResourceBlacklistRequest()
- .getBlacklistAdditions()) {
- SubClusterId subClusterId = getSubClusterForNode(resourceName);
- if (subClusterId != null) {
- AllocateRequest newRequest = findOrCreateAllocateRequestForSubCluster(
- subClusterId, request, requestMap);
- newRequest.getResourceBlacklistRequest().getBlacklistAdditions()
- .add(resourceName);
+ if (request.getResourceBlacklistRequest() != null) {
+ if (!isNullOrEmpty(
+ request.getResourceBlacklistRequest().getBlacklistAdditions())) {
+ for (String resourceName : request.getResourceBlacklistRequest()
+ .getBlacklistAdditions()) {
+ SubClusterId subClusterId = getSubClusterForNode(resourceName);
+ if (subClusterId != null) {
+ AllocateRequest newRequest =
+ findOrCreateAllocateRequestForSubCluster(subClusterId, request,
+ requestMap);
+ newRequest.getResourceBlacklistRequest().getBlacklistAdditions()
+ .add(resourceName);
+ }
}
}
- }
-
- if (request.getResourceBlacklistRequest() != null && !isNullOrEmpty(
- request.getResourceBlacklistRequest().getBlacklistRemovals())) {
- for (String resourceName : request.getResourceBlacklistRequest()
- .getBlacklistRemovals()) {
- SubClusterId subClusterId = getSubClusterForNode(resourceName);
- if (subClusterId != null) {
- AllocateRequest newRequest = findOrCreateAllocateRequestForSubCluster(
- subClusterId, request, requestMap);
- newRequest.getResourceBlacklistRequest().getBlacklistRemovals()
- .add(resourceName);
+ if (!isNullOrEmpty(
+ request.getResourceBlacklistRequest().getBlacklistRemovals())) {
+ for (String resourceName : request.getResourceBlacklistRequest()
+ .getBlacklistRemovals()) {
+ SubClusterId subClusterId = getSubClusterForNode(resourceName);
+ if (subClusterId != null) {
+ AllocateRequest newRequest =
+ findOrCreateAllocateRequestForSubCluster(subClusterId, request,
+ requestMap);
+ newRequest.getResourceBlacklistRequest().getBlacklistRemovals()
+ .add(resourceName);
+ }
}
}
}
@@ -896,13 +899,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
}
}
- if (!isNullOrEmpty(otherResponse.getNMTokens())) {
- if (!isNullOrEmpty(homeResponse.getNMTokens())) {
- homeResponse.getNMTokens().addAll(otherResponse.getNMTokens());
- } else {
- homeResponse.setNMTokens(otherResponse.getNMTokens());
- }
- }
+ homeResponse.setNumClusterNodes(
+ homeResponse.getNumClusterNodes() + otherResponse.getNumClusterNodes());
PreemptionMessage homePreempMessage = homeResponse.getPreemptionMessage();
PreemptionMessage otherPreempMessage = otherResponse.getPreemptionMessage();
@@ -935,6 +933,31 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
spar1.getContainers().addAll(spar2.getContainers());
}
}
+
+ if (!isNullOrEmpty(otherResponse.getNMTokens())) {
+ if (!isNullOrEmpty(homeResponse.getNMTokens())) {
+ homeResponse.getNMTokens().addAll(otherResponse.getNMTokens());
+ } else {
+ homeResponse.setNMTokens(otherResponse.getNMTokens());
+ }
+ }
+
+ if (!isNullOrEmpty(otherResponse.getUpdatedContainers())) {
+ if (!isNullOrEmpty(homeResponse.getUpdatedContainers())) {
+ homeResponse.getUpdatedContainers()
+ .addAll(otherResponse.getUpdatedContainers());
+ } else {
+ homeResponse.setUpdatedContainers(otherResponse.getUpdatedContainers());
+ }
+ }
+
+ if (!isNullOrEmpty(otherResponse.getUpdateErrors())) {
+ if (!isNullOrEmpty(homeResponse.getUpdateErrors())) {
+ homeResponse.getUpdateErrors().addAll(otherResponse.getUpdateErrors());
+ } else {
+ homeResponse.setUpdateErrors(otherResponse.getUpdateErrors());
+ }
+ }
}
/**
@@ -1052,6 +1075,11 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
return this.uamPool.getAllUAMIds().size();
}
+ @VisibleForTesting
+ public Map<SubClusterId, List<AllocateResponse>> getAsyncResponseSink() {
+ return this.asyncResponseSink;
+ }
+
/**
* Private structure for encapsulating SubClusterId and
* RegisterApplicationMasterResponse instances.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca669f9f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
index 34b0741..3db0e35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
@@ -20,7 +20,9 @@ package org.apache.hadoop.yarn.server.nodemanager.amrmproxy;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
@@ -36,8 +38,15 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NMToken;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.PreemptionMessage;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerError;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -493,4 +502,49 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
} catch (YarnException e) {
}
}
+
+ @Test
+ public void testAllocateResponse() throws Exception {
+ interceptor.registerApplicationMaster(
+ RegisterApplicationMasterRequest.newInstance(null, 0, null));
+ AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
+
+ Map<SubClusterId, List<AllocateResponse>> asyncResponseSink =
+ interceptor.getAsyncResponseSink();
+
+ ContainerId cid = ContainerId.newContainerId(attemptId, 0);
+ ContainerStatus cStatus = Records.newRecord(ContainerStatus.class);
+ cStatus.setContainerId(cid);
+ Container container =
+ Container.newInstance(cid, null, null, null, null, null);
+
+ AllocateResponse response = Records.newRecord(AllocateResponse.class);
+ response.setAllocatedContainers(Collections.singletonList(container));
+ response.setCompletedContainersStatuses(Collections.singletonList(cStatus));
+ response.setUpdatedNodes(
+ Collections.singletonList(Records.newRecord(NodeReport.class)));
+ response.setNMTokens(
+ Collections.singletonList(Records.newRecord(NMToken.class)));
+ response.setUpdatedContainers(
+ Collections.singletonList(Records.newRecord(UpdatedContainer.class)));
+ response.setUpdateErrors(Collections
+ .singletonList(Records.newRecord(UpdateContainerError.class)));
+ response.setAvailableResources(Records.newRecord(Resource.class));
+ response.setPreemptionMessage(Records.newRecord(PreemptionMessage.class));
+
+ List<AllocateResponse> list = new ArrayList<>();
+ list.add(response);
+ asyncResponseSink.put(SubClusterId.newInstance("SC-1"), list);
+
+ response = interceptor.allocate(allocateRequest);
+
+ Assert.assertEquals(1, response.getAllocatedContainers().size());
+ Assert.assertNotNull(response.getAvailableResources());
+ Assert.assertEquals(1, response.getCompletedContainersStatuses().size());
+ Assert.assertEquals(1, response.getUpdatedNodes().size());
+ Assert.assertNotNull(response.getPreemptionMessage());
+ Assert.assertEquals(1, response.getNMTokens().size());
+ Assert.assertEquals(1, response.getUpdatedContainers().size());
+ Assert.assertEquals(1, response.getUpdateErrors().size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/35] hadoop git commit: HDFS-12271. Incorrect statement in
Downgrade section of HDFS Rolling Upgrade document. Contributed by
Nandakumar.
Posted by ae...@apache.org.
HDFS-12271. Incorrect statement in Downgrade section of HDFS Rolling Upgrade document. Contributed by Nandakumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c62ff7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c62ff7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c62ff7f
Branch: refs/heads/HDFS-7240
Commit: 2c62ff7fd4392d55fe41573da297e2c0bc4f4c41
Parents: 442ea85
Author: Chen Liang <cl...@apache.org>
Authored: Mon Oct 2 13:18:00 2017 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Mon Oct 2 13:18:00 2017 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c62ff7f/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
index cabdd74..386ab6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
@@ -179,7 +179,7 @@ Below are the steps for rolling downgrade without downtime:
1. Start *NN2* as standby normally.
1. Failover from *NN1* to *NN2*
so that *NN2* becomes active and *NN1* becomes standby.
- 1. Shutdown and upgrade *NN1*.
+ 1. Shutdown and downgrade *NN1*.
1. Start *NN1* as standby normally.
1. Finalize Rolling Downgrade
1. Run "[`hdfs dfsadmin -rollingUpgrade finalize`](#dfsadmin_-rollingUpgrade)"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/35] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Posted by ae...@apache.org.
Merge branch 'trunk' into HDFS-7240
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75ce3fcc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75ce3fcc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75ce3fcc
Branch: refs/heads/HDFS-7240
Commit: 75ce3fcc766a66c63afd4912922f0991afbcd1d5
Parents: 579434a 453d48b
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 3 09:25:36 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 3 09:25:36 2017 -0700
----------------------------------------------------------------------
.../hadoop/fs/shell/TestCopyFromLocal.java | 15 +-
.../hadoop-hdfs/src/main/bin/start-dfs.sh | 2 +-
.../hadoop-hdfs/src/main/bin/stop-dfs.sh | 2 +-
.../server/namenode/FSDirErasureCodingOp.java | 3 +-
.../hdfs/server/namenode/FSDirXAttrOp.java | 4 +
.../hdfs/server/namenode/FSNamesystem.java | 9 +-
.../src/site/markdown/HdfsRollingUpgrade.md | 2 +-
.../hadoop/hdfs/TestFileConcurrentReader.java | 8 +-
.../apache/hadoop/hdfs/TestFileCreation.java | 8 +-
.../hadoop/hdfs/TestFileCreationClient.java | 10 +-
.../org/apache/hadoop/hdfs/TestFileStatus.java | 6 +-
.../apache/hadoop/hdfs/TestLeaseRecovery2.java | 8 +-
.../apache/hadoop/hdfs/TestPersistBlocks.java | 6 +-
.../hadoop/hdfs/TestReadWhileWriting.java | 6 +-
.../TestUnsetAndChangeDirectoryEcPolicy.java | 13 +
.../TestBlocksWithNotEnoughRacks.java | 6 +-
.../server/namenode/TestFSNamesystemLock.java | 2 +-
.../hadoop/mapred/LocalContainerLauncher.java | 13 +-
.../hadoop/mapred/TaskAttemptListenerImpl.java | 11 +-
.../org/apache/hadoop/mapred/YarnChild.java | 19 +-
.../apache/hadoop/mapred/YarnOutputFiles.java | 2 -
.../jobhistory/JobHistoryCopyService.java | 7 +-
.../jobhistory/JobHistoryEventHandler.java | 6 +-
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 15 +-
.../mapreduce/v2/app/TaskHeartbeatHandler.java | 7 +-
.../v2/app/client/MRClientService.java | 6 +-
.../v2/app/commit/CommitterEventHandler.java | 8 +-
.../mapreduce/v2/app/job/impl/JobImpl.java | 6 +-
.../v2/app/job/impl/TaskAttemptImpl.java | 7 +-
.../mapreduce/v2/app/job/impl/TaskImpl.java | 6 +-
.../v2/app/launcher/ContainerLauncherImpl.java | 7 +-
.../v2/app/local/LocalContainerAllocator.java | 8 +-
.../mapreduce/v2/app/rm/RMCommunicator.java | 7 +-
.../v2/app/rm/RMContainerAllocator.java | 6 +-
.../v2/app/rm/RMContainerRequestor.java | 7 +-
.../CheckpointAMPreemptionPolicy.java | 8 +-
.../rm/preemption/KillAMPreemptionPolicy.java | 8 +-
.../v2/app/speculate/DefaultSpeculator.java | 7 +-
.../mapreduce/v2/app/webapp/AppController.java | 7 +-
.../mapred/TestLocalContainerLauncher.java | 8 +-
.../jobhistory/TestJobHistoryEventHandler.java | 8 +-
.../mapreduce/jobhistory/TestJobSummary.java | 7 +-
.../apache/hadoop/mapreduce/v2/app/MRApp.java | 6 +-
.../hadoop/mapreduce/v2/app/MRAppBenchmark.java | 8 +-
.../mapreduce/v2/app/TestMRAppMaster.java | 15 +-
.../hadoop/mapreduce/v2/app/TestRecovery.java | 6 +-
.../mapreduce/v2/app/TestRuntimeEstimators.java | 7 +-
.../v2/app/job/impl/TestMapReduceChildJVM.java | 7 +-
.../mapreduce/v2/app/job/impl/TestTaskImpl.java | 6 +-
.../v2/app/launcher/TestContainerLauncher.java | 9 +-
.../app/launcher/TestContainerLauncherImpl.java | 7 +-
.../v2/app/rm/TestRMContainerAllocator.java | 8 +-
.../yarn/api/ApplicationMasterProtocol.java | 21 +-
.../records/ApplicationSubmissionContext.java | 17 +-
.../ifile/IndexedFileAggregatedLogsBlock.java | 14 +-
.../LogAggregationIndexedFileController.java | 397 +++++++++++++------
.../TestLogAggregationIndexFileController.java | 67 +++-
.../server/nodemanager/ContainerExecutor.java | 44 +-
.../nodemanager/LinuxContainerExecutor.java | 8 +
.../launcher/ContainerLaunch.java | 25 --
.../runtime/DefaultLinuxContainerRuntime.java | 6 +
.../DelegatingLinuxContainerRuntime.java | 11 +
.../runtime/DockerLinuxContainerRuntime.java | 7 +
.../runtime/ContainerRuntime.java | 11 +
.../launcher/TestContainerLaunch.java | 73 +++-
.../ApplicationMasterService.java | 26 +-
.../resourcemanager/DefaultAMSProcessor.java | 5 +
.../rmapp/attempt/RMAppAttemptImpl.java | 6 +-
.../scheduler/AbstractYarnScheduler.java | 13 +-
.../TestWorkPreservingUnmanagedAM.java | 159 ++++++++
.../yarn/server/router/webapp/AboutBlock.java | 91 +++++
.../yarn/server/router/webapp/AboutPage.java | 37 ++
.../yarn/server/router/webapp/AppsBlock.java | 130 ++++++
.../yarn/server/router/webapp/AppsPage.java | 77 ++++
.../server/router/webapp/FederationBlock.java | 176 ++++++++
.../server/router/webapp/FederationPage.java | 57 +++
.../yarn/server/router/webapp/NavBlock.java | 46 +++
.../yarn/server/router/webapp/NodesBlock.java | 109 +++++
.../yarn/server/router/webapp/NodesPage.java | 60 +++
.../server/router/webapp/RouterController.java | 60 +++
.../yarn/server/router/webapp/RouterView.java | 52 +++
.../yarn/server/router/webapp/RouterWebApp.java | 6 +
.../src/site/markdown/Federation.md | 6 +-
83 files changed, 1807 insertions(+), 400 deletions(-)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/35] hadoop git commit: YARN-6623. Add support to turn off
launching privileged containers in the container-executor. (Varun Vasudev via
wangda)
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
index 60fce40..05b44b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
@@ -114,8 +114,10 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("rm " + MOCK_CONTAINER_ID, dockerCommands.get(0));
+ assertEquals(3, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=rm", dockerCommands.get(1));
+ assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(2));
}
@Test
@@ -130,8 +132,10 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("stop " + MOCK_CONTAINER_ID, dockerCommands.get(0));
+ assertEquals(3, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=stop", dockerCommands.get(1));
+ assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(2));
}
@Test
@@ -147,9 +151,12 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("inspect --format='{{.State.Status}}' " + MOCK_CONTAINER_ID,
- dockerCommands.get(0));
+ assertEquals(4, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=inspect", dockerCommands.get(1));
+ assertEquals(" format={{.State.Status}}", dockerCommands.get(2));
+ assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(3));
+
}
@Test
@@ -165,8 +172,10 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("pull " + MOCK_IMAGE_NAME, dockerCommands.get(0));
+ assertEquals(3, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=pull", dockerCommands.get(1));
+ assertEquals(" image=" + MOCK_IMAGE_NAME, dockerCommands.get(2));
}
@Test
@@ -182,8 +191,12 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("load --i=" + MOCK_LOCAL_IMAGE_NAME, dockerCommands.get(0));
+ assertEquals(3, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=load", dockerCommands.get(1));
+ assertEquals(" image=" + MOCK_LOCAL_IMAGE_NAME, dockerCommands.get(2));
+
+
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
index 619f202..4092e6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -44,16 +46,29 @@ public class TestDockerInspectCommand {
@Test
public void testGetContainerStatus() throws Exception {
dockerInspectCommand.getContainerStatus();
- assertEquals("inspect --format='{{.State.Status}}' foo",
- dockerInspectCommand.getCommandWithArguments());
+ assertEquals("inspect", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("{{.State.Status}}", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments().get("format")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals(3,
+ dockerInspectCommand.getDockerCommandWithArguments().size());
}
@Test
public void testGetIpAndHost() throws Exception {
dockerInspectCommand.getIpAndHost();
- assertEquals(
- "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}}"
- + ",{{end}}{{.Config.Hostname}}' foo",
- dockerInspectCommand.getCommandWithArguments());
+ assertEquals("inspect", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("{{range(.NetworkSettings.Networks)}}"
+ + "{{.IPAddress}},{{end}}{{.Config.Hostname}}", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments().get("format")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals(3,
+ dockerInspectCommand.getDockerCommandWithArguments().size());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java
index 85fa0f8..e5bff26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -42,7 +43,11 @@ public class TestDockerLoadCommand {
@Test
public void testGetCommandWithArguments() {
- assertEquals("load --i=foo",
- dockerLoadCommand.getCommandWithArguments());
+ assertEquals("load", StringUtils.join(",",
+ dockerLoadCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerLoadCommand.getDockerCommandWithArguments().get("image")));
+ assertEquals(2, dockerLoadCommand.getDockerCommandWithArguments().size());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java
index 89157ff..ccf7000 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -42,7 +43,12 @@ public class TestDockerPullCommand {
@Test
public void testGetCommandWithArguments() {
- assertEquals("pull foo", dockerPullCommand.getCommandWithArguments());
+ assertEquals("pull", StringUtils.join(",",
+ dockerPullCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerPullCommand.getDockerCommandWithArguments().get("image")));
+ assertEquals(2, dockerPullCommand.getDockerCommandWithArguments().size());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
index d1b9904..a8d4bdd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -42,7 +44,11 @@ public class TestDockerRmCommand {
@Test
public void testGetCommandWithArguments() {
- assertEquals("rm foo", dockerRmCommand.getCommandWithArguments());
+ assertEquals("rm", StringUtils.join(",",
+ dockerRmCommand.getDockerCommandWithArguments().get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerRmCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals(2, dockerRmCommand.getDockerCommandWithArguments().size());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
index 85bccd2..e51d7ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -56,8 +57,24 @@ public class TestDockerRunCommand {
commands.add("launch_command");
dockerRunCommand.setOverrideCommandWithArgs(commands);
dockerRunCommand.removeContainerOnExit();
- assertEquals("run --name=foo --user=user_id --device=source:dest --rm "
- + "image_name launch_command",
- dockerRunCommand.getCommandWithArguments());
+
+ assertEquals("run", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals("user_id", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments().get("user")));
+ assertEquals("image_name", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments().get("image")));
+
+ assertEquals("source:dest", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments().get("devices")));
+ assertEquals("true", StringUtils
+ .join(",", dockerRunCommand.getDockerCommandWithArguments().get("rm")));
+ assertEquals("launch_command", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments()
+ .get("launch-command")));
+ assertEquals(7, dockerRunCommand.getDockerCommandWithArguments().size());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java
index c9743f3..efbde77 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java
@@ -21,6 +21,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -48,8 +50,13 @@ public class TestDockerStopCommand {
@Test
public void testSetGracePeriod() throws Exception {
dockerStopCommand.setGracePeriod(GRACE_PERIOD);
- assertEquals("stop foo --time=10",
- dockerStopCommand.getCommandWithArguments());
-
+ assertEquals("stop", StringUtils.join(",",
+ dockerStopCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerStopCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals("10", StringUtils.join(",",
+ dockerStopCommand.getDockerCommandWithArguments().get("time")));
+ assertEquals(3, dockerStopCommand.getDockerCommandWithArguments().size());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index 23f4134..36c391a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -167,7 +167,24 @@ The following properties are required to enable Docker support:
|Configuration Name | Description |
|:---- |:---- |
| `yarn.nodemanager.linux-container-executor.group` | The Unix group of the NodeManager. It should match the yarn.nodemanager.linux-container-executor.group in the yarn-site.xml file. |
-| `feature.docker.enabled` | Must be 0 or 1. 0 means launching Docker containers is disabled. 1 means launching Docker containers is allowed. |
+
+The container-executor.cfg must contain a section to determine the capabilities that containers
+are allowed. It contains the following properties:
+
+|Configuration Name | Description |
+|:---- |:---- |
+| `module.enabled` | Must be "true" or "false" to enable or disable launching Docker containers respectively. Default value is 0. |
+| `docker.binary` | The binary used to launch Docker containers. /usr/bin/docker by default. |
+| `docker.allowed.capabilities` | Comma separated capabilities that containers are allowed to add. By default no capabilities are allowed to be added. |
+| `docker.allowed.devices` | Comma separated devices that containers are allowed to mount. By default no devices are allowed to be added. |
+| `docker.allowed.networks` | Comma separated networks that containers are allowed to use. If no network is specified when launching the container, the default Docker network will be used. |
+| `docker.allowed.ro-mounts` | Comma separated directories that containers are allowed to mount in read-only mode. By default, no directories are allowed to mounted. |
+| `docker.allowed.rw-mounts` | Comma separated directories that containers are allowed to mount in read-write mode. By default, no directories are allowed to mounted. |
+| `docker.privileged-containers.enabled` | Set to 1 or 0 to enable or disable launching privileged containers. Default value is 0. |
+
+Please note that if you wish to run Docker containers that require access to the YARN local directories, you must add them to the docker.allowed.rw-mounts list.
+
+In addition, containers are not permitted to mount any parent of the container-executor.cfg directory in read-write mode.
The following properties are optional:
@@ -176,9 +193,21 @@ The following properties are optional:
| `min.user.id` | The minimum UID that is allowed to launch applications. The default is no minimum |
| `banned.users` | A comma-separated list of usernames who should not be allowed to launch applications. The default setting is: yarn, mapred, hdfs, and bin. |
| `allowed.system.users` | A comma-separated list of usernames who should be allowed to launch applications even if their UIDs are below the configured minimum. If a user appears in allowed.system.users and banned.users, the user will be considered banned. |
-| `docker.binary` | The path to the Docker binary. The default is "docker". |
| `feature.tc.enabled` | Must be 0 or 1. 0 means traffic control commands are disabled. 1 means traffic control commands are allowed. |
+Part of a container-executor.cfg which allows Docker containers to be launched is below:
+
+```
+yarn.nodemanager.linux-container-executor.group=yarn
+[docker]
+ module.enabled=true
+ docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE
+ docker.allowed.networks=bridge,host,none
+ docker.allowed.ro-mounts=/sys/fs/cgroup
+ docker.allowed.rw-mounts=/var/hadoop/yarn/local-dir,/var/hadoop/yarn/log-dir
+
+```
+
Docker Image Requirements
-------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/35] hadoop git commit: MAPREDUCE-6971. Moving logging APIs over
to slf4j in hadoop-mapreduce-client-app. Contributed by Jinjiang Ling.
Posted by ae...@apache.org.
MAPREDUCE-6971. Moving logging APIs over to slf4j in hadoop-mapreduce-client-app. Contributed by Jinjiang Ling.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/453d48bd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/453d48bd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/453d48bd
Branch: refs/heads/HDFS-7240
Commit: 453d48bdfbb67ed3e66c33c4aef239c3d7bdd3bc
Parents: 7eb8499
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Oct 3 12:14:54 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Oct 3 12:14:54 2017 +0900
----------------------------------------------------------------------
.../hadoop/mapred/LocalContainerLauncher.java | 13 +++++++------
.../hadoop/mapred/TaskAttemptListenerImpl.java | 11 ++++++-----
.../java/org/apache/hadoop/mapred/YarnChild.java | 19 ++++++++-----------
.../apache/hadoop/mapred/YarnOutputFiles.java | 2 --
.../jobhistory/JobHistoryCopyService.java | 7 ++++---
.../jobhistory/JobHistoryEventHandler.java | 6 +++---
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 15 ++++++---------
.../mapreduce/v2/app/TaskHeartbeatHandler.java | 7 ++++---
.../mapreduce/v2/app/client/MRClientService.java | 6 +++---
.../v2/app/commit/CommitterEventHandler.java | 8 ++++----
.../mapreduce/v2/app/job/impl/JobImpl.java | 6 +++---
.../v2/app/job/impl/TaskAttemptImpl.java | 7 ++++---
.../mapreduce/v2/app/job/impl/TaskImpl.java | 6 +++---
.../v2/app/launcher/ContainerLauncherImpl.java | 7 ++++---
.../v2/app/local/LocalContainerAllocator.java | 8 ++++----
.../mapreduce/v2/app/rm/RMCommunicator.java | 7 ++++---
.../v2/app/rm/RMContainerAllocator.java | 6 +++---
.../v2/app/rm/RMContainerRequestor.java | 7 ++++---
.../preemption/CheckpointAMPreemptionPolicy.java | 8 ++++----
.../rm/preemption/KillAMPreemptionPolicy.java | 8 ++++----
.../v2/app/speculate/DefaultSpeculator.java | 7 ++++---
.../mapreduce/v2/app/webapp/AppController.java | 7 ++++---
.../mapred/TestLocalContainerLauncher.java | 8 ++++----
.../jobhistory/TestJobHistoryEventHandler.java | 8 ++++----
.../mapreduce/jobhistory/TestJobSummary.java | 7 ++++---
.../apache/hadoop/mapreduce/v2/app/MRApp.java | 6 +++---
.../hadoop/mapreduce/v2/app/MRAppBenchmark.java | 8 +++-----
.../hadoop/mapreduce/v2/app/TestMRAppMaster.java | 15 +++++++--------
.../hadoop/mapreduce/v2/app/TestRecovery.java | 6 +++---
.../mapreduce/v2/app/TestRuntimeEstimators.java | 7 ++++---
.../v2/app/job/impl/TestMapReduceChildJVM.java | 7 ++++---
.../mapreduce/v2/app/job/impl/TestTaskImpl.java | 6 +++---
.../v2/app/launcher/TestContainerLauncher.java | 9 +++++----
.../app/launcher/TestContainerLauncherImpl.java | 7 ++++---
.../v2/app/rm/TestRMContainerAllocator.java | 8 ++++----
35 files changed, 142 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
index 0b942b0..6f9cc34 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
@@ -37,8 +37,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
@@ -70,6 +68,8 @@ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Runs the container task locally in a thread.
@@ -80,7 +80,8 @@ public class LocalContainerLauncher extends AbstractService implements
ContainerLauncher {
private static final File curDir = new File(".");
- private static final Log LOG = LogFactory.getLog(LocalContainerLauncher.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LocalContainerLauncher.class);
private FileContext curFC = null;
private Set<File> localizedFiles = new HashSet<File>();
@@ -376,7 +377,7 @@ public class LocalContainerLauncher extends AbstractService implements
// if umbilical itself barfs (in error-handler of runSubMap()),
// we're pretty much hosed, so do what YarnChild main() does
// (i.e., exit clumsily--but can never happen, so no worries!)
- LOG.fatal("oopsie... this can never happen: "
+ LOG.error("oopsie... this can never happen: "
+ StringUtils.stringifyException(ioe));
ExitUtil.terminate(-1);
} finally {
@@ -477,7 +478,7 @@ public class LocalContainerLauncher extends AbstractService implements
}
} catch (FSError e) {
- LOG.fatal("FSError from child", e);
+ LOG.error("FSError from child", e);
// umbilical: MRAppMaster creates (taskAttemptListener), passes to us
if (!ShutdownHookManager.get().isShutdownInProgress()) {
umbilical.fsError(classicAttemptID, e.getMessage());
@@ -502,7 +503,7 @@ public class LocalContainerLauncher extends AbstractService implements
throw new RuntimeException();
} catch (Throwable throwable) {
- LOG.fatal("Error running local (uberized) 'child' : "
+ LOG.error("Error running local (uberized) 'child' : "
+ StringUtils.stringifyException(throwable));
if (!ShutdownHookManager.get().isShutdownInProgress()) {
Throwable tCause = throwable.getCause();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
index 5669f3e..9b6148c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
@@ -26,8 +26,6 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.ProtocolSignature;
@@ -57,6 +55,8 @@ import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class is responsible for talking to the task umblical.
@@ -72,7 +72,8 @@ public class TaskAttemptListenerImpl extends CompositeService
private static final JvmTask TASK_FOR_INVALID_JVM = new JvmTask(null, true);
- private static final Log LOG = LogFactory.getLog(TaskAttemptListenerImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TaskAttemptListenerImpl.class);
private AppContext context;
private Server server;
@@ -274,7 +275,7 @@ public class TaskAttemptListenerImpl extends CompositeService
public void fatalError(TaskAttemptID taskAttemptID, String msg)
throws IOException {
// This happens only in Child and in the Task.
- LOG.fatal("Task: " + taskAttemptID + " - exited : " + msg);
+ LOG.error("Task: " + taskAttemptID + " - exited : " + msg);
reportDiagnosticInfo(taskAttemptID, "Error: " + msg);
org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
@@ -291,7 +292,7 @@ public class TaskAttemptListenerImpl extends CompositeService
public void fsError(TaskAttemptID taskAttemptID, String message)
throws IOException {
// This happens only in Child.
- LOG.fatal("Task: " + taskAttemptID + " - failed due to FSError: "
+ LOG.error("Task: " + taskAttemptID + " - failed due to FSError: "
+ message);
reportDiagnosticInfo(taskAttemptID, "FSError: " + message);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
index 97642a5..7ae7a1e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
@@ -26,8 +26,6 @@ import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.concurrent.ScheduledExecutorService;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
@@ -60,13 +58,15 @@ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The main() for MapReduce task processes.
*/
class YarnChild {
- private static final Log LOG = LogFactory.getLog(YarnChild.class);
+ private static final Logger LOG = LoggerFactory.getLogger(YarnChild.class);
static volatile TaskAttemptID taskid = null;
@@ -100,10 +100,7 @@ class YarnChild {
// Security framework already loaded the tokens into current ugi
Credentials credentials =
UserGroupInformation.getCurrentUser().getCredentials();
- LOG.info("Executing with tokens:");
- for (Token<?> token: credentials.getAllTokens()) {
- LOG.info(token);
- }
+ LOG.info("Executing with tokens: {}", credentials.getAllTokens());
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner =
@@ -129,7 +126,7 @@ class YarnChild {
try {
int idleLoopCount = 0;
- JvmTask myTask = null;;
+ JvmTask myTask = null;
// poll for new task
for (int idle = 0; null == myTask; ++idle) {
long sleepTimeMilliSecs = Math.min(idle * 500, 1500);
@@ -179,7 +176,7 @@ class YarnChild {
}
});
} catch (FSError e) {
- LOG.fatal("FSError from child", e);
+ LOG.error("FSError from child", e);
if (!ShutdownHookManager.get().isShutdownInProgress()) {
umbilical.fsError(taskid, e.getMessage());
}
@@ -213,7 +210,7 @@ class YarnChild {
}
}
} catch (Throwable throwable) {
- LOG.fatal("Error running child : "
+ LOG.error("Error running child : "
+ StringUtils.stringifyException(throwable));
if (taskid != null) {
if (!ShutdownHookManager.get().isShutdownInProgress()) {
@@ -352,7 +349,7 @@ class YarnChild {
out = FileSystem.create(localFs, jobFile, urw_gr);
conf.writeXml(out);
} finally {
- IOUtils.cleanup(LOG, out);
+ IOUtils.cleanupWithLogger(LOG, out);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
index e099b8f..ebcb3e7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java
index 4bb1e38..ee4ec2c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.mapreduce.jobhistory;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileContext;
@@ -34,6 +32,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Reads in history events from the JobHistoryFile and sends them out again
@@ -41,7 +41,8 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
*/
public class JobHistoryCopyService extends CompositeService implements HistoryEventHandler {
- private static final Log LOG = LogFactory.getLog(JobHistoryCopyService.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(JobHistoryCopyService.class);
private final ApplicationAttemptId applicationAttemptId;
private final EventHandler<Event> handler;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 8fa417b..ae46129 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -31,8 +31,6 @@ import java.util.TimerTask;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -82,6 +80,8 @@ import org.apache.hadoop.yarn.util.TimelineServiceHelper;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.ClientHandlerException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The job history events get routed to this class. This class writes the Job
@@ -123,7 +123,7 @@ public class JobHistoryEventHandler extends AbstractService
private volatile boolean stopped;
private final Object lock = new Object();
- private static final Log LOG = LogFactory.getLog(
+ private static final Logger LOG = LoggerFactory.getLogger(
JobHistoryEventHandler.class);
protected static final Map<JobId, MetaInfo> fileMap =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index f511f19..df61928 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -41,8 +41,6 @@ import java.util.regex.Pattern;
import javax.crypto.KeyGenerator;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -155,6 +153,8 @@ import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The Map-Reduce Application Master.
@@ -177,7 +177,7 @@ import com.google.common.annotations.VisibleForTesting;
@SuppressWarnings("rawtypes")
public class MRAppMaster extends CompositeService {
- private static final Log LOG = LogFactory.getLog(MRAppMaster.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MRAppMaster.class);
/**
* Priority of the MRAppMaster shutdown hook.
@@ -327,7 +327,7 @@ public class MRAppMaster extends CompositeService {
errorHappenedShutDown = true;
forcedState = JobStateInternal.ERROR;
shutDownMessage = "Staging dir does not exist " + stagingDir;
- LOG.fatal(shutDownMessage);
+ LOG.error(shutDownMessage);
} else if (commitStarted) {
//A commit was started so this is the last time, we just need to know
// what result we will use to notify, and how we will unregister
@@ -1667,7 +1667,7 @@ public class MRAppMaster extends CompositeService {
conf.set(MRJobConfig.USER_NAME, jobUserName);
initAndStartAppMaster(appMaster, conf, jobUserName);
} catch (Throwable t) {
- LOG.fatal("Error starting MRAppMaster", t);
+ LOG.error("Error starting MRAppMaster", t);
ExitUtil.terminate(1, t);
}
}
@@ -1716,10 +1716,7 @@ public class MRAppMaster extends CompositeService {
// them
Credentials credentials =
UserGroupInformation.getCurrentUser().getCredentials();
- LOG.info("Executing with tokens:");
- for (Token<?> token : credentials.getAllTokens()) {
- LOG.info(token);
- }
+ LOG.info("Executing with tokens: {}", credentials.getAllTokens());
UserGroupInformation appMasterUgi = UserGroupInformation
.createRemoteUser(jobUserName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
index 6a716c7..c438b35 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
@@ -24,8 +24,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.util.MRJobConfUtil;
@@ -36,6 +34,8 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.Clock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -63,7 +63,8 @@ public class TaskHeartbeatHandler extends AbstractService {
}
}
- private static final Log LOG = LogFactory.getLog(TaskHeartbeatHandler.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TaskHeartbeatHandler.class);
//thread which runs periodically to see the last time since a heartbeat is
//received from a task.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
index 6eb42be..b2dc8ad 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
@@ -23,8 +23,6 @@ import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Collection;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.http.HttpConfig.Policy;
@@ -90,6 +88,8 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This module is responsible for talking to the
@@ -98,7 +98,7 @@ import org.apache.hadoop.yarn.webapp.WebApps;
*/
public class MRClientService extends AbstractService implements ClientService {
- static final Log LOG = LogFactory.getLog(MRClientService.class);
+ static final Logger LOG = LoggerFactory.getLogger(MRClientService.class);
private MRClientProtocol protocolHandler;
private Server server;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java
index 0b1be70..849c914 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java
@@ -26,8 +26,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -54,12 +52,14 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class CommitterEventHandler extends AbstractService
implements EventHandler<CommitterEvent> {
- private static final Log LOG =
- LogFactory.getLog(CommitterEventHandler.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CommitterEventHandler.class);
private final AppContext context;
private final OutputCommitter committer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index abc3e61..757c545 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -38,8 +38,6 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
@@ -133,6 +131,8 @@ import org.apache.hadoop.yarn.util.Clock;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** Implementation of Job interface. Maintains the state machines of Job.
* The read and write calls use ReadWriteLock for concurrency.
@@ -147,7 +147,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
private static final TaskCompletionEvent[]
EMPTY_TASK_COMPLETION_EVENTS = new TaskCompletionEvent[0];
- private static final Log LOG = LogFactory.getLog(JobImpl.class);
+ private static final Logger LOG = LoggerFactory.getLogger(JobImpl.class);
//The maximum fraction of fetch failures allowed for a map
private float maxAllowedFetchFailuresFraction;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 3faad48..650f387 100755
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -38,8 +38,6 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Pattern;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -139,6 +137,8 @@ import org.apache.hadoop.yarn.util.RackResolver;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Implementation of TaskAttempt interface.
@@ -149,7 +149,8 @@ public abstract class TaskAttemptImpl implements
EventHandler<TaskAttemptEvent> {
static final Counters EMPTY_COUNTERS = new Counters();
- private static final Log LOG = LogFactory.getLog(TaskAttemptImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TaskAttemptImpl.class);
private static final long MEMORY_SPLITS_RESOLUTION = 1024; //TODO Make configurable?
private final static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
index 228ae24..086d4d5 100755
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
@@ -32,8 +32,6 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Counters;
@@ -92,6 +90,8 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
@@ -101,7 +101,7 @@ import com.google.common.annotations.VisibleForTesting;
@SuppressWarnings({ "rawtypes", "unchecked" })
public abstract class TaskImpl implements Task, EventHandler<TaskEvent> {
- private static final Log LOG = LogFactory.getLog(TaskImpl.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TaskImpl.class);
private static final String SPECULATION = "Speculation: ";
protected final JobConf conf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
index cb4f0c9..288a743 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
@@ -32,8 +32,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -60,6 +58,8 @@ import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.C
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class is responsible for launching of containers.
@@ -67,7 +67,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class ContainerLauncherImpl extends AbstractService implements
ContainerLauncher {
- static final Log LOG = LogFactory.getLog(ContainerLauncherImpl.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(ContainerLauncherImpl.class);
private ConcurrentHashMap<ContainerId, Container> containers =
new ConcurrentHashMap<ContainerId, Container>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
index 269e5f4..01338b4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapreduce.v2.app.local;
import java.io.IOException;
import java.util.ArrayList;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.JobCounter;
@@ -55,6 +53,8 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Allocates containers locally. Doesn't allocate a real container;
@@ -63,8 +63,8 @@ import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
public class LocalContainerAllocator extends RMCommunicator
implements ContainerAllocator {
- private static final Log LOG =
- LogFactory.getLog(LocalContainerAllocator.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LocalContainerAllocator.class);
@SuppressWarnings("rawtypes")
private final EventHandler eventHandler;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
index a7058e0..7bcb2ce 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
@@ -26,8 +26,6 @@ import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -61,13 +59,16 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Registers/unregisters to RM and sends heartbeats to RM.
*/
public abstract class RMCommunicator extends AbstractService
implements RMHeartbeatHandler {
- private static final Log LOG = LogFactory.getLog(RMCommunicator.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RMCommunicator.class);
private int rmPollInterval;//millis
protected ApplicationId applicationId;
private final AtomicBoolean stopped;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index 0dc7642..9d030b8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -35,8 +35,6 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
@@ -90,6 +88,8 @@ import org.apache.hadoop.yarn.util.RackResolver;
import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Allocates the container from the ResourceManager scheduler.
@@ -97,7 +97,7 @@ import com.google.common.annotations.VisibleForTesting;
public class RMContainerAllocator extends RMContainerRequestor
implements ContainerAllocator {
- static final Log LOG = LogFactory.getLog(RMContainerAllocator.class);
+ static final Logger LOG = LoggerFactory.getLogger(RMContainerAllocator.class);
public static final
float DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART = 0.05f;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
index f4579ab..bb3e1fa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
@@ -30,8 +30,6 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -55,6 +53,8 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -62,7 +62,8 @@ import org.apache.hadoop.yarn.util.resource.Resources;
*/
public abstract class RMContainerRequestor extends RMCommunicator {
- private static final Log LOG = LogFactory.getLog(RMContainerRequestor.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RMContainerRequestor.class);
private static final ResourceRequestComparator RESOURCE_REQUEST_COMPARATOR =
new ResourceRequestComparator();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java
index b0e399f..6192c4d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java
@@ -25,8 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapreduce.JobCounter;
import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
@@ -44,6 +42,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.StrictPreemptionContract;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This policy works in combination with an implementation of task
@@ -75,8 +75,8 @@ public class CheckpointAMPreemptionPolicy implements AMPreemptionPolicy {
@SuppressWarnings("rawtypes")
private EventHandler eventHandler;
- static final Log LOG = LogFactory
- .getLog(CheckpointAMPreemptionPolicy.class);
+ static final Logger LOG = LoggerFactory
+ .getLogger(CheckpointAMPreemptionPolicy.class);
public CheckpointAMPreemptionPolicy() {
this(Collections.synchronizedSet(new HashSet<TaskAttemptId>()),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java
index 09237aa..eec6854 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.mapreduce.v2.app.rm.preemption;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapreduce.JobCounter;
import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
@@ -33,14 +31,16 @@ import org.apache.hadoop.yarn.api.records.PreemptionContract;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.StrictPreemptionContract;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Sample policy that aggressively kills tasks when requested.
*/
public class KillAMPreemptionPolicy implements AMPreemptionPolicy {
- private static final Log LOG =
- LogFactory.getLog(KillAMPreemptionPolicy.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(KillAMPreemptionPolicy.class);
@SuppressWarnings("rawtypes")
private EventHandler dispatcher = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
index ed06493..fa65383 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
@@ -30,8 +30,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -53,6 +51,8 @@ import org.apache.hadoop.yarn.util.Clock;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.event.Event;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class DefaultSpeculator extends AbstractService implements
Speculator {
@@ -70,7 +70,8 @@ public class DefaultSpeculator extends AbstractService implements
private double proportionTotalTasksSpeculatable;
private int minimumAllowedSpeculativeTasks;
- private static final Log LOG = LogFactory.getLog(DefaultSpeculator.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DefaultSpeculator.class);
private final ConcurrentMap<TaskId, Boolean> runningTasks
= new ConcurrentHashMap<TaskId, Boolean>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
index 6db1274..1ba37c9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
@@ -26,8 +26,6 @@ import java.net.URLDecoder;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -45,12 +43,15 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.base.Joiner;
import com.google.inject.Inject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class renders the various pages that the web app supports.
*/
public class AppController extends Controller implements AMParams {
- private static final Log LOG = LogFactory.getLog(AppController.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AppController.class);
private static final Joiner JOINER = Joiner.on("");
protected final App app;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestLocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestLocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestLocalContainerLauncher.java
index 232fe6b..16686ab 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestLocalContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestLocalContainerLauncher.java
@@ -31,8 +31,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
@@ -61,10 +59,12 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestLocalContainerLauncher {
- private static final Log LOG =
- LogFactory.getLog(TestLocalContainerLauncher.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestLocalContainerLauncher.class);
private static File testWorkDir;
private static final String[] localDirs = new String[2];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index 47caf44..d951944 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -36,8 +36,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileContext;
@@ -87,12 +85,14 @@ import org.mockito.Mockito;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestJobHistoryEventHandler {
- private static final Log LOG = LogFactory
- .getLog(TestJobHistoryEventHandler.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestJobHistoryEventHandler.class);
private static MiniDFSCluster dfsCluster = null;
private static String coreSitePath;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java
index b6d8bbf..b81f716 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java
@@ -18,19 +18,20 @@
package org.apache.hadoop.mapreduce.jobhistory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class TestJobSummary {
- private static final Log LOG = LogFactory.getLog(TestJobSummary.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestJobSummary.class);
private JobSummary summary = new JobSummary();
@Before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
index 2d0b5ea..a6e57ca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
@@ -24,8 +24,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.EnumSet;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
@@ -101,6 +99,8 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -109,7 +109,7 @@ import org.junit.Assert;
*/
@SuppressWarnings("unchecked")
public class MRApp extends MRAppMaster {
- private static final Log LOG = LogFactory.getLog(MRApp.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MRApp.class);
/**
* The available resource of each container allocated.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
index 322984e..025a8fa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator;
import org.apache.hadoop.mapreduce.v2.app.rm.preemption.AMPreemptionPolicy;
import org.apache.hadoop.mapreduce.v2.app.rm.preemption.NoopAMPreemptionPolicy;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -52,10 +53,8 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.Records;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
import org.junit.Test;
+import org.slf4j.event.Level;
public class MRAppBenchmark {
@@ -67,8 +66,7 @@ public class MRAppBenchmark {
* @throws Exception On application failure
*/
public void run(MRApp app) throws Exception {
- Logger rootLogger = LogManager.getRootLogger();
- rootLogger.setLevel(Level.WARN);
+ GenericTestUtils.setRootLogLevel(Level.WARN);
long startTime = System.currentTimeMillis();
Job job = app.submit(new Configuration());
while (!job.getReport().getJobState().equals(JobState.SUCCEEDED)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
index 203958d..fbce003 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
@@ -41,8 +41,6 @@ import java.util.Map;
import org.junit.Assert;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
@@ -77,6 +75,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -84,18 +83,19 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
+import org.slf4j.event.Level;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestMRAppMaster {
- private static final Log LOG = LogFactory.getLog(TestMRAppMaster.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestMRAppMaster.class);
private static final Path TEST_ROOT_DIR =
new Path(System.getProperty("test.build.data", "target/test-dir"));
private static final Path testDir = new Path(TEST_ROOT_DIR,
@@ -408,8 +408,7 @@ public class TestMRAppMaster {
@Test
public void testMRAppMasterCredentials() throws Exception {
- Logger rootLogger = LogManager.getRootLogger();
- rootLogger.setLevel(Level.DEBUG);
+ GenericTestUtils.setRootLogLevel(Level.DEBUG);
// Simulate credentials passed to AM via client->RM->NM
Credentials credentials = new Credentials();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
index 6332c5d..17e07b1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
@@ -38,8 +38,6 @@ import java.util.Map;
import java.util.concurrent.TimeoutException;
import org.junit.Assert;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
@@ -103,11 +101,13 @@ import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings({"unchecked", "rawtypes"})
public class TestRecovery {
- private static final Log LOG = LogFactory.getLog(TestRecovery.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TestRecovery.class);
private static Path outputDir = new Path(new File("target",
TestRecovery.class.getName()).getAbsolutePath() +
Path.SEPARATOR + "out");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
index e1fa198..ed70a98 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
@@ -29,8 +29,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.TaskCompletionEvent;
@@ -81,6 +79,8 @@ import org.apache.hadoop.yarn.util.ControlledClock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings({"unchecked", "rawtypes"})
public class TestRuntimeEstimators {
@@ -98,7 +98,8 @@ public class TestRuntimeEstimators {
AppContext myAppContext;
- private static final Log LOG = LogFactory.getLog(TestRuntimeEstimators.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRuntimeEstimators.class);
private final AtomicInteger slotsInUse = new AtomicInteger(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
index 57573cc..4a5d480 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
@@ -24,8 +24,6 @@ import java.util.Map;
import org.apache.hadoop.mapreduce.TaskType;
import org.junit.Assert;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.MRConfig;
@@ -40,10 +38,13 @@ import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestMapReduceChildJVM {
- private static final Log LOG = LogFactory.getLog(TestMapReduceChildJVM.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestMapReduceChildJVM.class);
@Test (timeout = 30000)
public void testCommandLine() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
index 4abdff8..62d4cc0 100755
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
@@ -28,8 +28,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Task;
@@ -69,11 +67,13 @@ import org.apache.hadoop.yarn.util.SystemClock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings("rawtypes")
public class TestTaskImpl {
- private static final Log LOG = LogFactory.getLog(TestTaskImpl.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TestTaskImpl.class);
private JobConf conf;
private TaskAttemptListener taskAttemptListener;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
index a93bf88..d5bf03d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
@@ -42,8 +42,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse;
import org.junit.Assert;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.Server;
@@ -93,6 +91,8 @@ import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestContainerLauncher {
@@ -101,7 +101,8 @@ public class TestContainerLauncher {
Configuration conf;
Server server;
- static final Log LOG = LogFactory.getLog(TestContainerLauncher.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(TestContainerLauncher.class);
@Test (timeout = 10000)
public void testPoolSize() throws InterruptedException {
@@ -446,7 +447,7 @@ public class TestContainerLauncher {
// make the thread sleep to look like its not going to respond
Thread.sleep(15000);
} catch (Exception e) {
- LOG.error(e);
+ LOG.error("Setup thread sleep interrupted: ", e);
throw new UndeclaredThrowableException(e);
}
status.setState(ContainerState.RUNNING);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
index 53af631..0ae0380 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
@@ -32,8 +32,6 @@ import java.util.Map;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -82,9 +80,12 @@ import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestContainerLauncherImpl {
- static final Log LOG = LogFactory.getLog(TestContainerLauncherImpl.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(TestContainerLauncherImpl.class);
private static final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index e4a8a1a..85e4181 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -48,8 +48,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -154,12 +152,14 @@ import org.junit.Test;
import com.google.common.base.Supplier;
import org.mockito.InOrder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings("unchecked")
public class TestRMContainerAllocator {
- static final Log LOG = LogFactory
- .getLog(TestRMContainerAllocator.class);
+ static final Logger LOG = LoggerFactory
+ .getLogger(TestRMContainerAllocator.class);
static final RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/35] hadoop git commit: YARN-6623. Add support to turn off
launching privileged containers in the container-executor. (Varun Vasudev via
wangda)
Posted by ae...@apache.org.
YARN-6623. Add support to turn off launching privileged containers in the container-executor. (Varun Vasudev via wangda)
Change-Id: I76aec68cbfb42d239279d7245a03290d691e87a4
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3b1c631
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3b1c631
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3b1c631
Branch: refs/heads/HDFS-7240
Commit: d3b1c6319546706c41a2011ead6c3fe208883200
Parents: c114da5
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Sep 28 16:41:09 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Sep 28 16:41:09 2017 -0700
----------------------------------------------------------------------
.../hadoop-yarn/conf/container-executor.cfg | 12 +
.../src/CMakeLists.txt | 5 +-
.../runtime/DockerLinuxContainerRuntime.java | 11 +-
.../linux/runtime/docker/DockerClient.java | 25 +-
.../linux/runtime/docker/DockerCommand.java | 55 +-
.../runtime/docker/DockerCommandExecutor.java | 3 +-
.../runtime/docker/DockerInspectCommand.java | 13 +-
.../linux/runtime/docker/DockerLoadCommand.java | 2 +-
.../linux/runtime/docker/DockerPullCommand.java | 2 +-
.../linux/runtime/docker/DockerRmCommand.java | 4 +-
.../linux/runtime/docker/DockerRunCommand.java | 68 +-
.../linux/runtime/docker/DockerStopCommand.java | 6 +-
.../container-executor/impl/configuration.c | 17 +
.../container-executor/impl/configuration.h | 19 +-
.../impl/container-executor.c | 316 +----
.../impl/container-executor.h | 9 -
.../container-executor/impl/get_executable.c | 3 -
.../container-executor/impl/get_executable.h | 29 +
.../main/native/container-executor/impl/main.c | 31 +-
.../impl/modules/common/module-configs.c | 3 +-
.../impl/modules/common/module-configs.h | 1 +
.../main/native/container-executor/impl/util.c | 60 +-
.../main/native/container-executor/impl/util.h | 46 +-
.../container-executor/impl/utils/docker-util.c | 998 ++++++++++++++++
.../container-executor/impl/utils/docker-util.h | 147 +++
.../impl/utils/string-utils.c | 1 -
.../docker-container-executor.cfg | 13 +
.../test/test-container-executor.c | 149 +--
.../native/container-executor/test/test_util.cc | 37 +-
.../test/utils/test-string-utils.cc | 37 +-
.../test/utils/test_docker_util.cc | 1122 ++++++++++++++++++
.../runtime/TestDockerContainerRuntime.java | 398 ++++---
.../docker/TestDockerCommandExecutor.java | 35 +-
.../docker/TestDockerInspectCommand.java | 29 +-
.../runtime/docker/TestDockerLoadCommand.java | 9 +-
.../runtime/docker/TestDockerPullCommand.java | 8 +-
.../runtime/docker/TestDockerRmCommand.java | 8 +-
.../runtime/docker/TestDockerRunCommand.java | 25 +-
.../runtime/docker/TestDockerStopCommand.java | 15 +-
.../src/site/markdown/DockerContainers.md | 33 +-
40 files changed, 3067 insertions(+), 737 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
index d68cee8..023654b 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
+++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
@@ -2,3 +2,15 @@ yarn.nodemanager.linux-container-executor.group=#configured value of yarn.nodema
banned.users=#comma separated list of users who can not run applications
min.user.id=1000#Prevent other super-users
allowed.system.users=##comma separated list of system users who CAN run applications
+feature.tc.enabled=0
+
+# The configs below deal with settings for Docker
+#[docker]
+# module.enabled=## enable/disable the module. set to "true" to enable, disabled by default
+# docker.binary=/usr/bin/docker
+# docker.allowed.capabilities=## comma seperated capabilities that can be granted, e.g CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE
+# docker.allowed.devices=## comma seperated list of devices that can be mounted into a container
+# docker.allowed.networks=## comma seperated networks that can be used. e.g bridge,host,none
+# docker.allowed.ro-mounts=## comma seperated volumes that can be mounted as read-only
+# docker.allowed.rw-mounts=## comma seperate volumes that can be mounted as read-write, add the yarn local and log dirs to this list to run Hadoop jobs
+# docker.privileged-containers.enabled=0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index 3d5b506..9d83bf3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -132,6 +132,7 @@ add_library(container
main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
main/native/container-executor/impl/modules/common/module-configs.c
main/native/container-executor/impl/modules/gpu/gpu-module.c
+ main/native/container-executor/impl/utils/docker-util.c
)
add_executable(container-executor
@@ -156,6 +157,7 @@ output_directory(test-container-executor target/usr/local/bin)
# unit tests for container executor
add_executable(cetest
+ main/native/container-executor/impl/get_executable.c
main/native/container-executor/impl/util.c
main/native/container-executor/test/test_configuration.cc
main/native/container-executor/test/test_main.cc
@@ -163,6 +165,7 @@ add_executable(cetest
main/native/container-executor/test/utils/test-path-utils.cc
main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
main/native/container-executor/test/modules/gpu/test-gpu-module.cc
- main/native/container-executor/test/test_util.cc)
+ main/native/container-executor/test/test_util.cc
+ main/native/container-executor/test/utils/test_docker_util.cc)
target_link_libraries(cetest gtest container)
output_directory(cetest test)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index b8d9b0a..29c6fe9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -559,8 +559,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
runCommand.setCapabilities(capabilities);
if(cgroupsRootDirectory != null) {
- runCommand.addMountLocation(cgroupsRootDirectory,
- cgroupsRootDirectory + ":ro", false);
+ runCommand.addReadOnlyMountLocation(cgroupsRootDirectory,
+ cgroupsRootDirectory, false);
}
List<String> allDirs = new ArrayList<>(containerLocalDirs);
@@ -584,7 +584,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
String src = validateMount(dir[0], localizedResources);
String dst = dir[1];
- runCommand.addMountLocation(src, dst + ":ro", true);
+ runCommand.addReadOnlyMountLocation(src, dst, true);
}
}
}
@@ -626,7 +626,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
launchOp, null, null, false, false);
} catch (PrivilegedOperationException e) {
LOG.warn("Launch container failed. Exception: ", e);
- LOG.info("Docker command used: " + runCommand.getCommandWithArguments());
+ LOG.info("Docker command used: " + runCommand);
throw new ContainerExecutionException("Launch container failed", e
.getExitCode(), e.getOutput(), e.getErrorOutput());
@@ -757,8 +757,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
launchOp.appendArgs(tcCommandFile);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Launching container with cmd: " + runCommand
- .getCommandWithArguments());
+ LOG.debug("Launching container with cmd: " + runCommand);
}
return launchOp;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
index 536a22d..77c53a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
@@ -23,7 +23,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,6 +34,8 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
+import java.util.List;
+import java.util.Map;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -68,10 +70,25 @@ public final class DockerClient {
TMP_FILE_SUFFIX, new
File(tmpDirPath));
- Writer writer = new OutputStreamWriter(new FileOutputStream(dockerCommandFile),
- "UTF-8");
+ Writer writer = new OutputStreamWriter(
+ new FileOutputStream(dockerCommandFile), "UTF-8");
PrintWriter printWriter = new PrintWriter(writer);
- printWriter.print(cmd.getCommandWithArguments());
+ printWriter.println("[docker-command-execution]");
+ for (Map.Entry<String, List<String>> entry :
+ cmd.getDockerCommandWithArguments().entrySet()) {
+ if (entry.getKey().contains("=")) {
+ throw new ContainerExecutionException(
+ "'=' found in entry for docker command file, key = " + entry
+ .getKey() + "; value = " + entry.getValue());
+ }
+ if (entry.getValue().contains("\n")) {
+ throw new ContainerExecutionException(
+ "'\\n' found in entry for docker command file, key = " + entry
+ .getKey() + "; value = " + entry.getValue());
+ }
+ printWriter.println(" " + entry.getKey() + "=" + StringUtils
+ .join(",", entry.getValue()));
+ }
printWriter.close();
return dockerCommandFile.getAbsolutePath();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
index 3b76a5c..7802209 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
@@ -25,8 +25,10 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.StringUtils;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -35,32 +37,55 @@ import java.util.List;
* e.g 'run', 'load', 'inspect' etc.,
*/
-public abstract class DockerCommand {
+public abstract class DockerCommand {
private final String command;
- private final List<String> commandWithArguments;
+ private final Map<String, List<String>> commandArguments;
protected DockerCommand(String command) {
+ String dockerCommandKey = "docker-command";
this.command = command;
- this.commandWithArguments = new ArrayList<>();
- commandWithArguments.add(command);
+ this.commandArguments = new TreeMap<>();
+ commandArguments.put(dockerCommandKey, new ArrayList<>());
+ commandArguments.get(dockerCommandKey).add(command);
}
- /** Returns the docker sub-command string being used
- * e.g 'run'
+ /**
+ * Returns the docker sub-command string being used
+ * e.g 'run'.
*/
public final String getCommandOption() {
return this.command;
}
- /** Add command commandWithArguments - this method is only meant for use by
- * sub-classes
- * @param arguments to be added
+ /**
+ * Add command commandWithArguments - this method is only meant for use by
+ * sub-classes.
+ *
+ * @param key name of the key to be added
+ * @param value value of the key
*/
- protected final void addCommandArguments(String... arguments) {
- this.commandWithArguments.addAll(Arrays.asList(arguments));
+ protected final void addCommandArguments(String key, String value) {
+ List<String> list = commandArguments.get(key);
+ if (list != null) {
+ list.add(value);
+ return;
+ }
+ list = new ArrayList<>();
+ list.add(value);
+ this.commandArguments.put(key, list);
}
- public String getCommandWithArguments() {
- return StringUtils.join(" ", commandWithArguments);
+ public Map<String, List<String>> getDockerCommandWithArguments() {
+ return Collections.unmodifiableMap(commandArguments);
}
-}
\ No newline at end of file
+
+ @Override
+ public String toString() {
+ StringBuffer ret = new StringBuffer(this.command);
+ for (Map.Entry<String, List<String>> entry : commandArguments.entrySet()) {
+ ret.append(" ").append(entry.getKey());
+ ret.append("=").append(StringUtils.join(",", entry.getValue()));
+ }
+ return ret.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
index 5739912..76b53af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
@@ -88,8 +88,7 @@ public final class DockerCommandExecutor {
dockerOp.disableFailureLogging();
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Running docker command: "
- + dockerCommand.getCommandWithArguments());
+ LOG.debug("Running docker command: " + dockerCommand);
}
try {
String result = privilegedOperationExecutor
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
index 812a35f..d27f74d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
@@ -26,16 +26,14 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
*/
public class DockerInspectCommand extends DockerCommand {
private static final String INSPECT_COMMAND = "inspect";
- private String containerName;
public DockerInspectCommand(String containerName) {
super(INSPECT_COMMAND);
- this.containerName = containerName;
+ super.addCommandArguments("name", containerName);
}
public DockerInspectCommand getContainerStatus() {
- super.addCommandArguments("--format='{{.State.Status}}'");
- super.addCommandArguments(containerName);
+ super.addCommandArguments("format", "{{.State.Status}}");
return this;
}
@@ -43,9 +41,8 @@ public class DockerInspectCommand extends DockerCommand {
// Be sure to not use space in the argument, otherwise the
// extract_values_delim method in container-executor binary
// cannot parse the arguments correctly.
- super.addCommandArguments("--format='{{range(.NetworkSettings.Networks)}}"
- + "{{.IPAddress}},{{end}}{{.Config.Hostname}}'");
- super.addCommandArguments(containerName);
+ super.addCommandArguments("format", "{{range(.NetworkSettings.Networks)}}"
+ + "{{.IPAddress}},{{end}}{{.Config.Hostname}}");
return this;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
index e4d92e0..fa2988c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
@@ -25,6 +25,6 @@ public class DockerLoadCommand extends DockerCommand {
public DockerLoadCommand(String localImageFile) {
super(LOAD_COMMAND);
- super.addCommandArguments("--i=" + localImageFile);
+ super.addCommandArguments("image", localImageFile);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
index 351e09e..5e6108e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
@@ -25,7 +25,7 @@ public class DockerPullCommand extends DockerCommand {
public DockerPullCommand(String imageName) {
super(PULL_COMMAND);
- super.addCommandArguments(imageName);
+ super.addCommandArguments("image", imageName);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
index b1aea61..dcfe777 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
@@ -25,6 +25,6 @@ public class DockerRmCommand extends DockerCommand {
public DockerRmCommand(String containerName) {
super(RM_COMMAND);
- super.addCommandArguments(containerName);
+ super.addCommandArguments("name", containerName);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index 1e1e6e8..c7bf827 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -20,42 +20,39 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
-import org.apache.hadoop.util.StringUtils;
-
import java.io.File;
-import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
public class DockerRunCommand extends DockerCommand {
private static final String RUN_COMMAND = "run";
- private final String image;
- private List<String> overrrideCommandWithArgs;
/** The following are mandatory: */
public DockerRunCommand(String containerId, String user, String image) {
super(RUN_COMMAND);
- super.addCommandArguments("--name=" + containerId, "--user=" + user);
- this.image = image;
+ super.addCommandArguments("name", containerId);
+ super.addCommandArguments("user", user);
+ super.addCommandArguments("image", image);
}
public DockerRunCommand removeContainerOnExit() {
- super.addCommandArguments("--rm");
+ super.addCommandArguments("rm", "true");
return this;
}
public DockerRunCommand detachOnRun() {
- super.addCommandArguments("-d");
+ super.addCommandArguments("detach", "true");
return this;
}
public DockerRunCommand setContainerWorkDir(String workdir) {
- super.addCommandArguments("--workdir=" + workdir);
+ super.addCommandArguments("workdir", workdir);
return this;
}
public DockerRunCommand setNetworkType(String type) {
- super.addCommandArguments("--net=" + type);
+ super.addCommandArguments("net", type);
return this;
}
@@ -65,79 +62,80 @@ public class DockerRunCommand extends DockerCommand {
if (!sourceExists && !createSource) {
return this;
}
- super.addCommandArguments("-v", sourcePath + ":" + destinationPath);
+ super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath);
+ return this;
+ }
+
+ public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
+ destinationPath, boolean createSource) {
+ boolean sourceExists = new File(sourcePath).exists();
+ if (!sourceExists && !createSource) {
+ return this;
+ }
+ super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath);
return this;
}
public DockerRunCommand setCGroupParent(String parentPath) {
- super.addCommandArguments("--cgroup-parent=" + parentPath);
+ super.addCommandArguments("cgroup-parent", parentPath);
return this;
}
/* Run a privileged container. Use with extreme care */
public DockerRunCommand setPrivileged() {
- super.addCommandArguments("--privileged");
+ super.addCommandArguments("privileged", "true");
return this;
}
public DockerRunCommand setCapabilities(Set<String> capabilties) {
//first, drop all capabilities
- super.addCommandArguments("--cap-drop=ALL");
+ super.addCommandArguments("cap-drop", "ALL");
//now, add the capabilities supplied
for (String capability : capabilties) {
- super.addCommandArguments("--cap-add=" + capability);
+ super.addCommandArguments("cap-add", capability);
}
return this;
}
public DockerRunCommand setHostname(String hostname) {
- super.addCommandArguments("--hostname=" + hostname);
+ super.addCommandArguments("hostname", hostname);
return this;
}
public DockerRunCommand addDevice(String sourceDevice, String
destinationDevice) {
- super.addCommandArguments("--device=" + sourceDevice + ":" +
+ super.addCommandArguments("devices", sourceDevice + ":" +
destinationDevice);
return this;
}
public DockerRunCommand enableDetach() {
- super.addCommandArguments("--detach=true");
+ super.addCommandArguments("detach", "true");
return this;
}
public DockerRunCommand disableDetach() {
- super.addCommandArguments("--detach=false");
+ super.addCommandArguments("detach", "false");
return this;
}
public DockerRunCommand groupAdd(String[] groups) {
- for(int i = 0; i < groups.length; i++) {
- super.addCommandArguments("--group-add " + groups[i]);
- }
+ super.addCommandArguments("group-add", String.join(",", groups));
return this;
}
public DockerRunCommand setOverrideCommandWithArgs(
List<String> overrideCommandWithArgs) {
- this.overrrideCommandWithArgs = overrideCommandWithArgs;
+ for(String override: overrideCommandWithArgs) {
+ super.addCommandArguments("launch-command", override);
+ }
return this;
}
@Override
- public String getCommandWithArguments() {
- List<String> argList = new ArrayList<>();
-
- argList.add(super.getCommandWithArguments());
- argList.add(image);
-
- if (overrrideCommandWithArgs != null) {
- argList.addAll(overrrideCommandWithArgs);
- }
-
- return StringUtils.join(" ", argList);
+ public Map<String, List<String>> getDockerCommandWithArguments() {
+ return super.getDockerCommandWithArguments();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
index e9d6c43..ccfff82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
@@ -29,11 +29,11 @@ public class DockerStopCommand extends DockerCommand {
public DockerStopCommand(String containerName) {
super(STOP_COMMAND);
- super.addCommandArguments(containerName);
+ super.addCommandArguments("name", containerName);
}
public DockerStopCommand setGracePeriod(int value) {
- super.addCommandArguments("--time=" + Integer.toString(value));
+ super.addCommandArguments("time", Integer.toString(value));
return this;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index 12dbc4c..f23cff0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -21,6 +21,7 @@
#include "configuration.h"
#include "util.h"
+#include "get_executable.h"
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
@@ -696,3 +697,19 @@ int get_kv_value(const char *input, char *out, size_t out_len) {
return 0;
}
+
+char *get_config_path(const char *argv0) {
+ char *executable_file = get_executable((char *) argv0);
+ if (!executable_file) {
+ fprintf(ERRORFILE, "realpath of executable: %s\n",
+ errno != 0 ? strerror(errno) : "unknown");
+ return NULL;
+ }
+
+ const char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
+ char *conf_file = resolve_config_path(orig_conf_file, executable_file);
+ if (conf_file == NULL) {
+ fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
+ }
+ return conf_file;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
index 1ea5561..7fa684e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
@@ -23,10 +23,21 @@
#define _WITH_GETLINE
#endif
-#include <stddef.h>
+#include "config.h"
+
+#define CONF_FILENAME "container-executor.cfg"
+
+// When building as part of a Maven build this value gets defined by using
+// container-executor.conf.dir property. See:
+// hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+// for details.
+// NOTE: if this ends up being a relative path it gets resolved relative to
+// the location of the container-executor binary itself, not getwd(3)
+#ifndef HADOOP_CONF_DIR
+#error HADOOP_CONF_DIR must be defined
+#endif
-/** Define a platform-independent constant instead of using PATH_MAX */
-#define EXECUTOR_PATH_MAX 4096
+#include <stddef.h>
// Configuration data structures.
struct kv_pair {
@@ -207,4 +218,6 @@ int get_kv_key(const char *input, char *out, size_t out_len);
*/
int get_kv_value(const char *input, char *out, size_t out_len);
+char *get_config_path(const char* argv0);
+
#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index e8bf564..08d69a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -18,7 +18,7 @@
#include "configuration.h"
#include "container-executor.h"
-#include "utils/string-utils.h"
+#include "utils/docker-util.h"
#include "util.h"
#include "config.h"
@@ -43,7 +43,6 @@
#include <sys/mount.h>
#include <sys/wait.h>
#include <getopt.h>
-#include <regex.h>
#ifndef HAVE_FCHMODAT
#include "compat/fchmodat.h"
@@ -81,11 +80,6 @@ static const char* TC_READ_STATS_OPTS [] = { "-s", "-b", NULL};
//struct to store the user details
struct passwd *user_detail = NULL;
-//Docker container related constants.
-static const char* DOCKER_CONTAINER_NAME_PREFIX = "container_";
-static const char* DOCKER_CLIENT_CONFIG_ARG = "--config=";
-static const char* DOCKER_PULL_COMMAND = "pull";
-
FILE* LOGFILE = NULL;
FILE* ERRORFILE = NULL;
@@ -465,8 +459,9 @@ int is_feature_enabled(const char* feature_key, int default_value,
}
int is_docker_support_enabled() {
- return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY,
- DEFAULT_DOCKER_SUPPORT_ENABLED, &executor_cfg);
+ return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY,
+ DEFAULT_DOCKER_SUPPORT_ENABLED, &executor_cfg)
+ || docker_module_enabled(&CFG);
}
int is_tc_support_enabled() {
@@ -474,13 +469,6 @@ int is_tc_support_enabled() {
DEFAULT_TC_SUPPORT_ENABLED, &executor_cfg);
}
-char* check_docker_binary(char *docker_binary) {
- if (docker_binary == NULL) {
- return "docker";
- }
- return docker_binary;
-}
-
/**
* Utility function to concatenate argB to argA using the concat_pattern.
*/
@@ -1157,273 +1145,28 @@ int initialize_app(const char *user, const char *app_id,
return -1;
}
-static char* escape_single_quote(const char *str) {
- int p = 0;
- int i = 0;
- char replacement[] = "'\"'\"'";
- size_t replacement_length = strlen(replacement);
- size_t ret_size = strlen(str) * replacement_length + 1;
- char *ret = (char *) calloc(ret_size, sizeof(char));
- if(ret == NULL) {
- exit(OUT_OF_MEMORY);
- }
- while(str[p] != '\0') {
- if(str[p] == '\'') {
- strncat(ret, replacement, ret_size - strlen(ret));
- i += replacement_length;
- }
- else {
- ret[i] = str[p];
- ret[i + 1] = '\0';
- i++;
- }
- p++;
- }
- return ret;
-}
-
-static void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) {
- char *tmp = escape_single_quote(arg);
- strcat(*str, param);
- strcat(*str, "'");
- if(strlen(*str) + strlen(tmp) > *size) {
- *str = (char *) realloc(*str, strlen(*str) + strlen(tmp) + 1024);
- if(*str == NULL) {
- exit(OUT_OF_MEMORY);
- }
- *size = strlen(*str) + strlen(tmp) + 1024;
- }
- strcat(*str, tmp);
- strcat(*str, "' ");
- free(tmp);
-}
-
-char** tokenize_docker_command(const char *input, int *split_counter) {
- char *line = (char *)calloc(strlen(input) + 1, sizeof(char));
- char **linesplit = (char **) malloc(sizeof(char *));
- char *p = NULL;
- *split_counter = 0;
- strncpy(line, input, strlen(input));
-
- p = strtok(line, " ");
- while(p != NULL) {
- linesplit[*split_counter] = p;
- (*split_counter)++;
- linesplit = realloc(linesplit, (sizeof(char *) * (*split_counter + 1)));
- if(linesplit == NULL) {
- fprintf(ERRORFILE, "Cannot allocate memory to parse docker command %s",
- strerror(errno));
- fflush(ERRORFILE);
- exit(OUT_OF_MEMORY);
- }
- p = strtok(NULL, " ");
- }
- linesplit[*split_counter] = NULL;
- return linesplit;
-}
-
-int execute_regex_match(const char *regex_str, const char *input) {
- regex_t regex;
- int regex_match;
- if (0 != regcomp(®ex, regex_str, REG_EXTENDED|REG_NOSUB)) {
- fprintf(LOGFILE, "Unable to compile regex.");
- fflush(LOGFILE);
- exit(ERROR_COMPILING_REGEX);
- }
- regex_match = regexec(®ex, input, (size_t) 0, NULL, 0);
- regfree(®ex);
- if(0 == regex_match) {
- return 0;
- }
- return 1;
-}
-
-int validate_docker_image_name(const char *image_name) {
- char *regex_str = "^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$";
- return execute_regex_match(regex_str, image_name);
-}
-
-char* sanitize_docker_command(const char *line) {
- static struct option long_options[] = {
- {"name", required_argument, 0, 'n' },
- {"user", required_argument, 0, 'u' },
- {"rm", no_argument, 0, 'r' },
- {"workdir", required_argument, 0, 'w' },
- {"net", required_argument, 0, 'e' },
- {"hostname", required_argument, 0, 'h' },
- {"cgroup-parent", required_argument, 0, 'g' },
- {"privileged", no_argument, 0, 'p' },
- {"cap-add", required_argument, 0, 'a' },
- {"cap-drop", required_argument, 0, 'o' },
- {"device", required_argument, 0, 'i' },
- {"detach", required_argument, 0, 't' },
- {"format", required_argument, 0, 'f' },
- {"group-add", required_argument, 0, 'x' },
- {0, 0, 0, 0}
- };
-
- int c = 0;
- int option_index = 0;
- char *output = NULL;
- size_t output_size = 0;
- char **linesplit;
- int split_counter = 0;
- int len = strlen(line);
-
- linesplit = tokenize_docker_command(line, &split_counter);
-
- output_size = len * 2;
- output = (char *) calloc(output_size, sizeof(char));
- if(output == NULL) {
- exit(OUT_OF_MEMORY);
- }
-
- // Handle docker client config option.
- if(0 == strncmp(linesplit[0], DOCKER_CLIENT_CONFIG_ARG, strlen(DOCKER_CLIENT_CONFIG_ARG))) {
- strcat(output, linesplit[0]);
- strcat(output, " ");
- long index = 0;
- while(index < split_counter) {
- linesplit[index] = linesplit[index + 1];
- if (linesplit[index] == NULL) {
- split_counter--;
- break;
- }
- index++;
- }
- }
-
- // Handle docker pull and image name validation.
- if (0 == strncmp(linesplit[0], DOCKER_PULL_COMMAND, strlen(DOCKER_PULL_COMMAND))) {
- if (0 != validate_docker_image_name(linesplit[1])) {
- fprintf(ERRORFILE, "Invalid Docker image name, exiting.");
- fflush(ERRORFILE);
- exit(DOCKER_IMAGE_INVALID);
- }
- strcat(output, linesplit[0]);
- strcat(output, " ");
- strcat(output, linesplit[1]);
- return output;
- }
-
- strcat(output, linesplit[0]);
- strcat(output, " ");
- optind = 1;
- while((c=getopt_long(split_counter, linesplit, "dv:", long_options, &option_index)) != -1) {
- switch(c) {
- case 'n':
- quote_and_append_arg(&output, &output_size, "--name=", optarg);
- break;
- case 'w':
- quote_and_append_arg(&output, &output_size, "--workdir=", optarg);
- break;
- case 'u':
- quote_and_append_arg(&output, &output_size, "--user=", optarg);
- break;
- case 'e':
- quote_and_append_arg(&output, &output_size, "--net=", optarg);
- break;
- case 'h':
- quote_and_append_arg(&output, &output_size, "--hostname=", optarg);
- break;
- case 'v':
- quote_and_append_arg(&output, &output_size, "-v ", optarg);
- break;
- case 'a':
- quote_and_append_arg(&output, &output_size, "--cap-add=", optarg);
- break;
- case 'o':
- quote_and_append_arg(&output, &output_size, "--cap-drop=", optarg);
- break;
- case 'd':
- strcat(output, "-d ");
- break;
- case 'r':
- strcat(output, "--rm ");
- break;
- case 'g':
- quote_and_append_arg(&output, &output_size, "--cgroup-parent=", optarg);
- break;
- case 'p':
- strcat(output, "--privileged ");
- break;
- case 'i':
- quote_and_append_arg(&output, &output_size, "--device=", optarg);
- break;
- case 't':
- quote_and_append_arg(&output, &output_size, "--detach=", optarg);
- break;
- case 'f':
- strcat(output, "--format=");
- strcat(output, optarg);
- strcat(output, " ");
- break;
- case 'x':
- quote_and_append_arg(&output, &output_size, "--group-add ", optarg);
- break;
- default:
- fprintf(LOGFILE, "Unknown option in docker command, character %d %c, optionindex = %d\n", c, c, optind);
- fflush(LOGFILE);
- return NULL;
- break;
- }
- }
-
- if(optind < split_counter) {
- while(optind < split_counter) {
- if (0 == strncmp(linesplit[optind], DOCKER_CONTAINER_NAME_PREFIX, strlen(DOCKER_CONTAINER_NAME_PREFIX))) {
- if (1 != validate_container_id(linesplit[optind])) {
- fprintf(ERRORFILE, "Specified container_id=%s is invalid\n", linesplit[optind]);
- fflush(ERRORFILE);
- exit(DOCKER_CONTAINER_NAME_INVALID);
- }
- strcat(output, linesplit[optind++]);
- } else {
- quote_and_append_arg(&output, &output_size, "", linesplit[optind++]);
- }
- }
- }
-
- return output;
-}
-
-char* parse_docker_command_file(const char* command_file) {
- size_t len = 0;
- char *line = NULL;
- ssize_t read;
- FILE *stream;
- stream = fopen(command_file, "r");
- if (stream == NULL) {
- fprintf(ERRORFILE, "Cannot open file %s - %s",
- command_file, strerror(errno));
- fflush(ERRORFILE);
- exit(ERROR_OPENING_DOCKER_FILE);
- }
- if ((read = getline(&line, &len, stream)) == -1) {
- fprintf(ERRORFILE, "Error reading command_file %s\n", command_file);
- fflush(ERRORFILE);
- exit(ERROR_READING_DOCKER_FILE);
- }
- fclose(stream);
-
- char* ret = sanitize_docker_command(line);
- if(ret == NULL) {
- exit(ERROR_SANITIZING_DOCKER_COMMAND);
+char *construct_docker_command(const char *command_file) {
+ int ret = 0;
+ size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
+ char *buffer = alloc_and_clear_memory(command_size, sizeof(char));
+ ret = get_docker_command(command_file, &CFG, buffer, command_size);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Error constructing docker command, docker error code=%d, error message='%s'\n", ret,
+ get_docker_error_message(ret));
+ fflush(ERRORFILE);
+ exit(DOCKER_RUN_FAILED);
}
- fprintf(ERRORFILE, "Using command %s\n", ret);
- fflush(ERRORFILE);
-
- return ret;
+ return buffer;
}
int run_docker(const char *command_file) {
- char* docker_command = parse_docker_command_file(command_file);
- char* docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
- docker_binary = check_docker_binary(docker_binary);
+ char* docker_command = construct_docker_command(command_file);
+ char* docker_binary = get_docker_binary(&CFG);
size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
- char* docker_command_with_binary = calloc(sizeof(char), command_size);
+ char* docker_command_with_binary = alloc_and_clear_memory(command_size, sizeof(char));
snprintf(docker_command_with_binary, command_size, "%s %s", docker_binary, docker_command);
+ fprintf(LOGFILE, "Invoking '%s'\n", docker_command_with_binary);
char **args = split_delimiter(docker_command_with_binary, " ");
int exit_code = -1;
@@ -1437,8 +1180,9 @@ int run_docker(const char *command_file) {
free(docker_command_with_binary);
free(docker_command);
exit_code = DOCKER_RUN_FAILED;
+ } else {
+ exit_code = 0;
}
- exit_code = 0;
return exit_code;
}
@@ -1583,18 +1327,17 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
- docker_command_with_binary = calloc(sizeof(char), command_size);
- docker_wait_command = calloc(sizeof(char), command_size);
- docker_logs_command = calloc(sizeof(char), command_size);
- docker_inspect_command = calloc(sizeof(char), command_size);
- docker_rm_command = calloc(sizeof(char), command_size);
+ docker_command_with_binary = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_wait_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_logs_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_inspect_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_rm_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
gid_t user_gid = getegid();
uid_t prev_uid = geteuid();
- char *docker_command = parse_docker_command_file(command_file);
- char *docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
- docker_binary = check_docker_binary(docker_binary);
+ char *docker_command = NULL;
+ char *docker_binary = NULL;
fprintf(LOGFILE, "Creating script paths...\n");
exit_code = create_script_paths(
@@ -1617,6 +1360,9 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
goto cleanup;
}
+ docker_command = construct_docker_command(command_file);
+ docker_binary = get_docker_binary(&CFG);
+
fprintf(LOGFILE, "Getting exit code file...\n");
exit_code_file = get_exit_code_file(pid_file);
if (NULL == exit_code_file) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index aa38abf..6d4220f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -59,16 +59,12 @@ enum operations {
#define MIN_USERID_KEY "min.user.id"
#define BANNED_USERS_KEY "banned.users"
#define ALLOWED_SYSTEM_USERS_KEY "allowed.system.users"
-#define DOCKER_BINARY_KEY "docker.binary"
#define DOCKER_SUPPORT_ENABLED_KEY "feature.docker.enabled"
#define TC_SUPPORT_ENABLED_KEY "feature.tc.enabled"
#define TMP_DIR "tmp"
extern struct passwd *user_detail;
-// get the executable's filename
-char* get_executable(char *argv0);
-
//function used to load the configurations present in the secure config
void read_executor_config(const char* file_name);
@@ -258,11 +254,6 @@ int is_docker_support_enabled();
*/
int run_docker(const char *command_file);
-/**
- * Sanitize docker commands. Returns NULL if there was any failure.
-*/
-char* sanitize_docker_command(const char *line);
-
/*
* Compile the regex_str and determine if the input string matches.
* Return 0 on match, 1 of non-match.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
index 55973a2..e1ec293 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
@@ -29,12 +29,9 @@
*/
#include "config.h"
-#include "configuration.h"
-#include "container-executor.h"
#include "util.h"
#include <errno.h>
-#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h
new file mode 100644
index 0000000..ee6b375
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_GET_EXECUTABLE_H__
+#define __YARN_POSIX_CONTAINER_EXECUTOR_GET_EXECUTABLE_H__
+
+/**
+ * Get the path to executable that is currently running
+ * @param argv0 the name of the executable
+ * @return the path to the currently running executable
+ */
+char* get_executable(char *argv0);
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index a05dc78..7b8f63f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -20,30 +20,15 @@
#include "configuration.h"
#include "container-executor.h"
#include "util.h"
+#include "get_executable.h"
#include "modules/gpu/gpu-module.h"
#include "modules/cgroups/cgroups-operations.h"
#include <errno.h>
#include <grp.h>
-#include <limits.h>
#include <unistd.h>
-#include <signal.h>
-#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <sys/stat.h>
-
-#define CONF_FILENAME "container-executor.cfg"
-
-// When building as part of a Maven build this value gets defined by using
-// container-executor.conf.dir property. See:
-// hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
-// for details.
-// NOTE: if this ends up being a relative path it gets resolved relative to
-// the location of the container-executor binary itself, not getwd(3)
-#ifndef HADOOP_CONF_DIR
- #error HADOOP_CONF_DIR must be defined
-#endif
static void display_usage(FILE *stream) {
fprintf(stream,
@@ -145,25 +130,21 @@ of whether an explicit checksetup operation is requested. */
static void assert_valid_setup(char *argv0) {
int ret;
char *executable_file = get_executable(argv0);
- if (!executable_file) {
- fprintf(ERRORFILE,"realpath of executable: %s\n",
- errno != 0 ? strerror(errno) : "unknown");
+ if (!executable_file || executable_file[0] == 0) {
+ fprintf(ERRORFILE, "realpath of executable: %s\n",
+ errno != 0 ? strerror(errno) : "unknown");
flush_and_close_log_files();
- exit(-1);
+ exit(INVALID_CONFIG_FILE);
}
- char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
- char *conf_file = resolve_config_path(orig_conf_file, executable_file);
+ char *conf_file = get_config_path(argv0);
if (conf_file == NULL) {
- free(executable_file);
- fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
if (check_configuration_permissions(conf_file) != 0) {
- free(executable_file);
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
index f0c6d16..4c76e51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
@@ -16,9 +16,8 @@
* limitations under the License.
*/
+#include "module-configs.h"
#include "util.h"
-#include "configuration.h"
-#include "container-executor.h"
#include "modules/common/constants.h"
#include <string.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
index d58c618..b8c57df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
@@ -23,6 +23,7 @@
#ifndef _MODULES_COMMON_MODULE_CONFIGS_H_
#define _MODULES_COMMON_MODULE_CONFIGS_H_
+#include "configuration.h"
/**
* check if module enabled given name of module.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
index 8e39ca8..a9539cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
@@ -17,10 +17,10 @@
*/
#include "util.h"
-#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
+#include <regex.h>
char** split_delimiter(char *value, const char *delim) {
char **return_values = NULL;
@@ -132,3 +132,61 @@ char* trim(const char* input) {
ret[val_end - val_begin] = '\0';
return ret;
}
+
+int execute_regex_match(const char *regex_str, const char *input) {
+ regex_t regex;
+ int regex_match;
+ if (0 != regcomp(®ex, regex_str, REG_EXTENDED|REG_NOSUB)) {
+ fprintf(LOGFILE, "Unable to compile regex.");
+ fflush(LOGFILE);
+ exit(ERROR_COMPILING_REGEX);
+ }
+ regex_match = regexec(®ex, input, (size_t) 0, NULL, 0);
+ regfree(®ex);
+ if(0 == regex_match) {
+ return 0;
+ }
+ return 1;
+}
+
+char* escape_single_quote(const char *str) {
+ int p = 0;
+ int i = 0;
+ char replacement[] = "'\"'\"'";
+ size_t replacement_length = strlen(replacement);
+ size_t ret_size = strlen(str) * replacement_length + 1;
+ char *ret = (char *) alloc_and_clear_memory(ret_size, sizeof(char));
+ if(ret == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ while(str[p] != '\0') {
+ if(str[p] == '\'') {
+ strncat(ret, replacement, ret_size - strlen(ret));
+ i += replacement_length;
+ }
+ else {
+ ret[i] = str[p];
+ i++;
+ }
+ p++;
+ }
+ ret[i] = '\0';
+ return ret;
+}
+
+void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) {
+ char *tmp = escape_single_quote(arg);
+ int alloc_block = 1024;
+ strcat(*str, param);
+ strcat(*str, "'");
+ if (strlen(*str) + strlen(tmp) > *size) {
+ *size = (strlen(*str) + strlen(tmp) + alloc_block) * sizeof(char);
+ *str = (char *) realloc(*str, *size);
+ if (*str == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ }
+ strcat(*str, tmp);
+ strcat(*str, "' ");
+ free(tmp);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
index fa21def..8758d90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
@@ -19,7 +19,11 @@
#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__
#define __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__
+/** Define a platform-independent constant instead of using PATH_MAX, set to 4K */
+#define EXECUTOR_PATH_MAX 4096
+
#include <stdio.h>
+#include <stdlib.h>
enum errorcodes {
INVALID_ARGUMENT_NUMBER = 1,
@@ -62,7 +66,7 @@ enum errorcodes {
ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38,
ERROR_SANITIZING_DOCKER_COMMAND = 39,
DOCKER_IMAGE_INVALID = 40,
- DOCKER_CONTAINER_NAME_INVALID = 41,
+ // DOCKER_CONTAINER_NAME_INVALID = 41, (NOT USED)
ERROR_COMPILING_REGEX = 42
};
@@ -119,4 +123,44 @@ void free_values(char **values);
*/
char* trim(const char *input);
+/**
+ * Run a regex to check if the provided input matches against it
+ * @param regex_str Regex to run
+ * @param input String to run the regex against
+ * @return 0 on match, non-zero on no match
+ */
+int execute_regex_match(const char *regex_str, const char *input);
+
+/**
+ * Helper function to escape single-quotes in a string. The assumption is that the string passed will be enclosed in
+ * single quotes and passed to bash for a command invocation.
+ * @param str The string in which to esacpe single quotes
+ * @return String with single quotes escaped, must be freed by the user.
+ */
+char* escape_single_quote(const char *str);
+
+/**
+ * Helper function to quote the argument to a parameter and then append it to the provided string.
+ * @param str Buffer to which the param='arg' string must be appended
+ * @param size Size of the buffer
+ * @param param Param name
+ * @param arg Argument to be quoted
+ */
+void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg);
+
+/**
+ * Helper function to allocate and clear a block of memory. It'll call exit if the allocation fails.
+ * @param num Num of elements to be allocated
+ * @param size Size of each element
+ * @return Pointer to the allocated memory, must be freed by the user
+ */
+inline void* alloc_and_clear_memory(size_t num, size_t size) {
+ void *ret = calloc(num, size);
+ if (ret == NULL) {
+ printf("Could not allocate memory, exiting\n");
+ exit(OUT_OF_MEMORY);
+ }
+ return ret;
+}
+
#endif
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/35] hadoop git commit: HADOOP-14909. Fix the word of erasure
encoding in the top page. Contributed by Takanobu Asanuma.
Posted by ae...@apache.org.
HADOOP-14909. Fix the word of erasure encoding in the top page. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/373d0a51
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/373d0a51
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/373d0a51
Branch: refs/heads/HDFS-7240
Commit: 373d0a51955cabff77e934a28ba2de308207374a
Parents: 8aca46e
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Sep 29 13:34:36 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Sep 29 13:34:36 2017 -0700
----------------------------------------------------------------------
hadoop-project/src/site/markdown/index.md.vm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/373d0a51/hadoop-project/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/markdown/index.md.vm b/hadoop-project/src/site/markdown/index.md.vm
index d9443d6..1526f59 100644
--- a/hadoop-project/src/site/markdown/index.md.vm
+++ b/hadoop-project/src/site/markdown/index.md.vm
@@ -34,7 +34,7 @@ Minimum required Java version increased from Java 7 to Java 8
All Hadoop JARs are now compiled targeting a runtime version of Java 8.
Users still using Java 7 or below must upgrade to Java 8.
-Support for erasure encoding in HDFS
+Support for erasure coding in HDFS
------------------
Erasure coding is a method for durably storing data with significant space
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/35] hadoop git commit: HADOOP-14095. Document caveats about the
default JavaKeyStoreProvider in KMS.
Posted by ae...@apache.org.
HADOOP-14095. Document caveats about the default JavaKeyStoreProvider in KMS.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06df6ab2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06df6ab2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06df6ab2
Branch: refs/heads/HDFS-7240
Commit: 06df6ab25432927cd55918440cc8a28fb4f03bfa
Parents: 66c4171
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Sep 29 19:17:32 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Sep 29 19:34:29 2017 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06df6ab2/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
index 1dd89e9..5490219 100644
--- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
@@ -80,6 +80,8 @@ The password file is looked up in the Hadoop's configuration directory via the c
NOTE: You need to restart the KMS for the configuration changes to take effect.
+NOTE: The KMS server can choose any `KeyProvider` implementation as the backing provider. The example here uses a JavaKeyStoreProvider, which should only be used for experimental purposes and never be used in production. For detailed usage and caveats of JavaKeyStoreProvider, please see [Keystore Passwords section of the Credential Provider API](../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html#Keystore_Passwords).
+
$H3 KMS HTTP Configuration
KMS pre-configures the HTTP port to 9600.
@@ -1184,4 +1186,4 @@ and `/stacks`, configure the following properties in `kms-site.xml`:
to all users and groups, e.g. '*', '* ' and ' *' are all valid.
</description>
</property>
-```
\ No newline at end of file
+```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/35] hadoop git commit: HDFS-12453. TestDataNodeHotSwapVolumes
fails in trunk Jenkins runs. (Lei (Eddy) Xu)
Posted by ae...@apache.org.
HDFS-12453. TestDataNodeHotSwapVolumes fails in trunk Jenkins runs. (Lei (Eddy) Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f40dbc17
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f40dbc17
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f40dbc17
Branch: refs/heads/HDFS-7240
Commit: f40dbc170e7f63b947d99b7945e42a14a588bea6
Parents: 7f6118f
Author: Lei Xu <le...@apache.org>
Authored: Fri Sep 29 10:46:17 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Fri Sep 29 10:46:17 2017 -0700
----------------------------------------------------------------------
.../datanode/TestDataNodeHotSwapVolumes.java | 129 ++++++++++++-------
1 file changed, 83 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f40dbc17/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index 5d4ac1e..df5e297 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.BlockMissingException;
+import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -36,6 +37,8 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
@@ -46,6 +49,7 @@ import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
+import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.junit.After;
@@ -68,6 +72,7 @@ import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -83,6 +88,7 @@ import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -775,12 +781,11 @@ public class TestDataNodeHotSwapVolumes {
private void testRemoveVolumeBeingWrittenForDatanode(int dataNodeIdx)
throws IOException, ReconfigurationException, TimeoutException,
InterruptedException, BrokenBarrierException {
- // Starts DFS cluster with 3 DataNodes to form a pipeline.
- startDFSCluster(1, 3);
+ startDFSCluster(1, 4);
final short REPLICATION = 3;
- final DataNode dn = cluster.getDataNodes().get(dataNodeIdx);
- final FileSystem fs = cluster.getFileSystem();
+ final DistributedFileSystem fs = cluster.getFileSystem();
+ final DFSClient client = fs.getClient();
final Path testFile = new Path("/test");
FSDataOutputStream out = fs.create(testFile, REPLICATION);
@@ -790,54 +795,93 @@ public class TestDataNodeHotSwapVolumes {
out.write(writeBuf);
out.hflush();
- // Make FsDatasetSpi#finalizeBlock a time-consuming operation. So if the
- // BlockReceiver releases volume reference before finalizeBlock(), the blocks
- // on the volume will be removed, and finalizeBlock() throws IOE.
- final FsDatasetSpi<? extends FsVolumeSpi> data = dn.data;
- dn.data = Mockito.spy(data);
- doAnswer(new Answer<Object>() {
- public Object answer(InvocationOnMock invocation)
- throws IOException, InterruptedException {
+ BlockLocation[] blocks = fs.getFileBlockLocations(testFile, 0, BLOCK_SIZE);
+ String[] dataNodeNames = blocks[0].getNames();
+ String dataNodeName = dataNodeNames[dataNodeIdx];
+ int xferPort = Integer.parseInt(dataNodeName.split(":")[1]);
+ DataNode dn = null;
+ for (DataNode dataNode : cluster.getDataNodes()) {
+ if (dataNode.getXferPort() == xferPort) {
+ dn = dataNode;
+ break;
+ }
+ }
+ assertNotNull(dn);
+
+ final CyclicBarrier barrier = new CyclicBarrier(4);
+ final AtomicBoolean done = new AtomicBoolean(false);
+ DataNodeFaultInjector newInjector = new DataNodeFaultInjector() {
+ public void logDelaySendingAckToUpstream(
+ final String upstreamAddr, final long delayMs) throws IOException {
+ try {
+ // Make all streams which hold the volume references to wait the
+ // reconfiguration thread to start.
+ // It should only block IO during the period of reconfiguration
+ // task running.
+ if (!done.get()) {
+ barrier.await();
+ // Add delays to allow the reconfiguration thread starts before
+ // IO finish.
Thread.sleep(1000);
- // Bypass the argument to FsDatasetImpl#finalizeBlock to verify that
- // the block is not removed, since the volume reference should not
- // be released at this point.
- data.finalizeBlock((ExtendedBlock) invocation.getArguments()[0],
- (boolean) invocation.getArguments()[1]);
- return null;
}
- }).when(dn.data).finalizeBlock(any(ExtendedBlock.class),
- Mockito.anyBoolean());
-
- final CyclicBarrier barrier = new CyclicBarrier(2);
+ } catch (InterruptedException | BrokenBarrierException e) {
+ throw new IOException(e);
+ }
+ }
+ };
+ DataNodeFaultInjector oldInjector = DataNodeFaultInjector.get();
- List<String> oldDirs = getDataDirs(dn);
- final String newDirs = oldDirs.get(1); // Remove the first volume.
- final List<Exception> exceptions = new ArrayList<>();
- Thread reconfigThread = new Thread() {
- public void run() {
+ try {
+ DataNodeFaultInjector.set(newInjector);
+
+ List<String> oldDirs = getDataDirs(dn);
+ LocatedBlocks lbs = client.getLocatedBlocks("/test", 0);
+ LocatedBlock block = lbs.get(0);
+ FsVolumeImpl volume =
+ (FsVolumeImpl) dn.getFSDataset().getVolume(block.getBlock());
+ final String newDirs = oldDirs.stream()
+ .filter((d) -> !d.contains(volume.getStorageLocation().toString()))
+ .collect(Collectors.joining(","));
+ final List<IOException> exceptions = new ArrayList<>();
+ final DataNode dataNode = dn;
+ final CyclicBarrier reconfigBarrier = new CyclicBarrier(2);
+
+ Thread reconfigThread = new Thread(() -> {
try {
+ reconfigBarrier.await();
+
+ // Wake up writing threads on the pipeline to finish the block.
barrier.await();
+
assertThat(
"DN did not update its own config",
- dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, newDirs),
- is(dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY)));
+ dataNode.reconfigurePropertyImpl(
+ DFS_DATANODE_DATA_DIR_KEY, newDirs),
+ is(dataNode.getConf().get(DFS_DATANODE_DATA_DIR_KEY)));
+ done.set(true);
} catch (ReconfigurationException |
InterruptedException |
BrokenBarrierException e) {
- exceptions.add(e);
+ exceptions.add(new IOException(e));
}
- }
- };
- reconfigThread.start();
+ });
+ reconfigThread.start();
- barrier.await();
- rb.nextBytes(writeBuf);
- out.write(writeBuf);
- out.hflush();
- out.close();
+ // Write more data to make sure the stream threads wait on the barrier.
+ rb.nextBytes(writeBuf);
+ out.write(writeBuf);
+ reconfigBarrier.await();
+ out.hflush();
+ out.close();
+
+ reconfigThread.join();
- reconfigThread.join();
+ if (!exceptions.isEmpty()) {
+ throw MultipleIOException.createIOException(exceptions);
+ }
+ } finally {
+ DataNodeFaultInjector.set(oldInjector);
+ }
// Verify if the data directory reconfigure was successful
FsDatasetSpi<? extends FsVolumeSpi> fsDatasetSpi = dn.getFSDataset();
@@ -851,19 +895,12 @@ public class TestDataNodeHotSwapVolumes {
1, fsVolumeReferences.size());
}
- // Add a new DataNode to help with the pipeline recover.
- cluster.startDataNodes(conf, 1, true, null, null, null);
-
// Verify the file has sufficient replications.
DFSTestUtil.waitReplication(fs, testFile, REPLICATION);
// Read the content back
byte[] content = DFSTestUtil.readFileBuffer(fs, testFile);
assertEquals(BLOCK_SIZE, content.length);
- if (!exceptions.isEmpty()) {
- throw new IOException(exceptions.get(0).getCause());
- }
-
// Write more files to make sure that the DataNode that has removed volume
// is still alive to receive data.
for (int i = 0; i < 10; i++) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/35] hadoop git commit: HADOOP-14768. Honoring sticky bit during
Deletion when authorization is enabled in WASB Contributed by Varada
Hemeswari
Posted by ae...@apache.org.
HADOOP-14768. Honoring sticky bit during Deletion when authorization is enabled in WASB
Contributed by Varada Hemeswari
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a530e7ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a530e7ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a530e7ab
Branch: refs/heads/HDFS-7240
Commit: a530e7ab3b3f5bd71143a91266b46787962ac532
Parents: 8e1bd11
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 28 19:52:56 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 28 19:52:56 2017 +0100
----------------------------------------------------------------------
.../hadoop/fs/azure/NativeAzureFileSystem.java | 522 +++++++++++++++--
...stNativeAzureFSAuthWithBlobSpecificKeys.java | 2 +-
.../ITestNativeAzureFSAuthorizationCaching.java | 2 +-
...veAzureFileSystemAuthorizationWithOwner.java | 122 ----
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 284 +++++-----
.../TestNativeAzureFileSystemAuthorization.java | 555 +++++++++++++++++--
6 files changed, 1136 insertions(+), 351 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a530e7ab/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 280c0e0..5f86f84 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -40,6 +40,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Arrays;
import java.util.List;
+import java.util.Stack;
+import java.util.HashMap;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
@@ -80,6 +82,8 @@ import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.fs.azure.NativeAzureFileSystemHelper.*;
+
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.microsoft.azure.storage.StorageException;
@@ -1849,31 +1853,256 @@ public class NativeAzureFileSystem extends FileSystem {
}
/**
- * Delete the specified file or folder. The parameter
- * skipParentFolderLastModifiedTimeUpdate
- * is used in the case of atomic folder rename redo. In that case, there is
- * a lease on the parent folder, so (without reworking the code) modifying
- * the parent folder update time will fail because of a conflict with the
- * lease. Since we are going to delete the folder soon anyway so accurate
- * modified time is not necessary, it's easier to just skip
- * the modified time update.
- *
- * @param f file path to be deleted.
- * @param recursive specify deleting recursively or not.
- * @param skipParentFolderLastModifiedTimeUpdate If true, don't update the folder last
- * modified time.
- * @return true if and only if the file is deleted
- * @throws IOException Thrown when fail to delete file or directory.
+ * Delete file or folder with authorization checks. Most of the code
+ * is duplicate of the actual delete implementation and will be merged
+ * once the performance and funcional aspects are guaranteed not to
+ * regress existing delete semantics.
*/
- public boolean delete(Path f, boolean recursive,
+ private boolean deleteWithAuthEnabled(Path f, boolean recursive,
boolean skipParentFolderLastModifiedTimeUpdate) throws IOException {
- LOG.debug("Deleting file: {}", f.toString());
+ LOG.debug("Deleting file: {}", f);
Path absolutePath = makeAbsolute(f);
Path parentPath = absolutePath.getParent();
- performAuthCheck(parentPath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
+ // If delete is issued for 'root', parentPath will be null
+ // In that case, we perform auth check for root itself before
+ // proceeding for deleting contents under root.
+ if (parentPath != null) {
+ performAuthCheck(parentPath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
+ } else {
+ performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
+ }
+
+ String key = pathToKey(absolutePath);
+
+ // Capture the metadata for the path.
+ FileMetadata metaFile = null;
+ try {
+ metaFile = store.retrieveMetadata(key);
+ } catch (IOException e) {
+
+ Throwable innerException = checkForAzureStorageException(e);
+
+ if (innerException instanceof StorageException
+ && isFileNotFoundException((StorageException) innerException)) {
+
+ return false;
+ }
+ throw e;
+ }
+
+ if (null == metaFile) {
+ // The path to be deleted does not exist.
+ return false;
+ }
+
+ FileMetadata parentMetadata = null;
+ String parentKey = null;
+ if (parentPath != null) {
+ parentKey = pathToKey(parentPath);
+
+ try {
+ parentMetadata = store.retrieveMetadata(parentKey);
+ } catch (IOException e) {
+ Throwable innerException = checkForAzureStorageException(e);
+ if (innerException instanceof StorageException) {
+ // Invalid State.
+ // A FileNotFoundException is not thrown here as the API returns false
+ // if the file not present. But not retrieving metadata here is an
+ // unrecoverable state and can only happen if there is a race condition
+ // hence throwing a IOException
+ if (isFileNotFoundException((StorageException) innerException)) {
+ throw new IOException("File " + f + " has a parent directory "
+ + parentPath + " whose metadata cannot be retrieved. Can't resolve");
+ }
+ }
+ throw e;
+ }
+
+ // Same case as unable to retrieve metadata
+ if (parentMetadata == null) {
+ throw new IOException("File " + f + " has a parent directory "
+ + parentPath + " whose metadata cannot be retrieved. Can't resolve");
+ }
+
+ if (!parentMetadata.isDir()) {
+ // Invalid state: the parent path is actually a file. Throw.
+ throw new AzureException("File " + f + " has a parent directory "
+ + parentPath + " which is also a file. Can't resolve.");
+ }
+ }
+
+ // The path exists, determine if it is a folder containing objects,
+ // an empty folder, or a simple file and take the appropriate actions.
+ if (!metaFile.isDir()) {
+ // The path specifies a file. We need to check the parent path
+ // to make sure it's a proper materialized directory before we
+ // delete the file. Otherwise we may get into a situation where
+ // the file we were deleting was the last one in an implicit directory
+ // (e.g. the blob store only contains the blob a/b and there's no
+ // corresponding directory blob a) and that would implicitly delete
+ // the directory as well, which is not correct.
+
+ if (parentPath != null && parentPath.getParent() != null) {// Not root
+
+ if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
+ LOG.debug("Found an implicit parent directory while trying to"
+ + " delete the file {}. Creating the directory blob for"
+ + " it in {}.", f, parentKey);
+
+ store.storeEmptyFolder(parentKey,
+ createPermissionStatus(FsPermission.getDefault()));
+ } else {
+ if (!skipParentFolderLastModifiedTimeUpdate) {
+ updateParentFolderLastModifiedTime(key);
+ }
+ }
+ }
+
+ // check if the file can be deleted based on sticky bit check
+ // This check will be performed only when authorization is enabled
+ if (isStickyBitCheckViolated(metaFile, parentMetadata)) {
+ throw new WasbAuthorizationException(String.format("%s has sticky bit set. "
+ + "File %s cannot be deleted.", parentPath, f));
+ }
+
+ try {
+ if (store.delete(key)) {
+ instrumentation.fileDeleted();
+ } else {
+ return false;
+ }
+ } catch(IOException e) {
+
+ Throwable innerException = checkForAzureStorageException(e);
+
+ if (innerException instanceof StorageException
+ && isFileNotFoundException((StorageException) innerException)) {
+ return false;
+ }
+
+ throw e;
+ }
+ } else {
+ // The path specifies a folder. Recursively delete all entries under the
+ // folder.
+ LOG.debug("Directory Delete encountered: {}", f);
+ if (parentPath != null && parentPath.getParent() != null) {
+
+ if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
+ LOG.debug("Found an implicit parent directory while trying to"
+ + " delete the directory {}. Creating the directory blob for"
+ + " it in {}. ", f, parentKey);
+
+ store.storeEmptyFolder(parentKey,
+ createPermissionStatus(FsPermission.getDefault()));
+ }
+ }
+
+ // check if the folder can be deleted based on sticky bit check on parent
+ // This check will be performed only when authorization is enabled.
+ if (!metaFile.getKey().equals("/")
+ && isStickyBitCheckViolated(metaFile, parentMetadata)) {
+
+ throw new WasbAuthorizationException(String.format("%s has sticky bit set. "
+ + "File %s cannot be deleted.", parentPath, f));
+ }
+
+ // Iterate through folder contents and get the list of files
+ // and folders that can be deleted. We might encounter IOException
+ // while listing blobs. In such cases, we return false.
+ ArrayList<FileMetadata> fileMetadataList = new ArrayList<>();
+ boolean isPartialDelete = false;
+
+ // Start time for list operation
+ long start = Time.monotonicNow();
+
+ try {
+ // Get list of files/folders that can be deleted
+ // based on authorization checks and stickybit checks
+ isPartialDelete = getFolderContentsToDelete(metaFile, fileMetadataList);
+ } catch (IOException e) {
+ Throwable innerException = checkForAzureStorageException(e);
+
+ if (innerException instanceof StorageException
+ && isFileNotFoundException((StorageException) innerException)) {
+ return false;
+ }
+ throw e;
+ }
+
+ long end = Time.monotonicNow();
+ LOG.debug("Time taken to list {} blobs for delete operation: {} ms",
+ fileMetadataList.size(), (end - start));
+
+ // Here contents holds the list of metadata of the files and folders that can be deleted
+ // under the path that is requested for delete(excluding itself).
+ final FileMetadata[] contents = fileMetadataList.toArray(new FileMetadata[fileMetadataList.size()]);
+
+ if (contents.length > 0 && !recursive) {
+ // The folder is non-empty and recursive delete was not specified.
+ // Throw an exception indicating that a non-recursive delete was
+ // specified for a non-empty folder.
+ throw new IOException("Non-recursive delete of non-empty directory "
+ + f);
+ }
+
+ // Delete all files / folders in current directory stored as list in 'contents'.
+ AzureFileSystemThreadTask task = new AzureFileSystemThreadTask() {
+ @Override
+ public boolean execute(FileMetadata file) throws IOException{
+ if (!deleteFile(file.getKey(), file.isDir())) {
+ LOG.warn("Attempt to delete non-existent {} {}",
+ file.isDir() ? "directory" : "file",
+ file.getKey());
+ }
+ return true;
+ }
+ };
+
+ AzureFileSystemThreadPoolExecutor executor = getThreadPoolExecutor(this.deleteThreadCount,
+ "AzureBlobDeleteThread", "Delete", key, AZURE_DELETE_THREADS);
+
+ if (!executor.executeParallel(contents, task)) {
+ LOG.error("Failed to delete files / subfolders in blob {}", key);
+ return false;
+ }
+
+ if (metaFile.getKey().equals("/")) {
+ LOG.error("Cannot delete root directory {}", f);
+ return false;
+ }
+
+ // Delete the current directory if all underlying contents are deleted
+ if (isPartialDelete || (store.retrieveMetadata(metaFile.getKey()) != null
+ && !deleteFile(metaFile.getKey(), metaFile.isDir()))) {
+ LOG.error("Failed delete directory : {}", f);
+ return false;
+ }
+
+ // Update parent directory last modified time
+ Path parent = absolutePath.getParent();
+ if (parent != null && parent.getParent() != null) { // not root
+ if (!skipParentFolderLastModifiedTimeUpdate) {
+ updateParentFolderLastModifiedTime(key);
+ }
+ }
+ }
+
+ // File or directory was successfully deleted.
+ LOG.debug("Delete Successful for : {}", f);
+ return true;
+ }
+
+ private boolean deleteWithoutAuth(Path f, boolean recursive,
+ boolean skipParentFolderLastModifiedTimeUpdate) throws IOException {
+
+ LOG.debug("Deleting file: {}", f);
+
+ Path absolutePath = makeAbsolute(f);
+ Path parentPath = absolutePath.getParent();
String key = pathToKey(absolutePath);
@@ -1884,10 +2113,10 @@ public class NativeAzureFileSystem extends FileSystem {
metaFile = store.retrieveMetadata(key);
} catch (IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
+ Throwable innerException = checkForAzureStorageException(e);
if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
+ && isFileNotFoundException((StorageException) innerException)) {
return false;
}
@@ -1918,7 +2147,7 @@ public class NativeAzureFileSystem extends FileSystem {
parentMetadata = store.retrieveMetadata(parentKey);
} catch (IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
+ Throwable innerException = checkForAzureStorageException(e);
if (innerException instanceof StorageException) {
// Invalid State.
@@ -1926,7 +2155,7 @@ public class NativeAzureFileSystem extends FileSystem {
// if the file not present. But not retrieving metadata here is an
// unrecoverable state and can only happen if there is a race condition
// hence throwing a IOException
- if (NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
+ if (isFileNotFoundException((StorageException) innerException)) {
throw new IOException("File " + f + " has a parent directory "
+ parentPath + " whose metadata cannot be retrieved. Can't resolve");
}
@@ -1972,10 +2201,10 @@ public class NativeAzureFileSystem extends FileSystem {
}
} catch(IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
+ Throwable innerException = checkForAzureStorageException(e);
if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
+ && isFileNotFoundException((StorageException) innerException)) {
return false;
}
@@ -1984,7 +2213,7 @@ public class NativeAzureFileSystem extends FileSystem {
} else {
// The path specifies a folder. Recursively delete all entries under the
// folder.
- LOG.debug("Directory Delete encountered: {}", f.toString());
+ LOG.debug("Directory Delete encountered: {}", f);
if (parentPath.getParent() != null) {
String parentKey = pathToKey(parentPath);
FileMetadata parentMetadata = null;
@@ -1993,7 +2222,7 @@ public class NativeAzureFileSystem extends FileSystem {
parentMetadata = store.retrieveMetadata(parentKey);
} catch (IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
+ Throwable innerException = checkForAzureStorageException(e);
if (innerException instanceof StorageException) {
// Invalid State.
@@ -2001,7 +2230,7 @@ public class NativeAzureFileSystem extends FileSystem {
// if the file not present. But not retrieving metadata here is an
// unrecoverable state and can only happen if there is a race condition
// hence throwing a IOException
- if (NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
+ if (isFileNotFoundException((StorageException) innerException)) {
throw new IOException("File " + f + " has a parent directory "
+ parentPath + " whose metadata cannot be retrieved. Can't resolve");
}
@@ -2046,10 +2275,10 @@ public class NativeAzureFileSystem extends FileSystem {
}
priorLastKey = listing.getPriorLastKey();
} catch (IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
+ Throwable innerException = checkForAzureStorageException(e);
if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
+ && isFileNotFoundException((StorageException) innerException)) {
return false;
}
@@ -2067,22 +2296,7 @@ public class NativeAzureFileSystem extends FileSystem {
// The folder is non-empty and recursive delete was not specified.
// Throw an exception indicating that a non-recursive delete was
// specified for a non-empty folder.
- throw new IOException("Non-recursive delete of non-empty directory "
- + f.toString());
- }
- else {
- // Check write-permissions on sub-tree including current folder
- // NOTE: Ideally the subtree needs read-write-execute access check.
- // But we will simplify it to write-access check.
- if (metaFile.isDir()) { // the absolute-path
- performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
- }
- for (FileMetadata meta : contents) {
- if (meta.isDir()) {
- Path subTreeDir = keyToPath(meta.getKey());
- performAuthCheck(subTreeDir, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
- }
- }
+ throw new IOException("Non-recursive delete of non-empty directory "+ f);
}
}
@@ -2108,8 +2322,9 @@ public class NativeAzureFileSystem extends FileSystem {
}
// Delete the current directory
- if (store.retrieveMetadata(metaFile.getKey()) != null && !deleteFile(metaFile.getKey(), metaFile.isDir())) {
- LOG.error("Failed delete directory {}", f.toString());
+ if (store.retrieveMetadata(metaFile.getKey()) != null
+ && !deleteFile(metaFile.getKey(), metaFile.isDir())) {
+ LOG.error("Failed delete directory : {}", f);
return false;
}
@@ -2123,16 +2338,227 @@ public class NativeAzureFileSystem extends FileSystem {
}
// File or directory was successfully deleted.
- LOG.debug("Delete Successful for : {}", f.toString());
+ LOG.debug("Delete Successful for : {}", f);
return true;
}
+ /**
+ * Delete the specified file or folder. The parameter
+ * skipParentFolderLastModifiedTimeUpdate
+ * is used in the case of atomic folder rename redo. In that case, there is
+ * a lease on the parent folder, so (without reworking the code) modifying
+ * the parent folder update time will fail because of a conflict with the
+ * lease. Since we are going to delete the folder soon anyway so accurate
+ * modified time is not necessary, it's easier to just skip
+ * the modified time update.
+ *
+ * @param f file path to be deleted.
+ * @param recursive specify deleting recursively or not.
+ * @param skipParentFolderLastModifiedTimeUpdate If true, don't update the folder last
+ * modified time.
+ * @return true if and only if the file is deleted
+ * @throws IOException Thrown when fail to delete file or directory.
+ */
+ public boolean delete(Path f, boolean recursive,
+ boolean skipParentFolderLastModifiedTimeUpdate) throws IOException {
+
+ if (this.azureAuthorization) {
+ return deleteWithAuthEnabled(f, recursive,
+ skipParentFolderLastModifiedTimeUpdate);
+ } else {
+ return deleteWithoutAuth(f, recursive,
+ skipParentFolderLastModifiedTimeUpdate);
+ }
+ }
+
public AzureFileSystemThreadPoolExecutor getThreadPoolExecutor(int threadCount,
String threadNamePrefix, String operation, String key, String config) {
return new AzureFileSystemThreadPoolExecutor(threadCount, threadNamePrefix, operation, key, config);
}
/**
+ * Gets list of contents that can be deleted based on authorization check calls
+ * performed on the sub-tree for the folderToDelete.
+ *
+ * @param folderToDelete - metadata of the folder whose delete is requested.
+ * @param finalList - list of metadata of all files/folders that can be deleted .
+ *
+ * @return 'true' only if all the contents of the folderToDelete can be deleted
+ * @throws IOException Thrown when current user cannot be retrieved.
+ */
+ private boolean getFolderContentsToDelete(FileMetadata folderToDelete,
+ ArrayList<FileMetadata> finalList) throws IOException {
+
+ final int maxListingDepth = 1;
+ Stack<FileMetadata> foldersToProcess = new Stack<FileMetadata>();
+ HashMap<String, FileMetadata> folderContentsMap = new HashMap<String, FileMetadata>();
+
+ boolean isPartialDelete = false;
+
+ Path pathToDelete = makeAbsolute(keyToPath(folderToDelete.getKey()));
+ foldersToProcess.push(folderToDelete);
+
+ while (!foldersToProcess.empty()) {
+
+ FileMetadata currentFolder = foldersToProcess.pop();
+ Path currentPath = makeAbsolute(keyToPath(currentFolder.getKey()));
+ boolean canDeleteChildren = true;
+
+ // If authorization is enabled, check for 'write' permission on current folder
+ // This check maps to subfolders 'write' check for deleting contents recursively.
+ try {
+ performAuthCheck(currentPath, WasbAuthorizationOperations.WRITE, "delete", pathToDelete);
+ } catch (WasbAuthorizationException we) {
+ LOG.debug("Authorization check failed for {}", currentPath);
+ // We cannot delete the children of currentFolder since 'write' check on parent failed
+ canDeleteChildren = false;
+ }
+
+ if (canDeleteChildren) {
+
+ // get immediate children list
+ ArrayList<FileMetadata> fileMetadataList = getChildrenMetadata(currentFolder.getKey(),
+ maxListingDepth);
+
+ // Process children of currentFolder and add them to list of contents
+ // that can be deleted. We Perform stickybit check on every file and
+ // folder under currentFolder in case stickybit is set on currentFolder.
+ for (FileMetadata childItem : fileMetadataList) {
+ if (isStickyBitCheckViolated(childItem, currentFolder, false)) {
+ // Stickybit check failed for the childItem that is being processed.
+ // This file/folder cannot be deleted and neither can the parent paths be deleted.
+ // Remove parent paths from list of contents that can be deleted.
+ canDeleteChildren = false;
+ Path filePath = makeAbsolute(keyToPath(childItem.getKey()));
+ LOG.error("User does not have permissions to delete {}. "
+ + "Parent directory has sticky bit set.", filePath);
+ } else {
+ // push the child directories to the stack to process their contents
+ if (childItem.isDir()) {
+ foldersToProcess.push(childItem);
+ }
+ // Add items to list of contents that can be deleted.
+ folderContentsMap.put(childItem.getKey(), childItem);
+ }
+ }
+
+ } else {
+ // Cannot delete children since parent permission check has not passed and
+ // if there are files/folders under currentFolder they will not be deleted.
+ LOG.error("Authorization check failed. Files or folders under {} "
+ + "will not be processed for deletion.", currentPath);
+ }
+
+ if (!canDeleteChildren) {
+ // We reach here if
+ // 1. cannot delete children since 'write' check on parent failed or
+ // 2. One of the files under the current folder cannot be deleted due to stickybit check.
+ // In this case we remove all the parent paths from the list of contents
+ // that can be deleted till we reach the original path of delete request
+ String pathToRemove = currentFolder.getKey();
+ while (!pathToRemove.equals(folderToDelete.getKey())) {
+ if (folderContentsMap.containsKey(pathToRemove)) {
+ LOG.debug("Cannot delete {} since some of its contents "
+ + "cannot be deleted", pathToRemove);
+ folderContentsMap.remove(pathToRemove);
+ }
+ Path parentPath = keyToPath(pathToRemove).getParent();
+ pathToRemove = pathToKey(parentPath);
+ }
+ // Since one or more files/folders cannot be deleted return value should indicate
+ // partial delete, so that the delete on the path requested by user is not performed
+ isPartialDelete = true;
+ }
+ }
+
+ // final list of contents that can be deleted
+ for (HashMap.Entry<String, FileMetadata> entry : folderContentsMap.entrySet()) {
+ finalList.add(entry.getValue());
+ }
+
+ return isPartialDelete;
+ }
+
+ private ArrayList<FileMetadata> getChildrenMetadata(String key, int maxListingDepth)
+ throws IOException {
+
+ String priorLastKey = null;
+ ArrayList<FileMetadata> fileMetadataList = new ArrayList<FileMetadata>();
+ do {
+ PartialListing listing = store.listAll(key, AZURE_LIST_ALL,
+ maxListingDepth, priorLastKey);
+ for (FileMetadata file : listing.getFiles()) {
+ fileMetadataList.add(file);
+ }
+ priorLastKey = listing.getPriorLastKey();
+ } while (priorLastKey != null);
+
+ return fileMetadataList;
+ }
+
+ private boolean isStickyBitCheckViolated(FileMetadata metaData,
+ FileMetadata parentMetadata, boolean throwOnException) throws IOException {
+ try {
+ return isStickyBitCheckViolated(metaData, parentMetadata);
+ } catch (FileNotFoundException ex) {
+ if (throwOnException) {
+ throw ex;
+ } else {
+ LOG.debug("Encountered FileNotFoundException while performing "
+ + "stickybit check operation for {}", metaData.getKey());
+ // swallow exception and return that stickyBit check has been violated
+ return true;
+ }
+ }
+ }
+
+ /**
+ * Checks if the Current user is not permitted access to a file/folder when
+ * sticky bit is set on parent path. Only the owner of parent path
+ * and owner of the file/folder itself are permitted to perform certain
+ * operations on file/folder based on sticky bit check. Sticky bit check will
+ * be performed only when authorization is enabled.
+ *
+ * @param metaData - metadata of the file/folder whose parent has sticky bit set.
+ * @param parentMetadata - metadata of the parent.
+ *
+ * @return true if Current user violates stickybit check
+ * @throws IOException Thrown when current user cannot be retrieved.
+ */
+ private boolean isStickyBitCheckViolated(FileMetadata metaData,
+ FileMetadata parentMetadata) throws IOException {
+
+ // In case stickybit check should not be performed,
+ // return value should indicate stickybit check is not violated.
+ if (!this.azureAuthorization) {
+ return false;
+ }
+
+ // This should never happen when the sticky bit check is invoked.
+ if (parentMetadata == null) {
+ throw new FileNotFoundException(
+ String.format("Parent metadata for '%s' not found!", metaData.getKey()));
+ }
+
+ // stickybit is not set on parent and hence cannot be violated
+ if (!parentMetadata.getPermissionStatus().getPermission().getStickyBit()) {
+ return false;
+ }
+
+ String currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
+ String parentDirectoryOwner = parentMetadata.getPermissionStatus().getUserName();
+ String currentFileOwner = metaData.getPermissionStatus().getUserName();
+
+ // Files/Folders with no owner set will not pass stickybit check
+ if ((parentDirectoryOwner.equalsIgnoreCase(currentUser))
+ || currentFileOwner.equalsIgnoreCase(currentUser)) {
+
+ return false;
+ }
+ return true;
+ }
+
+ /**
* Delete the specified file or directory and increment metrics.
* If the file or directory does not exist, the operation returns false.
* @param path the path to a file or directory.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a530e7ab/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java
index d7e4831..0f3d127 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java
@@ -27,7 +27,7 @@ import static org.apache.hadoop.fs.azure.SecureStorageInterfaceImpl.KEY_USE_CONT
* to access storage.
*/
public class ITestNativeAzureFSAuthWithBlobSpecificKeys
- extends ITestNativeAzureFileSystemAuthorizationWithOwner {
+ extends TestNativeAzureFileSystemAuthorization {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a530e7ab/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java
index c73b1cc..138063d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java
@@ -27,7 +27,7 @@ import static org.apache.hadoop.fs.azure.CachingAuthorizer.KEY_AUTH_SERVICE_CACH
* Test class to hold all WASB authorization caching related tests.
*/
public class ITestNativeAzureFSAuthorizationCaching
- extends ITestNativeAzureFileSystemAuthorizationWithOwner {
+ extends TestNativeAzureFileSystemAuthorization {
private static final int DUMMY_TTL_VALUE = 5000;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a530e7ab/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAuthorizationWithOwner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAuthorizationWithOwner.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAuthorizationWithOwner.java
deleted file mode 100644
index 3ec42f0..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAuthorizationWithOwner.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import java.security.PrivilegedExceptionAction;
-
-import org.apache.hadoop.fs.Path;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test class that runs wasb authorization tests with owner check enabled.
- */
-public class ITestNativeAzureFileSystemAuthorizationWithOwner
- extends TestNativeAzureFileSystemAuthorization {
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- authorizer.init(fs.getConf(), true);
- }
-
- /**
- * Test case when owner matches current user.
- */
- @Test
- public void testOwnerPermissionPositive() throws Throwable {
-
- Path parentDir = new Path("/testOwnerPermissionPositive");
- Path testPath = new Path(parentDir, "test.data");
-
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
- authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
- // additional rule used for assertPathExists
- authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.READ.toString(), true);
- fs.updateWasbAuthorizer(authorizer);
-
- try {
- // creates parentDir with owner as current user
- fs.mkdirs(parentDir);
- ContractTestUtils.assertPathExists(fs, "parentDir does not exist", parentDir);
-
- fs.create(testPath);
- fs.getFileStatus(testPath);
- ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath);
-
- } finally {
- allowRecursiveDelete(fs, parentDir.toString());
- fs.delete(parentDir, true);
- }
- }
-
- /**
- * Negative test case for owner does not match current user.
- */
- @Test
- public void testOwnerPermissionNegative() throws Throwable {
- expectedEx.expect(WasbAuthorizationException.class);
-
- Path parentDir = new Path("/testOwnerPermissionNegative");
- Path childDir = new Path(parentDir, "childDir");
-
- setExpectedFailureMessage("mkdirs", childDir);
-
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
-
- fs.updateWasbAuthorizer(authorizer);
-
- try{
- fs.mkdirs(parentDir);
- UserGroupInformation ugiSuperUser = UserGroupInformation.createUserForTesting(
- "testuser", new String[] {});
-
- ugiSuperUser.doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- fs.mkdirs(childDir);
- return null;
- }
- });
-
- } finally {
- allowRecursiveDelete(fs, parentDir.toString());
- fs.delete(parentDir, true);
- }
- }
-
- /**
- * Test to verify that retrieving owner information does not
- * throw when file/folder does not exist.
- */
- @Test
- public void testRetrievingOwnerDoesNotFailWhenFileDoesNotExist() throws Throwable {
-
- Path testdirectory = new Path("/testDirectory123454565");
-
- String owner = fs.getOwnerForPath(testdirectory);
- assertEquals("", owner);
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a530e7ab/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
index 7354499..1f5072d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
@@ -34,165 +34,183 @@ import org.apache.hadoop.fs.Path;
public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface {
- private Map<AuthorizationComponent, Boolean> authRules;
- private boolean performOwnerMatch;
- private CachingAuthorizer<CachedAuthorizerEntry, Boolean> cache;
-
- // The full qualified URL to the root directory
- private String qualifiedPrefixUrl;
-
- public MockWasbAuthorizerImpl(NativeAzureFileSystem fs) {
- qualifiedPrefixUrl = new Path("/").makeQualified(fs.getUri(),
- fs.getWorkingDirectory())
- .toString().replaceAll("/$", "");
- cache = new CachingAuthorizer<>(TimeUnit.MINUTES.convert(5L, TimeUnit.MINUTES), "AUTHORIZATION");
- }
-
- @Override
- public void init(Configuration conf) {
- init(conf, false);
- }
-
- /*
- authorization matches owner with currentUserShortName while evaluating auth rules
- if currentUserShortName is set to a string that is not empty
- */
- public void init(Configuration conf, boolean matchOwner) {
- cache.init(conf);
- authRules = new HashMap<>();
- this.performOwnerMatch = matchOwner;
- }
-
- public void addAuthRule(String wasbAbsolutePath,
- String accessType, boolean access) {
- wasbAbsolutePath = qualifiedPrefixUrl + wasbAbsolutePath;
- AuthorizationComponent component = wasbAbsolutePath.endsWith("*")
- ? new AuthorizationComponent("^" + wasbAbsolutePath.replace("*", ".*"),
- accessType)
- : new AuthorizationComponent(wasbAbsolutePath, accessType);
-
- this.authRules.put(component, access);
- }
-
- @Override
- public boolean authorize(String wasbAbsolutePath,
- String accessType,
- String owner)
- throws WasbAuthorizationException {
-
- if (wasbAbsolutePath.endsWith(
- NativeAzureFileSystem.FolderRenamePending.SUFFIX)) {
- return true;
- }
+ private Map<AuthorizationComponent, Boolean> authRules;
+ private CachingAuthorizer<CachedAuthorizerEntry, Boolean> cache;
+
+ // The full qualified URL to the root directory
+ private String qualifiedPrefixUrl;
- CachedAuthorizerEntry cacheKey = new CachedAuthorizerEntry(wasbAbsolutePath, accessType, owner);
- Boolean cacheresult = cache.get(cacheKey);
- if (cacheresult != null) {
- return cacheresult;
+ public MockWasbAuthorizerImpl(NativeAzureFileSystem fs) {
+ qualifiedPrefixUrl = new Path("/").makeQualified(fs.getUri(),
+ fs.getWorkingDirectory())
+ .toString().replaceAll("/$", "");
+ cache = new CachingAuthorizer<>(TimeUnit.MINUTES.convert(5L, TimeUnit.MINUTES), "AUTHORIZATION");
}
- boolean authorizeresult = authorizeInternal(wasbAbsolutePath, accessType, owner);
- cache.put(cacheKey, authorizeresult);
+ @Override
+ public void init(Configuration conf) {
+ cache.init(conf);
+ authRules = new HashMap<>();
+ }
- return authorizeresult;
- }
+ public void addAuthRuleForOwner(String wasbAbsolutePath,
+ String accessType, boolean access) {
+ addAuthRule(wasbAbsolutePath, accessType, "owner", access);
+ }
- private boolean authorizeInternal(String wasbAbsolutePath, String accessType, String owner)
- throws WasbAuthorizationException {
+ public void addAuthRule(String wasbAbsolutePath,
+ String accessType, String user, boolean access) {
+ wasbAbsolutePath = qualifiedPrefixUrl + wasbAbsolutePath;
+ AuthorizationComponent component = wasbAbsolutePath.endsWith("*")
+ ? new AuthorizationComponent("^" + wasbAbsolutePath.replace("*", ".*"),
+ accessType, user)
+ : new AuthorizationComponent(wasbAbsolutePath, accessType, user);
- String currentUserShortName = "";
- if (this.performOwnerMatch) {
- try {
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- currentUserShortName = ugi.getShortUserName();
- } catch (Exception e) {
- //no op
- }
+ this.authRules.put(component, access);
}
- // In case of root("/"), owner match does not happen because owner is returned as empty string.
- // we try to force owner match just for purpose of tests to make sure all operations work seemlessly with owner.
- if (this.performOwnerMatch
- && StringUtils.equalsIgnoreCase(wasbAbsolutePath,
- qualifiedPrefixUrl + "/")) {
- owner = currentUserShortName;
- }
+ @Override
+ public boolean authorize(String wasbAbsolutePath,
+ String accessType,
+ String owner)
+ throws WasbAuthorizationException {
+
+ if (wasbAbsolutePath.endsWith(
+ NativeAzureFileSystem.FolderRenamePending.SUFFIX)) {
+ return true;
+ }
- boolean shouldEvaluateOwnerAccess = owner != null && !owner.isEmpty()
- && this.performOwnerMatch;
-
- boolean isOwnerMatch = StringUtils.equalsIgnoreCase(currentUserShortName,
- owner);
-
- AuthorizationComponent component =
- new AuthorizationComponent(wasbAbsolutePath, accessType);
-
- if (authRules.containsKey(component)) {
- return shouldEvaluateOwnerAccess ? isOwnerMatch && authRules.get(
- component) : authRules.get(component);
- } else {
- // Regex-pattern match if we don't have a straight match
- for (Map.Entry<AuthorizationComponent, Boolean> entry : authRules.entrySet()) {
- AuthorizationComponent key = entry.getKey();
- String keyPath = key.getWasbAbsolutePath();
- String keyAccess = key.getAccessType();
-
- if (keyPath.endsWith("*") && Pattern.matches(keyPath, wasbAbsolutePath)
- && keyAccess.equals(accessType)) {
- return shouldEvaluateOwnerAccess
- ? isOwnerMatch && entry.getValue()
- : entry.getValue();
+ CachedAuthorizerEntry cacheKey = new CachedAuthorizerEntry(wasbAbsolutePath, accessType, owner);
+ Boolean cacheresult = cache.get(cacheKey);
+ if (cacheresult != null) {
+ return cacheresult;
}
- }
- return false;
+
+ boolean authorizeresult = authorizeInternal(wasbAbsolutePath, accessType, owner);
+ cache.put(cacheKey, authorizeresult);
+
+ return authorizeresult;
}
- }
- public void deleteAllAuthRules() {
- authRules.clear();
- cache.clear();
- }
+ private boolean authorizeInternal(String wasbAbsolutePath, String accessType, String owner)
+ throws WasbAuthorizationException {
- private static class AuthorizationComponent {
+ String currentUserShortName = "";
+ try {
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ currentUserShortName = ugi.getShortUserName();
+ } catch (Exception e) {
+ //no op
+ }
- private final String wasbAbsolutePath;
- private final String accessType;
+ // In case of root("/"), owner match does not happen
+ // because owner is returned as empty string.
+ // we try to force owner match just for purpose of tests
+ // to make sure all operations work seemlessly with owner.
+ if (StringUtils.equalsIgnoreCase(wasbAbsolutePath, qualifiedPrefixUrl + "/")) {
+ owner = currentUserShortName;
+ }
- AuthorizationComponent(String wasbAbsolutePath,
- String accessType) {
- this.wasbAbsolutePath = wasbAbsolutePath;
- this.accessType = accessType;
+ AuthorizationComponent component = new AuthorizationComponent(wasbAbsolutePath,
+ accessType, currentUserShortName);
+
+ return processRules(authRules, component, owner);
}
- @Override
- public int hashCode() {
- return this.wasbAbsolutePath.hashCode() ^ this.accessType.hashCode();
+ private boolean processRules(Map<AuthorizationComponent, Boolean> authRules,
+ AuthorizationComponent component, String owner) {
+
+ // Direct match of rules and access request
+ if (authRules.containsKey(component)) {
+ return authRules.get(component);
+ } else {
+ // Regex-pattern match if we don't have a straight match for path
+ // Current user match if we don't have a owner match
+ for (Map.Entry<AuthorizationComponent, Boolean> entry : authRules.entrySet()) {
+ AuthorizationComponent key = entry.getKey();
+ String keyPath = key.getWasbAbsolutePath();
+ String keyAccess = key.getAccessType();
+ String keyUser = key.getUser();
+
+ boolean foundMatchingOwnerRule = keyPath.equals(component.getWasbAbsolutePath())
+ && keyAccess.equals(component.getAccessType())
+ && keyUser.equalsIgnoreCase("owner")
+ && owner.equals(component.getUser());
+
+ boolean foundMatchingPatternRule = keyPath.endsWith("*")
+ && Pattern.matches(keyPath, component.getWasbAbsolutePath())
+ && keyAccess.equals(component.getAccessType())
+ && keyUser.equalsIgnoreCase(component.getUser());
+
+ boolean foundMatchingPatternOwnerRule = keyPath.endsWith("*")
+ && Pattern.matches(keyPath, component.getWasbAbsolutePath())
+ && keyAccess.equals(component.getAccessType())
+ && keyUser.equalsIgnoreCase("owner")
+ && owner.equals(component.getUser());
+
+ if (foundMatchingOwnerRule
+ || foundMatchingPatternRule
+ || foundMatchingPatternOwnerRule) {
+ return entry.getValue();
+ }
+ }
+ return false;
+ }
}
- @Override
- public boolean equals(Object obj) {
+ public void deleteAllAuthRules() {
+ authRules.clear();
+ cache.clear();
+ }
+
+ private static class AuthorizationComponent {
+
+ private final String wasbAbsolutePath;
+ private final String accessType;
+ private final String user;
- if (obj == this) {
- return true;
+ AuthorizationComponent(String wasbAbsolutePath,
+ String accessType, String user) {
+ this.wasbAbsolutePath = wasbAbsolutePath;
+ this.accessType = accessType;
+ this.user = user;
}
- if (obj == null
- || !(obj instanceof AuthorizationComponent)) {
- return false;
+ @Override
+ public int hashCode() {
+ return this.wasbAbsolutePath.hashCode() ^ this.accessType.hashCode();
}
- return ((AuthorizationComponent) obj).
+ @Override
+ public boolean equals(Object obj) {
+
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj == null
+ || !(obj instanceof AuthorizationComponent)) {
+ return false;
+ }
+
+ return ((AuthorizationComponent) obj).
getWasbAbsolutePath().equals(this.wasbAbsolutePath)
&& ((AuthorizationComponent) obj).
- getAccessType().equals(this.accessType);
- }
+ getAccessType().equals(this.accessType)
+ && ((AuthorizationComponent) obj).
+ getUser().equals(this.user);
+ }
- public String getWasbAbsolutePath() {
- return this.wasbAbsolutePath;
- }
+ public String getWasbAbsolutePath() {
+ return this.wasbAbsolutePath;
+ }
- public String getAccessType() {
- return accessType;
- }
- }
+ public String getAccessType() {
+ return accessType;
+ }
+
+ public String getUser() {
+ return user;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a530e7ab/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
index 4bf6f04..2129b33 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
@@ -19,12 +19,14 @@
package org.apache.hadoop.fs.azure;
import java.security.PrivilegedExceptionAction;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
@@ -47,6 +49,13 @@ public class TestNativeAzureFileSystemAuthorization
@VisibleForTesting
protected MockWasbAuthorizerImpl authorizer;
+ @VisibleForTesting
+ protected static final short STICKYBIT_PERMISSION_CONSTANT = 1700;
+ @VisibleForTesting
+ protected static final String READ = WasbAuthorizationOperations.READ.toString();
+ @VisibleForTesting
+ protected static final String WRITE = WasbAuthorizationOperations.WRITE.toString();
+
@Override
public Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
@@ -80,14 +89,16 @@ public class TestNativeAzureFileSystemAuthorization
/**
* Setup up permissions to allow a recursive delete for cleanup purposes.
*/
- protected void allowRecursiveDelete(NativeAzureFileSystem fs, String path) {
+ protected void allowRecursiveDelete(NativeAzureFileSystem fs, String path)
+ throws IOException {
int index = path.lastIndexOf('/');
String parent = (index == 0) ? "/" : path.substring(0, index);
authorizer.deleteAllAuthRules();
- authorizer.addAuthRule(parent, WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule((path.endsWith("*") ? path : path+"*"), WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(parent, WRITE, getCurrentUserShortName(), true);
+ authorizer.addAuthRule((path.endsWith("*") ? path : path+"*"), WRITE,
+ getCurrentUserShortName(), true);
fs.updateWasbAuthorizer(authorizer);
}
@@ -101,6 +112,13 @@ public class TestNativeAzureFileSystemAuthorization
}
/**
+ * get current user short name for user context
+ */
+ protected String getCurrentUserShortName() throws IOException {
+ return UserGroupInformation.getCurrentUser().getShortUserName();
+ }
+
+ /**
* Positive test to verify Create access check.
* The file is created directly under an existing folder.
* No intermediate folders need to be created.
@@ -112,7 +130,7 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDir = new Path("/");
Path testPath = new Path(parentDir, "test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -137,7 +155,7 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDir = new Path("/testCreateAccessCheckPositive/1/2/3");
Path testPath = new Path(parentDir, "test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -164,7 +182,7 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("create", testPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -188,8 +206,8 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDir = new Path("/");
Path testPath = new Path(parentDir, "test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -215,7 +233,7 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("create", testPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false);
+ authorizer.addAuthRuleForOwner("/", WRITE, false);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -239,8 +257,8 @@ public class TestNativeAzureFileSystemAuthorization
Path intermediateFolders = new Path(parentDir, "1/2/3/");
Path testPath = new Path(intermediateFolders, "test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -266,8 +284,8 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("liststatus", testPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), false);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, false);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -291,8 +309,10 @@ public class TestNativeAzureFileSystemAuthorization
Path srcPath = new Path(parentDir, "test1.dat");
Path dstPath = new Path(parentDir, "test2.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parentDir */
- authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true); /* for rename */
+ /* to create parentDir */
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ /* for rename */
+ authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -321,8 +341,9 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("rename", srcPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dir */
- authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), false);
+ /* to create parent dir */
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, false);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -352,9 +373,9 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("rename", dstPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dir */
- authorizer.addAuthRule(parentSrcDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(parentDstDir.toString(), WasbAuthorizationOperations.WRITE.toString(), false);
+ authorizer.addAuthRuleForOwner("/", WRITE, true); /* to create parent dir */
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDstDir.toString(), WRITE, false);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -381,9 +402,9 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDstDir = new Path("/testRenameAccessCheckPositiveDst");
Path dstPath = new Path(parentDstDir, "test2.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dirs */
- authorizer.addAuthRule(parentSrcDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(parentDstDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true); /* to create parent dirs */
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDstDir.toString(), WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -412,8 +433,8 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDir = new Path("/testReadAccessCheckPositive");
Path testPath = new Path(parentDir, "test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
FSDataInputStream inputStream = null;
@@ -453,8 +474,8 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("read", testPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), false);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, false);
fs.updateWasbAuthorizer(authorizer);
FSDataInputStream inputStream = null;
@@ -490,7 +511,7 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDir = new Path("/");
Path testPath = new Path(parentDir, "test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
fs.create(testPath);
@@ -514,7 +535,7 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("delete", testPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
fs.create(testPath);
@@ -523,7 +544,7 @@ public class TestNativeAzureFileSystemAuthorization
/* Remove permissions for delete to force failure */
authorizer.deleteAllAuthRules();
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false);
+ authorizer.addAuthRuleForOwner("/", WRITE, false);
fs.updateWasbAuthorizer(authorizer);
fs.delete(testPath, false);
@@ -531,7 +552,7 @@ public class TestNativeAzureFileSystemAuthorization
finally {
/* Restore permissions to force a successful delete */
authorizer.deleteAllAuthRules();
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
fs.delete(testPath, false);
@@ -550,9 +571,9 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDir = new Path("/testDeleteIntermediateFolder");
Path testPath = new Path(parentDir, "1/2/test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); // for create and delete
- authorizer.addAuthRule("/testDeleteIntermediateFolder*",
- WasbAuthorizationOperations.WRITE.toString(), true); // for recursive delete
+ authorizer.addAuthRuleForOwner("/", WRITE, true); // for create and delete
+ authorizer.addAuthRuleForOwner("/testDeleteIntermediateFolder*",
+ WRITE, true); // for recursive delete
fs.updateWasbAuthorizer(authorizer);
try {
@@ -568,6 +589,365 @@ public class TestNativeAzureFileSystemAuthorization
}
/**
+ * Test to verify access check failure leaves intermediate folders undeleted.
+ * @throws Throwable
+ */
+ @Test
+ public void testDeleteAuthCheckFailureLeavesFilesUndeleted() throws Throwable {
+
+ Path parentDir = new Path("/testDeleteAuthCheckFailureLeavesFilesUndeleted");
+ Path testPath1 = new Path(parentDir, "child1/test.dat");
+ Path testPath2 = new Path(parentDir, "child2/test.dat");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted*",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testPath1);
+ fs.create(testPath2);
+ ContractTestUtils.assertPathExists(fs, "testPath1 was not created", testPath1);
+ ContractTestUtils.assertPathExists(fs, "testPath2 was not created", testPath2);
+
+ // revoke write on one of the child folders
+ authorizer.deleteAllAuthRules();
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted",
+ WRITE, true);
+ authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted/child2",
+ WRITE, true);
+ authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted/child1",
+ WRITE, false);
+
+ assertFalse(fs.delete(parentDir, true));
+
+ // Assert that only child2 contents are deleted
+ ContractTestUtils.assertPathExists(fs, "child1 is deleted!", testPath1);
+ ContractTestUtils.assertPathDoesNotExist(fs, "child2 exists after deletion!", testPath2);
+ ContractTestUtils.assertPathDoesNotExist(fs, "child2 exists after deletion!",
+ new Path("/testDeleteAuthCheckFailureLeavesFilesUndeleted/childPath2"));
+ ContractTestUtils.assertPathExists(fs, "parentDir is deleted!", parentDir);
+
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Positive test to verify file delete with sticky bit on parent.
+ * @throws Throwable
+ */
+ @Test
+ public void testSingleFileDeleteWithStickyBitPositive() throws Throwable {
+
+ Path parentDir = new Path("/testSingleFileDeleteWithStickyBitPositive");
+ Path testPath = new Path(parentDir, "test.dat");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testSingleFileDeleteWithStickyBitPositive",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testPath);
+ ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath);
+
+ // set stickybit on parent directory
+ fs.setPermission(parentDir, new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+
+ assertTrue(fs.delete(testPath, true));
+ ContractTestUtils.assertPathDoesNotExist(fs,
+ "testPath exists after deletion!", testPath);
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Negative test to verify file delete fails when sticky bit is set on parent
+ * and non-owner user performs delete
+ * @throws Throwable
+ */
+ @Test
+ public void testSingleFileDeleteWithStickyBitNegative() throws Throwable {
+
+ Path parentDir = new Path("/testSingleFileDeleteWithStickyBitNegative");
+ Path testPath = new Path(parentDir, "test.dat");
+
+ expectedEx.expect(WasbAuthorizationException.class);
+ expectedEx.expectMessage(String.format("%s has sticky bit set. File %s cannot be deleted.",
+ parentDir.toString(), testPath.toString()));
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testSingleFileDeleteWithStickyBitNegative",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testPath);
+ ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath);
+ // set stickybit on parent directory
+ fs.setPermission(parentDir, new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+
+ UserGroupInformation dummyUser = UserGroupInformation.createUserForTesting(
+ "dummyUser", new String[] {"dummygroup"});
+
+ dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ authorizer.addAuthRule(parentDir.toString(), WRITE,
+ getCurrentUserShortName(), true);
+ fs.delete(testPath, true);
+ return null;
+ }
+ });
+ }
+ finally {
+ ContractTestUtils.assertPathExists(fs, "testPath should not be deleted!", testPath);
+
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Positive test to verify file and folder delete succeeds with stickybit
+ * when the owner of the files deletes the file.
+ * @throws Throwable
+ */
+ @Test
+ public void testRecursiveDeleteSucceedsWithStickybit() throws Throwable {
+
+ Path parentDir = new Path("/testRecursiveDeleteSucceedsWithStickybit");
+ Path testFilePath = new Path(parentDir, "child/test.dat");
+ Path testFolderPath = new Path(parentDir, "child/testDirectory");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testRecursiveDeleteSucceedsWithStickybit*",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testFilePath);
+ ContractTestUtils.assertPathExists(fs, "file was not created", testFilePath);
+ fs.mkdirs(testFolderPath);
+ ContractTestUtils.assertPathExists(fs, "folder was not created", testFolderPath);
+ // set stickybit on child directory
+ fs.setPermission(new Path(parentDir, "child"),
+ new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+ // perform delete as owner of the files
+ assertTrue(fs.delete(parentDir, true));
+ ContractTestUtils.assertPathDoesNotExist(fs, "parentDir exists after deletion!", parentDir);
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Test to verify delete fails for child files and folders when
+ * non-owner user performs delete and stickybit is set on parent
+ * @throws Throwable
+ */
+ @Test
+ public void testRecursiveDeleteFailsWithStickybit() throws Throwable {
+
+ Path parentDir = new Path("/testRecursiveDeleteFailsWithStickybit");
+ Path testFilePath = new Path(parentDir, "child/test.dat");
+ Path testFolderPath = new Path(parentDir, "child/testDirectory");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testRecursiveDeleteFailsWithStickybit*",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testFilePath);
+ ContractTestUtils.assertPathExists(fs, "file was not created", testFilePath);
+ fs.mkdirs(testFolderPath);
+ ContractTestUtils.assertPathExists(fs, "folder was not created", testFolderPath);
+
+ // set stickybit on child directory
+ fs.setPermission(new Path(parentDir, "child"),
+ new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+
+ UserGroupInformation dummyUser = UserGroupInformation.createUserForTesting(
+ "dummyUser", new String[] {"dummygroup"});
+
+ dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // Add auth rules for dummyuser
+ authorizer.addAuthRule("/", WRITE,
+ getCurrentUserShortName(), true);
+ authorizer.addAuthRule("/testRecursiveDeleteFailsWithStickybit*",
+ WRITE, getCurrentUserShortName(), true);
+
+ assertFalse(fs.delete(parentDir, true));
+ return null;
+ }
+ });
+
+ ContractTestUtils.assertPathExists(fs, "parentDir is deleted!", parentDir);
+ ContractTestUtils.assertPathExists(fs, "file is deleted!", testFilePath);
+ ContractTestUtils.assertPathExists(fs, "folder is deleted!", testFolderPath);
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Test delete scenario where sticky bit check leaves files/folders not owned
+ * by a specific user intact and the files owned by him/her are deleted
+ * @throws Throwable
+ */
+ @Test
+ public void testDeleteSucceedsForOnlyFilesOwnedByUserWithStickybitSet()
+ throws Throwable {
+
+ Path parentDir = new Path("/testDeleteSucceedsForOnlyFilesOwnedByUserWithStickybitSet");
+ Path testFilePath = new Path(parentDir, "test.dat");
+ Path testFolderPath = new Path(parentDir, "testDirectory");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(
+ "/testDeleteSucceedsForOnlyFilesOwnedByUserWithStickybitSet*",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testFilePath);
+ ContractTestUtils.assertPathExists(fs, "file was not created", testFilePath);
+
+ fs.setPermission(parentDir, new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+
+ UserGroupInformation dummyUser = UserGroupInformation.createUserForTesting(
+ "dummyuser", new String[] {"dummygroup"});
+ dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ authorizer.addAuthRule("/", WRITE,
+ getCurrentUserShortName(), true);
+ authorizer.addAuthRule("/testDeleteSucceedsForOnlyFilesOwnedByUserWithStickybitSet*",
+ WRITE, getCurrentUserShortName(), true);
+
+ fs.create(testFolderPath); // the folder will have owner as dummyuser
+ ContractTestUtils.assertPathExists(fs, "folder was not created", testFolderPath);
+ assertFalse(fs.delete(parentDir, true));
+
+ ContractTestUtils.assertPathDoesNotExist(fs, "folder should have been deleted!",
+ testFolderPath);
+ ContractTestUtils.assertPathExists(fs, "parentDir is deleted!", parentDir);
+ ContractTestUtils.assertPathExists(fs, "file is deleted!", testFilePath);
+ return null;
+ }
+ });
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Test delete scenario where sticky bit is set and the owner of parent
+ * directory can delete child files/folders which he does not own.
+ * This is according to the sticky bit behaviour specified in hdfs permission
+ * guide which is as follows - The sticky bit can be set on directories,
+ * preventing anyone except the superuser, directory owner or file owner
+ * from deleting or moving the files within the directory
+ * @throws Throwable
+ */
+ @Test
+ public void testDeleteSucceedsForParentDirectoryOwnerUserWithStickybit() throws Throwable {
+
+ Path parentDir = new Path("/testDeleteSucceedsForParentDirectoryOwnerUserWithStickybit");
+ Path testFilePath = new Path(parentDir, "test.dat");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(
+ "/testDeleteSucceedsForParentDirectoryOwnerUserWithStickybit*",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ // create folder with owner as current user
+ fs.mkdirs(parentDir);
+ ContractTestUtils.assertPathExists(fs, "folder was not created", parentDir);
+
+ // create child with owner as dummyUser
+ UserGroupInformation dummyUser = UserGroupInformation.createUserForTesting(
+ "dummyUser", new String[] {"dummygroup"});
+ dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ authorizer.addAuthRule("/testDeleteSucceedsForParentDirectoryOwnerUserWithStickybit",
+ WRITE, getCurrentUserShortName(), true);
+ fs.create(testFilePath);
+ ContractTestUtils.assertPathExists(fs, "file was not created", testFilePath);
+
+ fs.setPermission(parentDir,
+ new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+ return null;
+ }
+ });
+
+ // invoke delete as current user
+ assertTrue(fs.delete(parentDir, true));
+ ContractTestUtils.assertPathDoesNotExist(fs, "parentDir is not deleted!", parentDir);
+ ContractTestUtils.assertPathDoesNotExist(fs, "file is not deleted!", testFilePath);
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Test to verify delete of root succeeds with proper permissions and
+ * leaves root after delete.
+ * @throws Throwable
+ */
+ @Test
+ public void testDeleteScenarioForRoot() throws Throwable {
+ Path rootPath = new Path("/");
+ Path parentDir = new Path("/testDeleteScenarioForRoot");
+ Path testPath1 = new Path(parentDir, "child1/test.dat");
+ Path testPath2 = new Path(parentDir, "child2/testFolder");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testDeleteScenarioForRoot*",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testPath1);
+ fs.create(testPath2);
+ ContractTestUtils.assertPathExists(fs, "testPath1 was not created", testPath1);
+ ContractTestUtils.assertPathExists(fs, "testPath2 was not created", testPath2);
+
+ assertFalse(fs.delete(rootPath, true));
+
+ ContractTestUtils.assertPathDoesNotExist(fs, "file exists after deletion!", testPath1);
+ ContractTestUtils.assertPathDoesNotExist(fs, "folder exists after deletion!", testPath2);
+ ContractTestUtils.assertPathDoesNotExist(fs, "parentDir exists after deletion!", parentDir);
+ ContractTestUtils.assertPathExists(fs, "Root should not have been deleted!", rootPath);
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
* Positive test for getFileStatus. No permissions are required for getting filestatus.
* @throws Throwable
*/
@@ -587,7 +967,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testMkdirsAccessCheckPositive/1/2/3");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -609,7 +989,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testMkdirsWithExistingHierarchyCheckPositive1");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -636,11 +1016,11 @@ public class TestNativeAzureFileSystemAuthorization
Path childPath2 = new Path(childPath1, "2");
Path childPath3 = new Path(childPath2, "3");
- authorizer.addAuthRule("/",
- WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/",
+ WRITE, true);
- authorizer.addAuthRule(childPath1.toString(),
- WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner(childPath1.toString(),
+ WRITE, true);
fs.updateWasbAuthorizer(authorizer);
@@ -675,7 +1055,7 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("mkdirs", testPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false);
+ authorizer.addAuthRuleForOwner("/", WRITE, false);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -697,13 +1077,96 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/");
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
Path testPathWithTripleSlash = new Path("wasb:///" + testPath);
fs.listStatus(testPathWithTripleSlash);
}
+ /**
+ * Test case when owner matches current user
+ */
+ @Test
+ public void testOwnerPermissionPositive() throws Throwable {
+
+ Path parentDir = new Path("/testOwnerPermissionPositive");
+ Path testPath = new Path(parentDir, "test.data");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
+ // additional rule used for assertPathExists
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ // creates parentDir with owner as current user
+ fs.mkdirs(parentDir);
+ ContractTestUtils.assertPathExists(fs, "parentDir does not exist", parentDir);
+
+ fs.create(testPath);
+ fs.getFileStatus(testPath);
+ ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath);
+
+ fs.delete(parentDir, true);
+ ContractTestUtils.assertPathDoesNotExist(fs, "testPath does not exist", testPath);
+
+ } finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Negative test case for owner does not match current user
+ */
+ @Test
+ public void testOwnerPermissionNegative() throws Throwable {
+
+ Path parentDir = new Path("/testOwnerPermissionNegative");
+ Path childDir = new Path(parentDir, "childDir");
+
+ setExpectedFailureMessage("mkdirs", childDir);
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
+
+ fs.updateWasbAuthorizer(authorizer);
+
+ try{
+ fs.mkdirs(parentDir);
+ UserGroupInformation ugiSuperUser = UserGroupInformation.createUserForTesting(
+ "testuser", new String[] {});
+
+ ugiSuperUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ fs.mkdirs(childDir);
+ return null;
+ }
+ });
+
+ } finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Test to verify that retrieving owner information does not
+ * throw when file/folder does not exist
+ */
+ @Test
+ public void testRetrievingOwnerDoesNotFailWhenFileDoesNotExist()
+ throws Throwable {
+
+ Path testdirectory = new Path("/testDirectory123454565");
+
+ String owner = fs.getOwnerForPath(testdirectory);
+ assertEquals("", owner);
+ }
+
/**
* Negative test for setOwner when Authorization is enabled.
*/
@@ -714,7 +1177,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testSetOwnerNegative");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
String owner = null;
@@ -748,7 +1211,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testSetOwnerPositive");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
String newOwner = "newowner";
@@ -793,7 +1256,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testSetOwnerPositiveWildcard");
authorizer.init(conf);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
String newOwner = "newowner";
@@ -839,7 +1302,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testSetOwnerFailsForIllegalSetup");
authorizer.init(conf);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
String owner = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/35] hadoop git commit: YARN-6509. Add a size threshold beyond
which yarn logs will require a force option. (Xuan Gong via wangda)
Posted by ae...@apache.org.
YARN-6509. Add a size threshold beyond which yarn logs will require a force option. (Xuan Gong via wangda)
Change-Id: I755fe903337d4ff9ec35dae5b9cce638794e1d0f
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec2ae306
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec2ae306
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec2ae306
Branch: refs/heads/HDFS-7240
Commit: ec2ae3060a807c8754826af2135a68c08b2e4f13
Parents: febeead
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Sep 29 16:42:56 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Sep 29 16:42:56 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 366 ++++++++++++-------
.../hadoop/yarn/client/cli/TestLogsCLI.java | 41 ++-
.../yarn/logaggregation/LogCLIHelpers.java | 8 +-
.../tfile/LogAggregationTFileController.java | 6 +
4 files changed, 279 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec2ae306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 9a8ba4a..74497ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -29,9 +29,12 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Pattern;
@@ -107,6 +110,7 @@ public class LogsCLI extends Configured implements Tool {
private static final String CLIENT_RETRY_INTERVAL_OPTION
= "client_retry_interval_ms";
public static final String HELP_CMD = "help";
+ private static final String SIZE_LIMIT_OPTION = "size_limit_mb";
private PrintStream outStream = System.out;
private YarnClient yarnClient = null;
@@ -115,6 +119,11 @@ public class LogsCLI extends Configured implements Tool {
private static final int DEFAULT_MAX_RETRIES = 30;
private static final long DEFAULT_RETRY_INTERVAL = 1000;
+ private static final long LOG_SIZE_LIMIT_DEFAULT = 10240L;
+
+ private long logSizeLeft = LOG_SIZE_LIMIT_DEFAULT * 1024 * 1024;
+ private long specifedLogLimits = LOG_SIZE_LIMIT_DEFAULT;
+
@Private
@VisibleForTesting
ClientConnectionRetry connectionRetry;
@@ -158,6 +167,7 @@ public class LogsCLI extends Configured implements Tool {
List<String> amContainersList = new ArrayList<String>();
String localDir = null;
long bytes = Long.MAX_VALUE;
+ boolean ignoreSizeLimit = false;
int maxRetries = DEFAULT_MAX_RETRIES;
long retryInterval = DEFAULT_RETRY_INTERVAL;
try {
@@ -199,6 +209,14 @@ public class LogsCLI extends Configured implements Tool {
retryInterval = Long.parseLong(commandLine.getOptionValue(
CLIENT_RETRY_INTERVAL_OPTION));
}
+ if (commandLine.hasOption(SIZE_LIMIT_OPTION)) {
+ specifedLogLimits = Long.parseLong(commandLine.getOptionValue(
+ SIZE_LIMIT_OPTION));
+ logSizeLeft = specifedLogLimits * 1024 * 1024;
+ }
+ if (logSizeLeft < 0L) {
+ ignoreSizeLimit = true;
+ }
} catch (ParseException e) {
System.err.println("options parsing failed: " + e.getMessage());
printHelpMessage(printOpts);
@@ -306,6 +324,7 @@ public class LogsCLI extends Configured implements Tool {
logs.addAll(Arrays.asList(logFilesRegex));
}
+
ContainerLogsRequest request = new ContainerLogsRequest(appId,
isApplicationFinished(appState), appOwner, nodeAddress, null,
containerIdStr, localDir, logs, bytes, null);
@@ -324,15 +343,17 @@ public class LogsCLI extends Configured implements Tool {
// To get am logs
if (getAMContainerLogs) {
return fetchAMContainerLogs(request, amContainersList,
- logCliHelper, useRegex);
+ logCliHelper, useRegex, ignoreSizeLimit);
}
int resultCode = 0;
if (containerIdStr != null) {
- return fetchContainerLogs(request, logCliHelper, useRegex);
+ return fetchContainerLogs(request, logCliHelper, useRegex,
+ ignoreSizeLimit);
} else {
if (nodeAddress == null) {
- resultCode = fetchApplicationLogs(request, logCliHelper, useRegex);
+ resultCode = fetchApplicationLogs(request, logCliHelper, useRegex,
+ ignoreSizeLimit);
} else {
System.err.println("Should at least provide ContainerId!");
printHelpMessage(printOpts);
@@ -524,35 +545,16 @@ public class LogsCLI extends Configured implements Tool {
@VisibleForTesting
public int printContainerLogsFromRunningApplication(Configuration conf,
ContainerLogsRequest request, LogCLIHelpers logCliHelper,
- boolean useRegex) throws IOException {
+ boolean useRegex, boolean ignoreSizeLimit) throws IOException {
String containerIdStr = request.getContainerId().toString();
String localDir = request.getOutputLocalDir();
- String nodeHttpAddress = request.getNodeHttpAddress();
- if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
- System.err.println("Can not get the logs for the container: "
- + containerIdStr);
- System.err.println("The node http address is required to get container "
- + "logs for the Running application.");
- return -1;
- }
String nodeId = request.getNodeId();
PrintStream out = LogToolUtils.createPrintStream(localDir, nodeId,
containerIdStr);
try {
- Set<String> matchedFiles = getMatchedContainerLogFiles(request,
- useRegex);
- if (matchedFiles.isEmpty()) {
- System.err.println("Can not find any log file matching the pattern: "
- + request.getLogTypes() + " for the container: " + containerIdStr
- + " within the application: " + request.getAppId());
- return -1;
- }
- ContainerLogsRequest newOptions = new ContainerLogsRequest(request);
- newOptions.setLogTypes(matchedFiles);
-
boolean foundAnyLogs = false;
byte[] buffer = new byte[65536];
- for (String logFile : newOptions.getLogTypes()) {
+ for (String logFile : request.getLogTypes()) {
InputStream is = null;
try {
ClientResponse response = getResponeFromNMWebService(conf,
@@ -595,50 +597,6 @@ public class LogsCLI extends Configured implements Tool {
}
}
- private int printContainerLogsForFinishedApplication(
- ContainerLogsRequest request, LogCLIHelpers logCliHelper,
- boolean useRegex) throws IOException {
- ContainerLogsRequest newOptions = getMatchedLogOptions(
- request, logCliHelper, useRegex);
- if (newOptions == null) {
- System.err.println("Can not find any log file matching the pattern: "
- + request.getLogTypes() + " for the container: "
- + request.getContainerId() + " within the application: "
- + request.getAppId());
- return -1;
- }
- return logCliHelper.dumpAContainerLogsForLogType(newOptions);
- }
-
- private int printContainerLogsForFinishedApplicationWithoutNodeId(
- ContainerLogsRequest request, LogCLIHelpers logCliHelper,
- boolean useRegex) throws IOException {
- ContainerLogsRequest newOptions = getMatchedLogOptions(
- request, logCliHelper, useRegex);
- if (newOptions == null) {
- System.err.println("Can not find any log file matching the pattern: "
- + request.getLogTypes() + " for the container: "
- + request.getContainerId() + " within the application: "
- + request.getAppId());
- return -1;
- }
- return logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId(
- newOptions);
- }
-
- private int printAggregatedContainerLogs(ContainerLogsRequest request,
- LogCLIHelpers logCliHelper, boolean useRegex) throws IOException {
- return printContainerLogsForFinishedApplication(request,
- logCliHelper, useRegex);
- }
-
- private int printAggregatedContainerLogsWithoutNodeId(
- ContainerLogsRequest request, LogCLIHelpers logCliHelper,
- boolean useRegex) throws IOException {
- return printContainerLogsForFinishedApplicationWithoutNodeId(request,
- logCliHelper, useRegex);
- }
-
@Private
@VisibleForTesting
public ContainerReport getContainerReport(String containerIdStr)
@@ -655,7 +613,8 @@ public class LogsCLI extends Configured implements Tool {
private int printAMContainerLogs(Configuration conf,
ContainerLogsRequest request, List<String> amContainers,
- LogCLIHelpers logCliHelper, boolean useRegex) throws Exception {
+ LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit)
+ throws Exception {
List<JSONObject> amContainersList = null;
List<ContainerLogsRequest> requests =
new ArrayList<ContainerLogsRequest>();
@@ -717,10 +676,9 @@ public class LogsCLI extends Configured implements Tool {
return -1;
}
+ List<ContainerLogsRequest> candidates = new ArrayList<>();
if (amContainers.contains("ALL")) {
- for (ContainerLogsRequest amRequest : requests) {
- outputAMContainerLogs(amRequest, conf, logCliHelper, useRegex);
- }
+ candidates.addAll(requests);
outStream.println();
outStream.println("Specified ALL for -am option. "
+ "Printed logs for all am containers.");
@@ -728,12 +686,10 @@ public class LogsCLI extends Configured implements Tool {
for (String amContainer : amContainers) {
int amContainerId = Integer.parseInt(amContainer.trim());
if (amContainerId == -1) {
- outputAMContainerLogs(requests.get(requests.size() - 1), conf,
- logCliHelper, useRegex);
+ candidates.add(requests.get(requests.size() - 1));
} else {
if (amContainerId <= requests.size()) {
- outputAMContainerLogs(requests.get(amContainerId - 1), conf,
- logCliHelper, useRegex);
+ candidates.add(requests.get(amContainerId - 1));
} else {
System.err.println(String.format("ERROR: Specified AM containerId"
+ " (%s) exceeds the number of AM containers (%s).",
@@ -743,12 +699,25 @@ public class LogsCLI extends Configured implements Tool {
}
}
}
+ Map<String, ContainerLogsRequest> newOptions = new HashMap<>();
+ if (request.isAppFinished()) {
+ newOptions = getMatchedLogTypesForFinishedApp(candidates,
+ logCliHelper, useRegex, ignoreSizeLimit);
+ } else {
+ newOptions = getMatchedLogTypesForRunningApp(candidates, useRegex,
+ ignoreSizeLimit);
+ }
+ for (Entry<String, ContainerLogsRequest> amRequest
+ : newOptions.entrySet()) {
+ outputAMContainerLogs(amRequest.getValue(), conf, logCliHelper,
+ useRegex, ignoreSizeLimit);
+ }
return 0;
}
private void outputAMContainerLogs(ContainerLogsRequest request,
- Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex)
- throws Exception {
+ Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex,
+ boolean ignoreSizeLimit) throws Exception {
String nodeHttpAddress = request.getNodeHttpAddress();
String containerId = request.getContainerId();
String nodeId = request.getNodeId();
@@ -756,11 +725,10 @@ public class LogsCLI extends Configured implements Tool {
if (request.isAppFinished()) {
if (containerId != null && !containerId.isEmpty()) {
if (nodeId != null && !nodeId.isEmpty()) {
- printContainerLogsForFinishedApplication(request,
- logCliHelper, useRegex);
+ logCliHelper.dumpAContainerLogsForLogType(request);
} else {
- printContainerLogsForFinishedApplicationWithoutNodeId(
- request, logCliHelper, useRegex);
+ logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId(
+ request);
}
}
} else {
@@ -770,7 +738,7 @@ public class LogsCLI extends Configured implements Tool {
.getContainerState();
request.setContainerState(containerState);
printContainerLogsFromRunningApplication(conf,
- request, logCliHelper, useRegex);
+ request, logCliHelper, useRegex, ignoreSizeLimit);
}
}
}
@@ -898,6 +866,13 @@ public class LogsCLI extends Configured implements Tool {
opts.addOption(CLIENT_RETRY_INTERVAL_OPTION, true,
"Work with --client_max_retries to create a retry client. "
+ "The default value is 1000.");
+ opts.addOption(SIZE_LIMIT_OPTION, true, "Use this option to limit "
+ + "the size of the total logs which could be fetched. "
+ + "By default, we only allow to fetch at most "
+ + LOG_SIZE_LIMIT_DEFAULT + " MB logs. If the total log size is "
+ + "larger than the specified number, the CLI would fail. "
+ + "The user could specify -1 to ignore the size limit "
+ + "and fetch all logs.");
opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID");
opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID");
opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address");
@@ -908,6 +883,7 @@ public class LogsCLI extends Configured implements Tool {
opts.getOption(CLIENT_MAX_RETRY_OPTION).setArgName("Max Retries");
opts.getOption(CLIENT_RETRY_INTERVAL_OPTION)
.setArgName("Retry Interval");
+ opts.getOption(SIZE_LIMIT_OPTION).setArgName("Size Limit");
return opts;
}
@@ -933,6 +909,7 @@ public class LogsCLI extends Configured implements Tool {
PER_CONTAINER_LOG_FILES_REGEX_OPTION));
printOpts.addOption(commandOpts.getOption(CLIENT_MAX_RETRY_OPTION));
printOpts.addOption(commandOpts.getOption(CLIENT_RETRY_INTERVAL_OPTION));
+ printOpts.addOption(commandOpts.getOption(SIZE_LIMIT_OPTION));
return printOpts;
}
@@ -969,15 +946,15 @@ public class LogsCLI extends Configured implements Tool {
private int fetchAMContainerLogs(ContainerLogsRequest request,
List<String> amContainersList, LogCLIHelpers logCliHelper,
- boolean useRegex) throws Exception {
+ boolean useRegex, boolean ignoreSizeLimit) throws Exception {
return printAMContainerLogs(getConf(), request, amContainersList,
- logCliHelper, useRegex);
+ logCliHelper, useRegex, ignoreSizeLimit);
}
private int fetchContainerLogs(ContainerLogsRequest request,
- LogCLIHelpers logCliHelper, boolean useRegex) throws IOException,
- ClientHandlerException, UniformInterfaceException, JSONException {
- int resultCode = 0;
+ LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit)
+ throws IOException, ClientHandlerException, UniformInterfaceException,
+ JSONException {
String appIdStr = request.getAppId().toString();
String containerIdStr = request.getContainerId();
String nodeAddress = request.getNodeId();
@@ -988,12 +965,20 @@ public class LogsCLI extends Configured implements Tool {
if (isAppFinished) {
// if user specified "ALL" as the logFiles param, pass empty list
// to logCliHelper so that it fetches all the logs
+ ContainerLogsRequest newOptions = getMatchedLogOptions(
+ request, logCliHelper, useRegex, ignoreSizeLimit);
+ if (newOptions == null) {
+ System.err.println("Can not find any log file matching the pattern: "
+ + request.getLogTypes() + " for the container: "
+ + request.getContainerId() + " within the application: "
+ + request.getAppId());
+ return -1;
+ }
if (nodeAddress != null && !nodeAddress.isEmpty()) {
- return printContainerLogsForFinishedApplication(
- request, logCliHelper, useRegex);
+ return logCliHelper.dumpAContainerLogsForLogType(newOptions);
} else {
- return printContainerLogsForFinishedApplicationWithoutNodeId(
- request, logCliHelper, useRegex);
+ return logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId(
+ newOptions);
}
}
String nodeHttpAddress = null;
@@ -1019,13 +1004,20 @@ public class LogsCLI extends Configured implements Tool {
} else {
// for the case, we have already uploaded partial logs in HDFS
int result = -1;
- if (nodeAddress != null && !nodeAddress.isEmpty()) {
- result = printAggregatedContainerLogs(request,
- logCliHelper, useRegex);
+ ContainerLogsRequest newOptions = getMatchedLogOptions(
+ request, logCliHelper, useRegex, ignoreSizeLimit);
+ if (newOptions == null) {
+ System.err.println("Can not find any log file matching the pattern: "
+ + request.getLogTypes() + " for the container: "
+ + request.getContainerId() + " within the application: "
+ + request.getAppId());
} else {
- result = printAggregatedContainerLogsWithoutNodeId(request,
- logCliHelper,
- useRegex);
+ if (nodeAddress != null && !nodeAddress.isEmpty()) {
+ result = logCliHelper.dumpAContainerLogsForLogType(newOptions);
+ } else {
+ result = logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId(
+ newOptions);
+ }
}
if (result == -1) {
System.err.println(
@@ -1043,14 +1035,18 @@ public class LogsCLI extends Configured implements Tool {
// If the application is not in the final state,
// we will provide the NodeHttpAddress and get the container logs
// by calling NodeManager webservice.
- resultCode = printContainerLogsFromRunningApplication(getConf(), request,
- logCliHelper, useRegex);
- return resultCode;
+ ContainerLogsRequest newRequest = getMatchedOptionForRunningApp(
+ request, useRegex, ignoreSizeLimit);
+ if (newRequest == null) {
+ return -1;
+ }
+ return printContainerLogsFromRunningApplication(getConf(), request,
+ logCliHelper, useRegex, ignoreSizeLimit);
}
private int fetchApplicationLogs(ContainerLogsRequest options,
- LogCLIHelpers logCliHelper, boolean useRegex) throws IOException,
- YarnException {
+ LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit)
+ throws IOException, YarnException {
// If the application has finished, we would fetch the logs
// from HDFS.
// If the application is still running, we would get the full
@@ -1059,7 +1055,7 @@ public class LogsCLI extends Configured implements Tool {
int resultCode = -1;
if (options.isAppFinished()) {
ContainerLogsRequest newOptions = getMatchedLogOptions(
- options, logCliHelper, useRegex);
+ options, logCliHelper, useRegex, ignoreSizeLimit);
if (newOptions == null) {
System.err.println("Can not find any log file matching the pattern: "
+ options.getLogTypes() + " for the application: "
@@ -1071,9 +1067,17 @@ public class LogsCLI extends Configured implements Tool {
} else {
List<ContainerLogsRequest> containerLogRequests =
getContainersLogRequestForRunningApplication(options);
- for (ContainerLogsRequest container : containerLogRequests) {
+
+ // get all matched container log types and check the total log size.
+ Map<String, ContainerLogsRequest> matchedLogTypes =
+ getMatchedLogTypesForRunningApp(containerLogRequests,
+ useRegex, ignoreSizeLimit);
+
+ for (Entry<String, ContainerLogsRequest> container
+ : matchedLogTypes.entrySet()) {
int result = printContainerLogsFromRunningApplication(getConf(),
- container, logCliHelper, useRegex);
+ container.getValue(), logCliHelper,
+ useRegex, ignoreSizeLimit);
if (result == 0) {
resultCode = 0;
}
@@ -1103,37 +1107,54 @@ public class LogsCLI extends Configured implements Tool {
private ContainerLogsRequest getMatchedLogOptions(
ContainerLogsRequest request, LogCLIHelpers logCliHelper,
- boolean useRegex) throws IOException {
+ boolean useRegex, boolean ignoreSizeLimit) throws IOException {
ContainerLogsRequest newOptions = new ContainerLogsRequest(request);
- if (request.getLogTypes() != null && !request.getLogTypes().isEmpty()) {
- Set<String> matchedFiles = new HashSet<String>();
- if (!request.getLogTypes().contains("ALL")) {
- Set<String> files = logCliHelper.listContainerLogs(request);
- matchedFiles = getMatchedLogFiles(request, files, useRegex);
- if (matchedFiles.isEmpty()) {
- return null;
- }
- }
+ Set<ContainerLogFileInfo> files = logCliHelper.listContainerLogs(
+ request);
+ Set<String> matchedFiles = getMatchedLogFiles(request, files,
+ useRegex, ignoreSizeLimit);
+ if (matchedFiles.isEmpty()) {
+ return null;
+ } else {
newOptions.setLogTypes(matchedFiles);
+ return newOptions;
}
- return newOptions;
}
private Set<String> getMatchedLogFiles(ContainerLogsRequest options,
- Collection<String> candidate, boolean useRegex) throws IOException {
+ Collection<ContainerLogFileInfo> candidate, boolean useRegex,
+ boolean ignoreSizeLimit) throws IOException {
Set<String> matchedFiles = new HashSet<String>();
Set<String> filePattern = options.getLogTypes();
- if (options.getLogTypes().contains("ALL")) {
- return new HashSet<String>(candidate);
- }
- for (String file : candidate) {
- if (useRegex) {
- if (isFileMatching(file, filePattern)) {
- matchedFiles.add(file);
+ long size = options.getBytes();
+ boolean getAll = options.getLogTypes().contains("ALL");
+ Iterator<ContainerLogFileInfo> iterator = candidate.iterator();
+ while(iterator.hasNext()) {
+ boolean matchedFile = false;
+ ContainerLogFileInfo logInfo = iterator.next();
+ if (getAll) {
+ matchedFile = true;
+ } else if (useRegex) {
+ if (isFileMatching(logInfo.getFileName(), filePattern)) {
+ matchedFile = true;
}
} else {
- if (filePattern.contains(file)) {
- matchedFiles.add(file);
+ if (filePattern.contains(logInfo.getFileName())) {
+ matchedFile = true;
+ }
+ }
+ if (matchedFile) {
+ matchedFiles.add(logInfo.getFileName());
+ if (!ignoreSizeLimit) {
+ decrLogSizeLimit(Math.min(
+ Long.parseLong(logInfo.getFileSize()), size));
+ if (getLogSizeLimitLeft() < 0) {
+ throw new RuntimeException("The total log size is too large."
+ + "The log size limit is " + specifedLogLimits + "MB. "
+ + "Please specify a proper value --size option or if you "
+ + "really want to fetch all, please "
+ + "specify -1 for --size_limit_mb option.");
+ }
}
}
}
@@ -1296,18 +1317,19 @@ public class LogsCLI extends Configured implements Tool {
@VisibleForTesting
public Set<String> getMatchedContainerLogFiles(ContainerLogsRequest request,
- boolean useRegex) throws IOException {
+ boolean useRegex, boolean ignoreSizeLimit) throws IOException {
// fetch all the log files for the container
// filter the log files based on the given -log_files pattern
List<Pair<ContainerLogFileInfo, String>> allLogFileInfos=
getContainerLogFiles(getConf(), request.getContainerId(),
request.getNodeHttpAddress());
- List<String> fileNames = new ArrayList<String>();
+ List<ContainerLogFileInfo> fileNames = new ArrayList<
+ ContainerLogFileInfo>();
for (Pair<ContainerLogFileInfo, String> fileInfo : allLogFileInfos) {
- fileNames.add(fileInfo.getKey().getFileName());
+ fileNames.add(fileInfo.getKey());
}
return getMatchedLogFiles(request, fileNames,
- useRegex);
+ useRegex, ignoreSizeLimit);
}
@VisibleForTesting
@@ -1451,4 +1473,86 @@ public class LogsCLI extends Configured implements Tool {
// The method to indicate if we should retry given the incoming exception
public abstract boolean shouldRetryOn(Exception e);
}
+
+ private long getLogSizeLimitLeft() {
+ return this.logSizeLeft;
+ }
+
+ private void decrLogSizeLimit(long used) {
+ this.logSizeLeft -= used;
+ }
+
+ @Private
+ @VisibleForTesting
+ public ContainerLogsRequest getMatchedOptionForRunningApp(
+ ContainerLogsRequest container, boolean useRegex,
+ boolean ignoreSizeLimit) throws IOException {
+ String containerIdStr = container.getContainerId().toString();
+ String nodeHttpAddress = container.getNodeHttpAddress();
+ if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
+ System.err.println("Can not get the logs for the container: "
+ + containerIdStr);
+ System.err.println("The node http address is required to get container "
+ + "logs for the Running application.");
+ return null;
+ }
+
+ Set<String> matchedFiles = getMatchedContainerLogFiles(container,
+ useRegex, ignoreSizeLimit);
+ if (matchedFiles.isEmpty()) {
+ System.err.println("Can not find any log file matching the pattern: "
+ + container.getLogTypes() + " for the container: " + containerIdStr
+ + " within the application: " + container.getAppId());
+ return null;
+ }
+ container.setLogTypes(matchedFiles);
+ return container;
+ }
+
+ @Private
+ @VisibleForTesting
+ public Map<String, ContainerLogsRequest> getMatchedLogTypesForRunningApp(
+ List<ContainerLogsRequest> containerLogRequests, boolean useRegex,
+ boolean ignoreSizeLimit) {
+ Map<String, ContainerLogsRequest> containerMatchedLog = new HashMap<>();
+ for (ContainerLogsRequest container : containerLogRequests) {
+ try {
+ ContainerLogsRequest request = getMatchedOptionForRunningApp(
+ container, useRegex, ignoreSizeLimit);
+ if (request == null) {
+ continue;
+ }
+ containerMatchedLog.put(container.getContainerId(), request);
+ } catch(IOException ex) {
+ System.err.println(ex);
+ continue;
+ }
+ }
+ return containerMatchedLog;
+ }
+
+ private Map<String, ContainerLogsRequest> getMatchedLogTypesForFinishedApp(
+ List<ContainerLogsRequest> containerLogRequests,
+ LogCLIHelpers logCliHelper, boolean useRegex,
+ boolean ignoreSizeLimit) {
+ Map<String, ContainerLogsRequest> containerMatchedLog = new HashMap<>();
+ for (ContainerLogsRequest container : containerLogRequests) {
+ try {
+ ContainerLogsRequest request = getMatchedLogOptions(container,
+ logCliHelper, useRegex, ignoreSizeLimit);
+ if (request == null) {
+ System.err.println("Can not find any log file matching the pattern: "
+ + container.getLogTypes() + " for the container: "
+ + container.getContainerId() + " within the application: "
+ + container.getAppId());
+ continue;
+ }
+ containerMatchedLog.put(container.getContainerId(), request);
+ } catch (IOException ex) {
+ System.err.println(ex);
+ continue;
+ }
+ }
+ return containerMatchedLog;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec2ae306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index fed7488..c292430 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -30,6 +30,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Matchers.*;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
@@ -300,6 +301,17 @@ public class TestLogsCLI {
pw.println(" end and positive values as");
pw.println(" bytes to read from the");
pw.println(" beginning.");
+ pw.println(" -size_limit_mb <Size Limit> Use this option to limit the");
+ pw.println(" size of the total logs which");
+ pw.println(" could be fetched. By");
+ pw.println(" default, we only allow to");
+ pw.println(" fetch at most 10240 MB logs.");
+ pw.println(" If the total log size is");
+ pw.println(" larger than the specified");
+ pw.println(" number, the CLI would fail.");
+ pw.println(" The user could specify -1 to");
+ pw.println(" ignore the size limit and");
+ pw.println(" fetch all logs.");
pw.close();
String appReportStr = baos.toString("UTF-8");
Assert.assertTrue(sysOutStream.toString().contains(appReportStr));
@@ -563,8 +575,7 @@ public class TestLogsCLI {
containerId0.toString() });
assertTrue(exitCode == -1);
assertTrue(sysErrStream.toString().contains(
- "Logs for container " + containerId0.toString()
- + " are not present in this log-file."));
+ "Can not find any log file matching the pattern"));
sysErrStream.reset();
// uploaded two logs for container3. The first log is named as syslog.
@@ -750,7 +761,7 @@ public class TestLogsCLI {
Set<String> logsSet = new HashSet<String>();
logsSet.add(fileName);
doReturn(logsSet).when(cli).getMatchedContainerLogFiles(
- any(ContainerLogsRequest.class), anyBoolean());
+ any(ContainerLogsRequest.class), anyBoolean(), anyBoolean());
ClientResponse mockReponse = mock(ClientResponse.class);
doReturn(Status.OK).when(mockReponse).getStatusInfo();
doReturn(fis).when(mockReponse).getEntityInputStream();
@@ -795,6 +806,7 @@ public class TestLogsCLI {
doReturn(nodeId).when(mockContainerReport1).getAssignedNode();
doReturn("http://localhost:2345").when(mockContainerReport1)
.getNodeHttpAddress();
+
ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
ContainerReport mockContainerReport2 = mock(ContainerReport.class);
doReturn(containerId2).when(mockContainerReport2).getContainerId();
@@ -812,7 +824,19 @@ public class TestLogsCLI {
LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient));
doReturn(0).when(cli).printContainerLogsFromRunningApplication(
any(Configuration.class), any(ContainerLogsRequest.class),
- any(LogCLIHelpers.class), anyBoolean());
+ any(LogCLIHelpers.class), anyBoolean(), anyBoolean());
+ Set<String> logTypes = new HashSet<>();
+ logTypes.add("ALL");
+ ContainerLogsRequest mockContainer1 = mock(ContainerLogsRequest.class);
+ doReturn(logTypes).when(mockContainer1).getLogTypes();
+ ContainerLogsRequest mockContainer2 = mock(ContainerLogsRequest.class);
+ doReturn(logTypes).when(mockContainer2).getLogTypes();
+ Map<String, ContainerLogsRequest> matchedLogTypes = new HashMap<>();
+ matchedLogTypes.put(containerId1.toString(), mockContainer1);
+ matchedLogTypes.put(containerId2.toString(), mockContainer2);
+ doReturn(matchedLogTypes).when(cli).getMatchedLogTypesForRunningApp(
+ anyListOf(ContainerLogsRequest.class), anyBoolean(),
+ anyBoolean());
cli.setConf(new YarnConfiguration());
int exitCode = cli.run(new String[] {"-applicationId", appId.toString()});
@@ -825,7 +849,7 @@ public class TestLogsCLI {
// printContainerLogsFromRunningApplication twice
verify(cli, times(2)).printContainerLogsFromRunningApplication(
any(Configuration.class), logsRequestCaptor.capture(),
- any(LogCLIHelpers.class), anyBoolean());
+ any(LogCLIHelpers.class), anyBoolean(), anyBoolean());
// Verify that the log-type is "ALL"
List<ContainerLogsRequest> capturedRequests =
@@ -839,9 +863,12 @@ public class TestLogsCLI {
mockYarnClient = createMockYarnClientWithException(
YarnApplicationState.RUNNING, ugi.getShortUserName());
LogsCLI cli2 = spy(new LogsCLIForTest(mockYarnClient));
+ ContainerLogsRequest newOption = mock(ContainerLogsRequest.class);
+ doReturn(newOption).when(cli2).getMatchedOptionForRunningApp(
+ any(ContainerLogsRequest.class), anyBoolean(), anyBoolean());
doReturn(0).when(cli2).printContainerLogsFromRunningApplication(
any(Configuration.class), any(ContainerLogsRequest.class),
- any(LogCLIHelpers.class), anyBoolean());
+ any(LogCLIHelpers.class), anyBoolean(), anyBoolean());
doReturn("123").when(cli2).getNodeHttpAddressFromRMWebString(
any(ContainerLogsRequest.class));
cli2.setConf(new YarnConfiguration());
@@ -851,7 +878,7 @@ public class TestLogsCLI {
assertTrue(exitCode == 0);
verify(cli2, times(1)).printContainerLogsFromRunningApplication(
any(Configuration.class), logsRequestCaptor.capture(),
- any(LogCLIHelpers.class), anyBoolean());
+ any(LogCLIHelpers.class), anyBoolean(), anyBoolean());
}
@Test (timeout = 15000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec2ae306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index 97b78ec..887d92d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -350,10 +350,10 @@ public class LogCLIHelpers implements Configurable {
}
@Private
- public Set<String> listContainerLogs(ContainerLogsRequest options)
- throws IOException {
+ public Set<ContainerLogFileInfo> listContainerLogs(
+ ContainerLogsRequest options) throws IOException {
List<ContainerLogMeta> containersLogMeta;
- Set<String> logTypes = new HashSet<String>();
+ Set<ContainerLogFileInfo> logTypes = new HashSet<ContainerLogFileInfo>();
try {
containersLogMeta = getFileController(options.getAppId(),
options.getAppOwner()).readAggregatedLogsMeta(
@@ -364,7 +364,7 @@ public class LogCLIHelpers implements Configurable {
}
for (ContainerLogMeta logMeta: containersLogMeta) {
for (ContainerLogFileInfo fileInfo : logMeta.getContainerLogMeta()) {
- logTypes.add(fileInfo.getFileName());
+ logTypes.add(fileInfo);
}
}
return logTypes;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec2ae306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
index 989b326..92e3a08 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
@@ -268,6 +268,12 @@ public class LogAggregationTFileController
}
while (nodeFiles.hasNext()) {
FileStatus thisNodeFile = nodeFiles.next();
+ if (thisNodeFile.getPath().getName().equals(appId + ".har")) {
+ Path p = new Path("har:///"
+ + thisNodeFile.getPath().toUri().getRawPath());
+ nodeFiles = HarFs.get(p.toUri(), conf).listStatusIterator(p);
+ continue;
+ }
if (nodeIdStr != null) {
if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) {
continue;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/35] hadoop git commit: HDFS-12576. JournalNodes are getting
started,
even though dfs.namenode.shared.edits.dir is not configured. Contributed by
Bharat Viswanadham.
Posted by ae...@apache.org.
HDFS-12576. JournalNodes are getting started, even though dfs.namenode.shared.edits.dir is not configured. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/015abcd8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/015abcd8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/015abcd8
Branch: refs/heads/HDFS-7240
Commit: 015abcd8ce1d360830ee2960f1cdc743e09f1629
Parents: 27ffd43
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Oct 2 15:40:00 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Oct 2 15:40:00 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh | 2 +-
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/015abcd8/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
index 12d5209..dbdf41e 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
@@ -146,7 +146,7 @@ fi
JOURNAL_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -journalNodes 2>&-)
-if [[ "{$JOURNAL_NODES-}" != $'\n' ]]; then
+if [[ "${#JOURNAL_NODES}" != 0 ]]; then
echo "Starting journal nodes [${JOURNAL_NODES}]"
hadoop_uservar_su hdfs journalnode "${HADOOP_HDFS_HOME}/bin/hdfs" \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/015abcd8/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
index 92974de..3da146e 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
@@ -102,7 +102,7 @@ fi
JOURNAL_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -journalNodes 2>&-)
-if [[ "{$JOURNAL_NODES-}" != $'\n' ]]; then
+if [[ "${#JOURNAL_NODES}" != 0 ]]; then
echo "Stopping journal nodes [${JOURNAL_NODES}]"
hadoop_uservar_su hdfs journalnode "${HADOOP_HDFS_HOME}/bin/hdfs" \
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org