You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2016/07/19 21:41:01 UTC
[01/50] hadoop git commit: YARN-5359. FileSystemTimelineReader/Writer
uses unix-specific default storage path (Sangjin Lee via Varun Saxena)
Repository: hadoop
Updated Branches:
refs/heads/YARN-2915 41b006e03 -> 85f4b8623
YARN-5359. FileSystemTimelineReader/Writer uses unix-specific default storage path (Sangjin Lee via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee086e70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee086e70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee086e70
Branch: refs/heads/YARN-2915
Commit: ee086e70db97485e0217f0019a190cfbb0524811
Parents: f2956e8
Author: Varun Saxena <va...@apache.org>
Authored: Wed Jul 13 21:15:05 2016 +0530
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:29 2016 -0700
----------------------------------------------------------------------
.../TestTimelineServiceClientIntegration.java | 8 ++++
.../storage/FileSystemTimelineReaderImpl.java | 28 +++++++-------
.../storage/FileSystemTimelineWriterImpl.java | 16 ++++----
.../reader/TestTimelineReaderWebServices.java | 12 +++++-
.../TestFileSystemTimelineReaderImpl.java | 39 ++++++++++++--------
.../TestFileSystemTimelineWriterImpl.java | 36 ++++++++++--------
6 files changed, 85 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee086e70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index 5a63547..3ec222f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -24,8 +24,10 @@ import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.io.File;
import java.io.IOException;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;
@@ -56,6 +58,9 @@ import org.junit.BeforeClass;
import org.junit.Test;
public class TestTimelineServiceClientIntegration {
+ private static final String ROOT_DIR = new File("target",
+ TestTimelineServiceClientIntegration.class.getSimpleName()).
+ getAbsolutePath();
private static NodeTimelineCollectorManager collectorManager;
private static PerNodeTimelineCollectorsAuxService auxService;
private static Configuration conf;
@@ -70,6 +75,8 @@ public class TestTimelineServiceClientIntegration {
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
FileSystemTimelineWriterImpl.class, TimelineWriter.class);
+ conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+ ROOT_DIR);
auxService =
PerNodeTimelineCollectorsAuxService.launchServer(new String[0],
collectorManager, conf);
@@ -84,6 +91,7 @@ public class TestTimelineServiceClientIntegration {
if (auxService != null) {
auxService.stop();
}
+ FileUtils.deleteDirectory(new File(ROOT_DIR));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee086e70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index 047f401..ebb73b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -77,15 +77,12 @@ public class FileSystemTimelineReaderImpl extends AbstractService
/** Default extension for output files. */
static final String APP_FLOW_MAPPING_FILE = "app_flow_mapping.csv";
- @VisibleForTesting
/** Config param for timeline service file system storage root. */
- static final String TIMELINE_SERVICE_STORAGE_DIR_ROOT =
+ public static final String TIMELINE_SERVICE_STORAGE_DIR_ROOT =
YarnConfiguration.TIMELINE_SERVICE_PREFIX + "fs-writer.root-dir";
- @VisibleForTesting
/** Default value for storage location on local disk. */
- static final String DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT =
- "/tmp/timeline_service_data";
+ private static final String STORAGE_DIR_ROOT = "timeline_service_data";
private final CSVFormat csvFormat =
CSVFormat.DEFAULT.withHeader("APP", "USER", "FLOW", "FLOWRUN");
@@ -159,13 +156,13 @@ public class FileSystemTimelineReaderImpl extends AbstractService
private String getFlowRunPath(String userId, String clusterId,
String flowName, Long flowRunId, String appId) throws IOException {
if (userId != null && flowName != null && flowRunId != null) {
- return userId + "/" + flowName + "/" + flowRunId;
+ return userId + File.separator + flowName + File.separator + flowRunId;
}
if (clusterId == null || appId == null) {
throw new IOException("Unable to get flow info");
}
- String appFlowMappingFile = rootPath + "/" + ENTITIES_DIR + "/" +
- clusterId + "/" + APP_FLOW_MAPPING_FILE;
+ String appFlowMappingFile = rootPath + File.separator + ENTITIES_DIR +
+ File.separator + clusterId + File.separator + APP_FLOW_MAPPING_FILE;
try (BufferedReader reader =
new BufferedReader(new InputStreamReader(
new FileInputStream(
@@ -180,8 +177,8 @@ public class FileSystemTimelineReaderImpl extends AbstractService
!applicationId.trim().equals(appId)) {
continue;
}
- return record.get(1).trim() + "/" + record.get(2).trim() + "/" +
- record.get(3).trim();
+ return record.get(1).trim() + File.separator + record.get(2).trim() +
+ File.separator + record.get(3).trim();
}
parser.close();
}
@@ -364,7 +361,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
@Override
public void serviceInit(Configuration conf) throws Exception {
rootPath = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT,
- DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT);
+ conf.get("hadoop.tmp.dir") + File.separator + STORAGE_DIR_ROOT);
super.serviceInit(conf);
}
@@ -375,8 +372,8 @@ public class FileSystemTimelineReaderImpl extends AbstractService
context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
context.getAppId());
File dir = new File(new File(rootPath, ENTITIES_DIR),
- context.getClusterId() + "/" + flowRunPath + "/" + context.getAppId() +
- "/" + context.getEntityType());
+ context.getClusterId() + File.separator + flowRunPath + File.separator +
+ context.getAppId() + File.separator + context.getEntityType());
File entityFile = new File(
dir, context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION);
try (BufferedReader reader =
@@ -401,8 +398,9 @@ public class FileSystemTimelineReaderImpl extends AbstractService
context.getAppId());
File dir =
new File(new File(rootPath, ENTITIES_DIR),
- context.getClusterId() + "/" + flowRunPath + "/" +
- context.getAppId() + "/" + context.getEntityType());
+ context.getClusterId() + File.separator + flowRunPath +
+ File.separator + context.getAppId() + File.separator +
+ context.getEntityType());
return getEntities(dir, context.getEntityType(), filters, dataToRetrieve);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee086e70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index 1bb77a0..1f527f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -36,6 +36,8 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* This implements a local file based backend for storing application timeline
* information. This implementation may not provide a complete implementation of
@@ -53,15 +55,14 @@ public class FileSystemTimelineWriterImpl extends AbstractService
public static final String TIMELINE_SERVICE_STORAGE_DIR_ROOT
= YarnConfiguration.TIMELINE_SERVICE_PREFIX + "fs-writer.root-dir";
- /** default value for storage location on local disk. */
- public static final String DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
- = "/tmp/timeline_service_data";
-
public static final String ENTITIES_DIR = "entities";
/** Default extension for output files. */
public static final String TIMELINE_SERVICE_STORAGE_EXTENSION = ".thist";
+ /** default value for storage location on local disk. */
+ private static final String STORAGE_DIR_ROOT = "timeline_service_data";
+
FileSystemTimelineWriterImpl() {
super((FileSystemTimelineWriterImpl.class.getName()));
}
@@ -117,14 +118,15 @@ public class FileSystemTimelineWriterImpl extends AbstractService
}
- public String getOutputRoot() {
+ @VisibleForTesting
+ String getOutputRoot() {
return outputRoot;
}
@Override
public void serviceInit(Configuration conf) throws Exception {
outputRoot = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT,
- DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT);
+ conf.get("hadoop.tmp.dir") + File.separator + STORAGE_DIR_ROOT);
}
@Override
@@ -140,7 +142,7 @@ public class FileSystemTimelineWriterImpl extends AbstractService
private static String mkdirs(String... dirStrs) throws IOException {
StringBuilder path = new StringBuilder();
for (String dirStr : dirStrs) {
- path.append(dirStr).append('/');
+ path.append(dirStr).append(File.separatorChar);
File dir = new File(path.toString());
if (!dir.exists()) {
if (!dir.mkdirs()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee086e70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
index 4ade024..a5ef66c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import java.io.File;
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
@@ -32,6 +33,7 @@ import java.util.Set;
import javax.ws.rs.core.MediaType;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
@@ -57,17 +59,21 @@ import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
public class TestTimelineReaderWebServices {
+
+ private static final String ROOT_DIR = new File("target",
+ TestTimelineReaderWebServices.class.getSimpleName()).getAbsolutePath();
+
private int serverPort;
private TimelineReaderServer server;
@BeforeClass
public static void setup() throws Exception {
- TestFileSystemTimelineReaderImpl.setup();
+ TestFileSystemTimelineReaderImpl.initializeDataDirectory(ROOT_DIR);
}
@AfterClass
public static void tearDown() throws Exception {
- TestFileSystemTimelineReaderImpl.tearDown();
+ FileUtils.deleteDirectory(new File(ROOT_DIR));
}
@Before
@@ -81,6 +87,8 @@ public class TestTimelineReaderWebServices {
config.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
config.setClass(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
FileSystemTimelineReaderImpl.class, TimelineReader.class);
+ config.set(FileSystemTimelineReaderImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+ ROOT_DIR);
server = new TimelineReaderServer();
server.init(config);
server.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee086e70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
index b58bbe3..90f11a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
@@ -43,11 +43,11 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilter
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.AfterClass;
@@ -58,17 +58,22 @@ import org.junit.Test;
public class TestFileSystemTimelineReaderImpl {
- private static final String ROOT_DIR =
- FileSystemTimelineReaderImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT;
+ private static final String ROOT_DIR = new File("target",
+ TestFileSystemTimelineReaderImpl.class.getSimpleName()).getAbsolutePath();
private FileSystemTimelineReaderImpl reader;
@BeforeClass
public static void setup() throws Exception {
- loadEntityData();
+ initializeDataDirectory(ROOT_DIR);
+ }
+
+ public static void initializeDataDirectory(String rootDir) throws Exception {
+ loadEntityData(rootDir);
// Create app flow mapping file.
CSVFormat format =
CSVFormat.DEFAULT.withHeader("APP", "USER", "FLOW", "FLOWRUN");
- String appFlowMappingFile = ROOT_DIR + "/entities/cluster1/" +
+ String appFlowMappingFile = rootDir + File.separator + "entities" +
+ File.separator + "cluster1" + File.separator +
FileSystemTimelineReaderImpl.APP_FLOW_MAPPING_FILE;
try (PrintWriter out =
new PrintWriter(new BufferedWriter(
@@ -78,7 +83,7 @@ public class TestFileSystemTimelineReaderImpl {
printer.printRecord("app2", "user1", "flow1,flow", 1);
printer.close();
}
- (new File(ROOT_DIR)).deleteOnExit();
+ (new File(rootDir)).deleteOnExit();
}
@AfterClass
@@ -102,7 +107,8 @@ public class TestFileSystemTimelineReaderImpl {
throw new IOException("Could not create directories for " + dir);
}
}
- String fileName = dir.getAbsolutePath() + "/" + entity.getId() + ".thist";
+ String fileName = dir.getAbsolutePath() + File.separator + entity.getId() +
+ ".thist";
try (PrintWriter out =
new PrintWriter(new BufferedWriter(new FileWriter(fileName, true)))){
out.println(TimelineUtils.dumpTimelineRecordtoJSON(entity));
@@ -111,9 +117,8 @@ public class TestFileSystemTimelineReaderImpl {
}
}
- private static void loadEntityData() throws Exception {
- File appDir = new File(ROOT_DIR +
- "/entities/cluster1/user1/flow1/1/app1/app/");
+ private static void loadEntityData(String rootDir) throws Exception {
+ File appDir = getAppDir(rootDir, "cluster1", "user1", "flow1", "1", "app1");
TimelineEntity entity11 = new TimelineEntity();
entity11.setId("id_1");
entity11.setType("app");
@@ -254,8 +259,8 @@ public class TestFileSystemTimelineReaderImpl {
entity4.addEvent(event44);
writeEntityFile(entity4, appDir);
- File appDir2 = new File(ROOT_DIR +
- "/entities/cluster1/user1/flow1,flow/1/app2/app/");
+ File appDir2 =
+ getAppDir(rootDir, "cluster1", "user1", "flow1,flow", "1", "app2");
TimelineEntity entity5 = new TimelineEntity();
entity5.setId("id_5");
entity5.setType("app");
@@ -263,8 +268,12 @@ public class TestFileSystemTimelineReaderImpl {
writeEntityFile(entity5, appDir2);
}
- public TimelineReader getTimelineReader() {
- return reader;
+ private static File getAppDir(String rootDir, String cluster, String user,
+ String flowName, String flowRunId, String appId) {
+ return new File(rootDir + File.separator + "entities" + File.separator +
+ cluster + File.separator + user + File.separator + flowName +
+ File.separator + flowRunId + File.separator + appId + File.separator +
+ "app" + File.separator);
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee086e70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
index 15be494..4f12c57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
@@ -29,16 +29,20 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
public class TestFileSystemTimelineWriterImpl {
+ @Rule
+ public TemporaryFolder tmpFolder = new TemporaryFolder();
/**
* Unit test for PoC YARN 3264.
@@ -79,14 +83,20 @@ public class TestFileSystemTimelineWriterImpl {
FileSystemTimelineWriterImpl fsi = null;
try {
fsi = new FileSystemTimelineWriterImpl();
- fsi.init(new YarnConfiguration());
+ Configuration conf = new YarnConfiguration();
+ String outputRoot = tmpFolder.newFolder().getAbsolutePath();
+ conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+ outputRoot);
+ fsi.init(conf);
fsi.start();
fsi.write("cluster_id", "user_id", "flow_name", "flow_version", 12345678L,
"app_id", te);
- String fileName = fsi.getOutputRoot() +
- "/entities/cluster_id/user_id/flow_name/flow_version/12345678/" +
- "app_id/" + type + "/" + id +
+ String fileName = fsi.getOutputRoot() + File.separator + "entities" +
+ File.separator + "cluster_id" + File.separator + "user_id" +
+ File.separator + "flow_name" + File.separator + "flow_version" +
+ File.separator + "12345678" + File.separator + "app_id" +
+ File.separator + type + File.separator + id +
FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
Path path = Paths.get(fileName);
File f = new File(fileName);
@@ -99,9 +109,11 @@ public class TestFileSystemTimelineWriterImpl {
assertEquals(d, TimelineUtils.dumpTimelineRecordtoJSON(entity));
// verify aggregated metrics
- String fileName2 = fsi.getOutputRoot() +
- "/entities/cluster_id/user_id/flow_name/flow_version/12345678/app_id/"
- + type2 + "/" + id2 +
+ String fileName2 = fsi.getOutputRoot() + File.separator + "entities" +
+ File.separator + "cluster_id" + File.separator + "user_id" +
+ File.separator + "flow_name" + File.separator + "flow_version" +
+ File.separator + "12345678" + File.separator + "app_id" +
+ File.separator + type2 + File.separator + id2 +
FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
Path path2 = Paths.get(fileName2);
File file = new File(fileName2);
@@ -113,15 +125,9 @@ public class TestFileSystemTimelineWriterImpl {
// confirm the contents same as what was written
assertEquals(metricToString,
TimelineUtils.dumpTimelineRecordtoJSON(entity2));
-
- // delete the directory
- File outputDir = new File(fsi.getOutputRoot());
- FileUtils.deleteDirectory(outputDir);
- assertTrue(!(f.exists()));
} finally {
if (fsi != null) {
fsi.close();
- FileUtils.deleteDirectory(new File(fsi.getOutputRoot()));
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] hadoop git commit: YARN-5339. Fixed "yarn logs" to fail when
a file is passed to -out option instead of a directory. Contributed by Xuan
Gong.
Posted by su...@apache.org.
YARN-5339. Fixed "yarn logs" to fail when a file is passed to -out option instead of a directory. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b697baa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b697baa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b697baa
Branch: refs/heads/YARN-2915
Commit: 9b697baa8daa51ab8315481f77a6401f6a67277d
Parents: e0eaa93
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Wed Jul 13 14:52:30 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 10 +++
.../hadoop/yarn/client/cli/TestLogsCLI.java | 65 +++++++++++++++++---
2 files changed, 68 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b697baa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 31395be..1560557 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.client.cli;
+import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
@@ -197,6 +198,15 @@ public class LogsCLI extends Configured implements Tool {
return -1;
}
+ if (localDir != null) {
+ File file = new File(localDir);
+ if (file.exists() && file.isFile()) {
+ System.err.println("Invalid value for -out option. "
+ + "Please provide a directory.");
+ return -1;
+ }
+ }
+
LogCLIHelpers logCliHelper = new LogCLIHelpers();
logCliHelper.setConf(getConf());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b697baa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index cb9a663..3fe53d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -711,6 +711,64 @@ public class TestLogsCLI {
}
}
+ @Test (timeout = 5000)
+ public void testLogsCLIWithInvalidArgs() throws Exception {
+ String localDir = "target/SaveLogs";
+ Path localPath = new Path(localDir);
+ Configuration configuration = new Configuration();
+ FileSystem fs = FileSystem.get(configuration);
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ YarnClient mockYarnClient =
+ createMockYarnClient(YarnApplicationState.FINISHED,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+ LogsCLI cli = new LogsCLIForTest(mockYarnClient);
+ cli.setConf(configuration);
+
+ // Specify an invalid applicationId
+ int exitCode = cli.run(new String[] {"-applicationId",
+ "123"});
+ assertTrue(exitCode == -1);
+ assertTrue(sysErrStream.toString().contains(
+ "Invalid ApplicationId specified"));
+ sysErrStream.reset();
+
+ // Specify an invalid containerId
+ exitCode = cli.run(new String[] {"-containerId",
+ "123"});
+ assertTrue(exitCode == -1);
+ assertTrue(sysErrStream.toString().contains(
+ "Invalid ContainerId specified"));
+ sysErrStream.reset();
+
+ // Specify show_container_log_info and show_application_log_info
+ // at the same time
+ exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_container_log_info", "-show_application_log_info"});
+ assertTrue(exitCode == -1);
+ assertTrue(sysErrStream.toString().contains("Invalid options. "
+ + "Can only accept one of show_application_log_info/"
+ + "show_container_log_info."));
+ sysErrStream.reset();
+
+ // Specify a file name to the option -out
+ try {
+ fs.mkdirs(localPath);
+ Path tmpFilePath = new Path(localPath, "tmpFile");
+ if (!fs.exists(tmpFilePath)) {
+ fs.createNewFile(tmpFilePath);
+ }
+ exitCode = cli.run(new String[] {"-applicationId",
+ appId.toString(),
+ "-out" , tmpFilePath.toString()});
+ assertTrue(exitCode == -1);
+ assertTrue(sysErrStream.toString().contains(
+ "Invalid value for -out option. Please provide a directory."));
+ } finally {
+ fs.delete(localPath, true);
+ }
+ }
+
+
@Test (timeout = 15000)
public void testSaveContainerLogsLocally() throws Exception {
String remoteLogRootDir = "target/logs/";
@@ -845,13 +903,6 @@ public class TestLogsCLI {
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration);
- int result = cli.run(new String[] {"-applicationId", appId.toString(),
- "-show_container_log_info", "-show_application_log_info"});
- assertTrue(result == -1);
- assertTrue(sysErrStream.toString().contains("Invalid options. "
- + "Can only accept one of show_application_log_info/"
- + "show_container_log_info."));
-
cli.run(new String[] {"-applicationId", appId.toString(),
"-show_container_log_info"});
assertTrue(sysOutStream.toString().contains(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] hadoop git commit: YARN-5303. Clean up ContainerExecutor
JavaDoc. Contributed by Daniel Templeton.
Posted by su...@apache.org.
YARN-5303. Clean up ContainerExecutor JavaDoc. Contributed by Daniel Templeton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ba99e69
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ba99e69
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ba99e69
Branch: refs/heads/YARN-2915
Commit: 1ba99e694ec6840ce73f36400d395f03ac6b3bfa
Parents: aedeabc
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu Jul 14 19:28:11 2016 +0530
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 444 +++++++++++++------
.../nodemanager/DefaultContainerExecutor.java | 1 -
.../nodemanager/DockerContainerExecutor.java | 1 -
.../nodemanager/LinuxContainerExecutor.java | 3 +-
.../WindowsSecureContainerExecutor.java | 12 +-
.../launcher/ContainerLaunch.java | 3 +-
6 files changed, 310 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba99e69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 98d45f4..2689480 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -60,20 +60,31 @@ import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
+/**
+ * This class is abstraction of the mechanism used to launch a container on the
+ * underlying OS. All executor implementations must extend ContainerExecutor.
+ */
public abstract class ContainerExecutor implements Configurable {
private static final String WILDCARD = "*";
private static final Log LOG = LogFactory.getLog(ContainerExecutor.class);
- final public static FsPermission TASK_LAUNCH_SCRIPT_PERMISSION =
- FsPermission.createImmutable((short) 0700);
+ /**
+ * The permissions to use when creating the launch script.
+ */
+ public static final FsPermission TASK_LAUNCH_SCRIPT_PERMISSION =
+ FsPermission.createImmutable((short)0700);
+
+ /**
+ * The relative path to which debug information will be written.
+ *
+ * @see ContainerLaunch.ShellScriptBuilder#listDebugInformation
+ */
public static final String DIRECTORY_CONTENTS = "directory.info";
private Configuration conf;
-
- private ConcurrentMap<ContainerId, Path> pidFiles =
- new ConcurrentHashMap<ContainerId, Path>();
-
- private ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ private final ConcurrentMap<ContainerId, Path> pidFiles =
+ new ConcurrentHashMap<>();
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
private final ReadLock readLock = lock.readLock();
private final WriteLock writeLock = lock.writeLock();
@@ -88,30 +99,34 @@ public abstract class ContainerExecutor implements Configurable {
}
/**
- * Run the executor initialization steps.
- * Verify that the necessary configs, permissions are in place.
- * @throws IOException
+ * Run the executor initialization steps.
+ * Verify that the necessary configs and permissions are in place.
+ *
+ * @throws IOException if initialization fails
*/
public abstract void init() throws IOException;
/**
- * On Windows the ContainerLaunch creates a temporary special jar manifest of
- * other jars to workaround the CLASSPATH length. In a secure cluster this
- * jar must be localized so that the container has access to it.
- * This function localizes on-demand the jar.
- *
- * @param classPathJar
- * @param owner
- * @throws IOException
+ * This function localizes the JAR file on-demand.
+ * On Windows the ContainerLaunch creates a temporary special JAR manifest of
+ * other JARs to workaround the CLASSPATH length. In a secure cluster this
+ * JAR must be localized so that the container has access to it.
+ * The default implementation returns the classpath passed to it, which
+ * is expected to have been created in the node manager's <i>fprivate</i>
+ * folder, which will not work with secure Windows clusters.
+ *
+ * @param jarPath the path to the JAR to localize
+ * @param target the directory where the JAR file should be localized
+ * @param owner the name of the user who should own the localized file
+ * @return the path to the localized JAR file
+ * @throws IOException if localization fails
*/
- public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner)
+ public Path localizeClasspathJar(Path jarPath, Path target, String owner)
throws IOException {
- // Non-secure executor simply use the classpath created
- // in the NM fprivate folder
- return classPathJar;
+ return jarPath;
}
-
-
+
+
/**
* Prepare the environment for containers in this application to execute.
* <pre>
@@ -123,10 +138,11 @@ public abstract class ContainerExecutor implements Configurable {
* For $rsrc in job resources
* Copy $rsrc {@literal ->} $N/$user/$appId/filecache/idef
* </pre>
+ *
* @param ctx LocalizerStartContext that encapsulates necessary information
* for starting a localizer.
- * @throws IOException For most application init failures
- * @throws InterruptedException If application init thread is halted by NM
+ * @throws IOException for most application init failures
+ * @throws InterruptedException if application init thread is halted by NM
*/
public abstract void startLocalizer(LocalizerStartContext ctx)
throws IOException, InterruptedException;
@@ -137,25 +153,28 @@ public abstract class ContainerExecutor implements Configurable {
* when the container exits.
* @param ctx Encapsulates information necessary for launching containers.
* @return the return status of the launch
- * @throws IOException
+ * @throws IOException if the container launch fails
*/
public abstract int launchContainer(ContainerStartContext ctx) throws
IOException;
/**
* Signal container with the specified signal.
+ *
* @param ctx Encapsulates information necessary for signaling containers.
* @return returns true if the operation succeeded
- * @throws IOException
+ * @throws IOException if signaling the container fails
*/
public abstract boolean signalContainer(ContainerSignalContext ctx)
throws IOException;
/**
* Delete specified directories as a given user.
+ *
* @param ctx Encapsulates information necessary for deletion.
- * @throws IOException
- * @throws InterruptedException
+ * @throws IOException if delete fails
+ * @throws InterruptedException if interrupted while waiting for the deletion
+ * operation to complete
*/
public abstract void deleteAsUser(DeletionAsUserContext ctx)
throws IOException, InterruptedException;
@@ -164,7 +183,8 @@ public abstract class ContainerExecutor implements Configurable {
* Check if a container is alive.
* @param ctx Encapsulates information necessary for container liveness check.
* @return true if container is still alive
- * @throws IOException
+ * @throws IOException if there is a failure while checking the container
+ * status
*/
public abstract boolean isContainerAlive(ContainerLivenessContext ctx)
throws IOException;
@@ -173,56 +193,63 @@ public abstract class ContainerExecutor implements Configurable {
* Recover an already existing container. This is a blocking call and returns
* only when the container exits. Note that the container must have been
* activated prior to this call.
+ *
* @param ctx encapsulates information necessary to reacquire container
* @return The exit code of the pre-existing container
- * @throws IOException
- * @throws InterruptedException
+ * @throws IOException if there is a failure while reacquiring the container
+ * @throws InterruptedException if interrupted while waiting to reacquire
+ * the container
*/
public int reacquireContainer(ContainerReacquisitionContext ctx)
throws IOException, InterruptedException {
Container container = ctx.getContainer();
String user = ctx.getUser();
ContainerId containerId = ctx.getContainerId();
-
-
Path pidPath = getPidFilePath(containerId);
+
if (pidPath == null) {
LOG.warn(containerId + " is not active, returning terminated error");
+
return ExitCode.TERMINATED.getExitCode();
}
- String pid = null;
- pid = ProcessIdFileReader.getProcessId(pidPath);
+ String pid = ProcessIdFileReader.getProcessId(pidPath);
+
if (pid == null) {
throw new IOException("Unable to determine pid for " + containerId);
}
LOG.info("Reacquiring " + containerId + " with pid " + pid);
+
ContainerLivenessContext livenessContext = new ContainerLivenessContext
.Builder()
.setContainer(container)
.setUser(user)
.setPid(pid)
.build();
- while(isContainerAlive(livenessContext)) {
+
+ while (isContainerAlive(livenessContext)) {
Thread.sleep(1000);
}
// wait for exit code file to appear
- String exitCodeFile = ContainerLaunch.getExitCodeFile(pidPath.toString());
- File file = new File(exitCodeFile);
final int sleepMsec = 100;
int msecLeft = 2000;
+ String exitCodeFile = ContainerLaunch.getExitCodeFile(pidPath.toString());
+ File file = new File(exitCodeFile);
+
while (!file.exists() && msecLeft >= 0) {
if (!isContainerActive(containerId)) {
LOG.info(containerId + " was deactivated");
+
return ExitCode.TERMINATED.getExitCode();
}
-
+
Thread.sleep(sleepMsec);
-
+
msecLeft -= sleepMsec;
}
+
if (msecLeft < 0) {
throw new IOException("Timeout while waiting for exit code from "
+ containerId);
@@ -236,15 +263,17 @@ public abstract class ContainerExecutor implements Configurable {
}
/**
- * This method writes out the launch environment of a container. This can be
- * overridden by extending ContainerExecutors to provide different behaviors
+ * This method writes out the launch environment of a container to the
+ * default container launch script. For the default container script path see
+ * {@link ContainerLaunch#CONTAINER_SCRIPT}.
+ *
* @param out the output stream to which the environment is written (usually
* a script file which will be executed by the Launcher)
- * @param environment The environment variables and their values
- * @param resources The resources which have been localized for this container
- * Symlinks will be created to these localized resources
- * @param command The command that will be run.
- * @param logDir The log dir to copy debugging information to
+ * @param environment the environment variables and their values
+ * @param resources the resources which have been localized for this
+ * container. Symlinks will be created to these localized resources
+ * @param command the command that will be run.
+ * @param logDir the log dir to copy debugging information to
* @throws IOException if any errors happened writing to the OutputStream,
* while creating symlinks
*/
@@ -255,6 +284,21 @@ public abstract class ContainerExecutor implements Configurable {
ContainerLaunch.CONTAINER_SCRIPT);
}
+ /**
+ * This method writes out the launch environment of a container to a specified
+ * path.
+ *
+ * @param out the output stream to which the environment is written (usually
+ * a script file which will be executed by the Launcher)
+ * @param environment the environment variables and their values
+ * @param resources the resources which have been localized for this
+ * container. Symlinks will be created to these localized resources
+ * @param command the command that will be run.
+ * @param logDir the log dir to copy debugging information to
+ * @param outFilename the path to which to write the launch environment
+ * @throws IOException if any errors happened writing to the OutputStream,
+ * while creating symlinks
+ */
@VisibleForTesting
public void writeLaunchEnv(OutputStream out,
Map<String, String> environment, Map<Path, List<String>> resources,
@@ -262,52 +306,57 @@ public abstract class ContainerExecutor implements Configurable {
throws IOException {
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
- Set<String> whitelist = new HashSet<String>();
+ Set<String> whitelist = new HashSet<>();
+
whitelist.add(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME);
whitelist.add(ApplicationConstants.Environment.HADOOP_YARN_HOME.name());
whitelist.add(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name());
whitelist.add(ApplicationConstants.Environment.HADOOP_HDFS_HOME.name());
whitelist.add(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
whitelist.add(ApplicationConstants.Environment.JAVA_HOME.name());
+
if (environment != null) {
for (Map.Entry<String,String> env : environment.entrySet()) {
if (!whitelist.contains(env.getKey())) {
- sb.env(env.getKey().toString(), env.getValue().toString());
+ sb.env(env.getKey(), env.getValue());
} else {
- sb.whitelistedEnv(env.getKey().toString(), env.getValue().toString());
+ sb.whitelistedEnv(env.getKey(), env.getValue());
}
}
}
+
if (resources != null) {
- for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
- for (String linkName : entry.getValue()) {
+ for (Path path: resources.keySet()) {
+ for (String linkName: resources.get(path)) {
if (new Path(linkName).getName().equals(WILDCARD)) {
// If this is a wildcarded path, link to everything in the
// directory from the working directory
- File directory = new File(entry.getKey().toString());
+ File directory = new File(path.toString());
for (File wildLink : directory.listFiles()) {
sb.symlink(new Path(wildLink.toString()),
new Path(wildLink.getName()));
}
} else {
- sb.symlink(entry.getKey(), new Path(linkName));
+ sb.symlink(path, new Path(linkName));
}
}
}
}
// dump debugging information if configured
- if (getConf() != null && getConf().getBoolean(
- YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO,
+ if (getConf() != null &&
+ getConf().getBoolean(YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO,
YarnConfiguration.DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO)) {
- sb.copyDebugInformation(new Path(outFilename), new Path(logDir, outFilename));
+ sb.copyDebugInformation(new Path(outFilename),
+ new Path(logDir, outFilename));
sb.listDebugInformation(new Path(logDir, DIRECTORY_CONTENTS));
}
sb.command(command);
PrintStream pout = null;
+
try {
pout = new PrintStream(out, false, "UTF-8");
sb.write(pout);
@@ -318,17 +367,25 @@ public abstract class ContainerExecutor implements Configurable {
}
}
+ /**
+ * The container exit code.
+ */
public enum ExitCode {
SUCCESS(0),
FORCE_KILLED(137),
TERMINATED(143),
LOST(154);
+
private final int code;
private ExitCode(int exitCode) {
this.code = exitCode;
}
+ /**
+ * Get the exit code as an int.
+ * @return the exit code as an int
+ */
public int getExitCode() {
return code;
}
@@ -343,25 +400,41 @@ public abstract class ContainerExecutor implements Configurable {
* The constants for the signals.
*/
public enum Signal {
- NULL(0, "NULL"), QUIT(3, "SIGQUIT"),
- KILL(9, "SIGKILL"), TERM(15, "SIGTERM");
+ NULL(0, "NULL"),
+ QUIT(3, "SIGQUIT"),
+ KILL(9, "SIGKILL"),
+ TERM(15, "SIGTERM");
+
private final int value;
private final String str;
+
private Signal(int value, String str) {
this.str = str;
this.value = value;
}
+
+ /**
+ * Get the signal number.
+ * @return the signal number
+ */
public int getValue() {
return value;
}
+
@Override
public String toString() {
return str;
}
}
+ /**
+ * Log each line of the output string as INFO level log messages.
+ *
+ * @param output the output string to log
+ */
protected void logOutput(String output) {
String shExecOutput = output;
+
if (shExecOutput != null) {
for (String str : shExecOutput.split("\n")) {
LOG.info(str);
@@ -371,7 +444,8 @@ public abstract class ContainerExecutor implements Configurable {
/**
* Get the pidFile of the container.
- * @param containerId
+ *
+ * @param containerId the container ID
* @return the path of the pid-file for the given containerId.
*/
protected Path getPidFilePath(ContainerId containerId) {
@@ -383,81 +457,151 @@ public abstract class ContainerExecutor implements Configurable {
}
}
+ /**
+ * Return a command line to execute the given command in the OS shell.
+ * On Windows, the {code}groupId{code} parameter can be used to launch
+ * and associate the given GID with a process group. On
+ * non-Windows hosts, the {code}groupId{code} parameter is ignored.
+ *
+ * @param command the command to execute
+ * @param groupId the job owner's GID
+ * @param userName the job owner's username
+ * @param pidFile the path to the container's PID file
+ * @param conf the configuration
+ * @return the command line to execute
+ */
protected String[] getRunCommand(String command, String groupId,
String userName, Path pidFile, Configuration conf) {
return getRunCommand(command, groupId, userName, pidFile, conf, null);
}
-
- /**
- * Return a command to execute the given command in OS shell.
- * On Windows, the passed in groupId can be used to launch
- * and associate the given groupId in a process group. On
- * non-Windows, groupId is ignored.
+
+ /**
+ * Return a command line to execute the given command in the OS shell.
+ * On Windows, the {code}groupId{code} parameter can be used to launch
+ * and associate the given GID with a process group. On
+ * non-Windows hosts, the {code}groupId{code} parameter is ignored.
+ *
+ * @param command the command to execute
+ * @param groupId the job owner's GID for Windows. On other operating systems
+ * it is ignored.
+ * @param userName the job owner's username for Windows. On other operating
+ * systems it is ignored.
+ * @param pidFile the path to the container's PID file on Windows. On other
+ * operating systems it is ignored.
+ * @param conf the configuration
+ * @param resource on Windows this parameter controls memory and CPU limits.
+ * If null, no limits are set. On other operating systems it is ignored.
+ * @return the command line to execute
*/
protected String[] getRunCommand(String command, String groupId,
String userName, Path pidFile, Configuration conf, Resource resource) {
+ if (Shell.WINDOWS) {
+ return getRunCommandForWindows(command, groupId, userName, pidFile,
+ conf, resource);
+ } else {
+ return getRunCommandForOther(command, conf);
+ }
+
+ }
+
+ /**
+ * Return a command line to execute the given command in the OS shell.
+ * The {code}groupId{code} parameter can be used to launch
+ * and associate the given GID with a process group.
+ *
+ * @param command the command to execute
+ * @param groupId the job owner's GID
+ * @param userName the job owner's username
+ * @param pidFile the path to the container's PID file
+ * @param conf the configuration
+ * @param resource this parameter controls memory and CPU limits.
+ * If null, no limits are set.
+ * @return the command line to execute
+ */
+ protected String[] getRunCommandForWindows(String command, String groupId,
+ String userName, Path pidFile, Configuration conf, Resource resource) {
+ int cpuRate = -1;
+ int memory = -1;
+
+ if (resource != null) {
+ if (conf.getBoolean(
+ YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED,
+ YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
+ memory = (int)resource.getMemorySize();
+ }
+
+ if (conf.getBoolean(
+ YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED,
+ YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) {
+ int containerVCores = resource.getVirtualCores();
+ int nodeVCores = NodeManagerHardwareUtils.getVCores(conf);
+ int nodeCpuPercentage =
+ NodeManagerHardwareUtils.getNodeCpuPercentage(conf);
+
+ float containerCpuPercentage =
+ (float)(nodeCpuPercentage * containerVCores) / nodeVCores;
+
+ // CPU should be set to a percentage * 100, e.g. 20% cpu rate limit
+ // should be set as 20 * 100.
+ cpuRate = Math.min(10000, (int)(containerCpuPercentage * 100));
+ }
+ }
+
+ return new String[] {
+ Shell.getWinUtilsPath(),
+ "task",
+ "create",
+ "-m",
+ String.valueOf(memory),
+ "-c",
+ String.valueOf(cpuRate),
+ groupId,
+ "cmd /c " + command
+ };
+ }
+
+ /**
+ * Return a command line to execute the given command in the OS shell.
+ *
+ * @param command the command to execute
+ * @param conf the configuration
+ * @return the command line to execute
+ */
+ protected String[] getRunCommandForOther(String command,
+ Configuration conf) {
+ List<String> retCommand = new ArrayList<>();
boolean containerSchedPriorityIsSet = false;
- int containerSchedPriorityAdjustment =
+ int containerSchedPriorityAdjustment =
YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY;
- if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) !=
+ if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) !=
null) {
containerSchedPriorityIsSet = true;
- containerSchedPriorityAdjustment = conf
- .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY,
+ containerSchedPriorityAdjustment = conf
+ .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY,
YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
}
-
- if (Shell.WINDOWS) {
- int cpuRate = -1;
- int memory = -1;
- if (resource != null) {
- if (conf
- .getBoolean(
- YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED,
- YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
- memory = (int) resource.getMemorySize();
- }
-
- if (conf.getBoolean(
- YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED,
- YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) {
- int containerVCores = resource.getVirtualCores();
- int nodeVCores = NodeManagerHardwareUtils.getVCores(conf);
- int nodeCpuPercentage =
- NodeManagerHardwareUtils.getNodeCpuPercentage(conf);
-
- float containerCpuPercentage =
- (float) (nodeCpuPercentage * containerVCores) / nodeVCores;
- // CPU should be set to a percentage * 100, e.g. 20% cpu rate limit
- // should be set as 20 * 100.
- cpuRate = Math.min(10000, (int) (containerCpuPercentage * 100));
- }
- }
- return new String[] { Shell.getWinUtilsPath(), "task", "create", "-m",
- String.valueOf(memory), "-c", String.valueOf(cpuRate), groupId,
- "cmd /c " + command };
- } else {
- List<String> retCommand = new ArrayList<String>();
- if (containerSchedPriorityIsSet) {
- retCommand.addAll(Arrays.asList("nice", "-n",
- Integer.toString(containerSchedPriorityAdjustment)));
- }
- retCommand.addAll(Arrays.asList("bash", command));
- return retCommand.toArray(new String[retCommand.size()]);
+ if (containerSchedPriorityIsSet) {
+ retCommand.addAll(Arrays.asList("nice", "-n",
+ Integer.toString(containerSchedPriorityAdjustment)));
}
+ retCommand.addAll(Arrays.asList("bash", command));
+
+ return retCommand.toArray(new String[retCommand.size()]);
}
/**
- * Is the container still active?
- * @param containerId
- * @return true if the container is active else false.
+ * Return whether the container is still active.
+ *
+ * @param containerId the target container's ID
+ * @return true if the container is active
*/
protected boolean isContainerActive(ContainerId containerId) {
try {
readLock.lock();
+
return (this.pidFiles.containsKey(containerId));
} finally {
readLock.unlock();
@@ -465,13 +609,11 @@ public abstract class ContainerExecutor implements Configurable {
}
/**
- * Mark the container as active
- *
- * @param containerId
- * the ContainerId
- * @param pidFilePath
- * Path where the executor should write the pid of the launched
- * process
+ * Mark the container as active.
+ *
+ * @param containerId the container ID
+ * @param pidFilePath the path where the executor should write the PID
+ * of the launched process
*/
public void activateContainer(ContainerId containerId, Path pidFilePath) {
try {
@@ -483,9 +625,10 @@ public abstract class ContainerExecutor implements Configurable {
}
/**
- * Mark the container as inactive.
- * Done iff the container is still active. Else treat it as
- * a no-op
+ * Mark the container as inactive. For inactive containers this
+ * method has no effect.
+ *
+ * @param containerId the container ID
*/
public void deactivateContainer(ContainerId containerId) {
try {
@@ -497,46 +640,63 @@ public abstract class ContainerExecutor implements Configurable {
}
/**
- * Get the process-identifier for the container
- *
- * @param containerID
- * @return the processid of the container if it has already launched,
- * otherwise return null
+ * Get the process-identifier for the container.
+ *
+ * @param containerID the container ID
+ * @return the process ID of the container if it has already launched,
+ * or null otherwise
*/
public String getProcessId(ContainerId containerID) {
String pid = null;
Path pidFile = pidFiles.get(containerID);
- if (pidFile == null) {
- // This container isn't even launched yet.
- return pid;
- }
- try {
- pid = ProcessIdFileReader.getProcessId(pidFile);
- } catch (IOException e) {
- LOG.error("Got exception reading pid from pid-file " + pidFile, e);
+
+ // If PID is null, this container hasn't launched yet.
+ if (pidFile != null) {
+ try {
+ pid = ProcessIdFileReader.getProcessId(pidFile);
+ } catch (IOException e) {
+ LOG.error("Got exception reading pid from pid-file " + pidFile, e);
+ }
}
+
return pid;
}
+ /**
+ * This class will signal a target container after a specified delay.
+ * @see #signalContainer
+ */
public static class DelayedProcessKiller extends Thread {
- private Container container;
+ private final Container container;
private final String user;
private final String pid;
private final long delay;
private final Signal signal;
private final ContainerExecutor containerExecutor;
+ /**
+ * Basic constructor.
+ *
+ * @param container the container to signal
+ * @param user the user as whow to send the signal
+ * @param pid the PID of the container process
+ * @param delayMS the period of time to wait in millis before signaling
+ * the container
+ * @param signal the signal to send
+ * @param containerExecutor the executor to use to send the signal
+ */
public DelayedProcessKiller(Container container, String user, String pid,
- long delay, Signal signal, ContainerExecutor containerExecutor) {
+ long delayMS, Signal signal, ContainerExecutor containerExecutor) {
this.container = container;
this.user = user;
this.pid = pid;
- this.delay = delay;
+ this.delay = delayMS;
this.signal = signal;
this.containerExecutor = containerExecutor;
setName("Task killer for " + pid);
setDaemon(false);
}
+
@Override
public void run() {
try {
@@ -548,13 +708,13 @@ public abstract class ContainerExecutor implements Configurable {
.setSignal(signal)
.build());
} catch (InterruptedException e) {
- return;
+ interrupt();
} catch (IOException e) {
String message = "Exception when user " + user + " killing task " + pid
+ " in DelayedProcessKiller: " + StringUtils.stringifyException(e);
LOG.warn(message);
- container.handle(new ContainerDiagnosticsUpdateEvent(container
- .getContainerId(), message));
+ container.handle(new ContainerDiagnosticsUpdateEvent(
+ container.getContainerId(), message));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba99e69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index f8f19bd..13ad9ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -60,7 +60,6 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba99e69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index 2b18469..7bf6989 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -63,7 +63,6 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba99e69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index be5d078..04e38fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -324,8 +324,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
resourcesHandler.preExecute(containerId,
container.getResource());
- String resourcesOptions = resourcesHandler.getResourcesOption(
- containerId);
+ String resourcesOptions = resourcesHandler.getResourcesOption(containerId);
String tcCommandFile = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba99e69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
index 8d30744..c75ecb1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
@@ -628,16 +628,16 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor {
}
@Override
- public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner)
+ public Path localizeClasspathJar(Path jarPath, Path target, String owner)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("localizeClasspathJar: %s %s o:%s",
- classPathJar, pwd, owner));
+ jarPath, target, owner));
}
- createDir(pwd, new FsPermission(DIR_PERM), true, owner);
- String fileName = classPathJar.getName();
- Path dst = new Path(pwd, fileName);
- Native.Elevated.move(classPathJar, dst, true);
+ createDir(target, new FsPermission(DIR_PERM), true, owner);
+ String fileName = jarPath.getName();
+ Path dst = new Path(target, fileName);
+ Native.Elevated.move(jarPath, dst, true);
Native.Elevated.chown(dst, owner, nodeManagerGroup);
return dst;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba99e69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index dffe120..81b6c1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -82,7 +82,6 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -835,7 +834,7 @@ public class ContainerLaunch implements Callable<Integer> {
throws IOException;
/**
- * Method to dump debug information to the a target file. This method will
+ * Method to dump debug information to a target file. This method will
* be called by ContainerExecutor when setting up the container launch
* script.
* @param output the file to which debug information is to be written
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] hadoop git commit: Revert "HADOOP-13073 RawLocalFileSystem
does not react on changing umask. Contributed by Andras Bokor"
Posted by su...@apache.org.
Revert "HADOOP-13073 RawLocalFileSystem does not react on changing umask. Contributed by Andras Bokor"
This reverts commit 45af585e7ad23e7d5a5bc4dade00723d315ef6b2.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1bec8a5a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1bec8a5a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1bec8a5a
Branch: refs/heads/YARN-2915
Commit: 1bec8a5a321d63d11d00a521e2c4a418acb401d2
Parents: 3d8b75c
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jul 14 10:30:36 2016 +0100
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/RawLocalFileSystem.java | 7 ++-
.../fs/TestLocalFileSystemPermission.java | 56 +-------------------
2 files changed, 7 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bec8a5a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index 0fcddcf..cc41f4a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -64,6 +64,8 @@ public class RawLocalFileSystem extends FileSystem {
// Temporary workaround for HADOOP-9652.
private static boolean useDeprecatedFileStatus = true;
+ private FsPermission umask;
+
@VisibleForTesting
public static void useStatIfAvailable() {
useDeprecatedFileStatus = !Stat.isAvailable();
@@ -97,6 +99,7 @@ public class RawLocalFileSystem extends FileSystem {
public void initialize(URI uri, Configuration conf) throws IOException {
super.initialize(uri, conf);
setConf(conf);
+ umask = FsPermission.getUMask(conf);
}
/*******************************************************
@@ -230,7 +233,7 @@ public class RawLocalFileSystem extends FileSystem {
if (permission == null) {
this.fos = new FileOutputStream(file, append);
} else {
- permission = permission.applyUMask(FsPermission.getUMask(getConf()));
+ permission = permission.applyUMask(umask);
if (Shell.WINDOWS && NativeIO.isAvailable()) {
this.fos = NativeIO.Windows.createFileOutputStreamWithMode(file,
append, permission.toShort());
@@ -507,7 +510,7 @@ public class RawLocalFileSystem extends FileSystem {
if (permission == null) {
permission = FsPermission.getDirDefault();
}
- permission = permission.applyUMask(FsPermission.getUMask(getConf()));
+ permission = permission.applyUMask(umask);
if (Shell.WINDOWS && NativeIO.isAvailable()) {
try {
NativeIO.Windows.createDirectoryWithMode(p2f, permission.toShort());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bec8a5a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
index 7252ab9..e37de19 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
@@ -29,10 +29,6 @@ import java.util.*;
import junit.framework.*;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.not;
-import static org.junit.Assert.assertThat;
-
/**
* This class tests the local file system via the FileSystem abstraction.
*/
@@ -71,9 +67,9 @@ public class TestLocalFileSystemPermission extends TestCase {
System.out.println("Cannot run test for Windows");
return;
}
- LocalFileSystem localfs = FileSystem.getLocal(new Configuration());
- Configuration conf = localfs.getConf();
+ Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "044");
+ LocalFileSystem localfs = FileSystem.getLocal(conf);
Path dir = new Path(TEST_PATH_PREFIX + "dir");
localfs.mkdirs(dir);
try {
@@ -241,54 +237,6 @@ public class TestLocalFileSystemPermission extends TestCase {
finally {cleanup(localfs, f);}
}
- /**
- * Steps:
- * 1. Create a directory with default permissions: 777 with umask 022
- * 2. Check the directory has good permissions: 755
- * 3. Set the umask to 062.
- * 4. Create a new directory with default permissions.
- * 5. For this directory we expect 715 as permission not 755
- * @throws Exception we can throw away all the exception.
- */
- public void testSetUmaskInRealTime() throws Exception {
- if (Path.WINDOWS) {
- System.out.println("Cannot run test for Windows");
- return;
- }
-
- LocalFileSystem localfs = FileSystem.getLocal(new Configuration());
- Configuration conf = localfs.getConf();
- conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "022");
- System.out.println(
- conf.get(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY));
- Path dir = new Path(TEST_PATH_PREFIX + "dir");
- Path dir2 = new Path(TEST_PATH_PREFIX + "dir2");
- try {
- assertTrue(localfs.mkdirs(dir));
- FsPermission initialPermission = getPermission(localfs, dir);
- assertEquals(
- "With umask 022 permission should be 755 since the default " +
- "permission is 777", new FsPermission("755"), initialPermission);
-
- // Modify umask and create a new directory
- // and check if new umask is applied
- conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "062");
- assertTrue(localfs.mkdirs(dir2));
- FsPermission finalPermission = localfs.getFileStatus(dir2)
- .getPermission();
- assertThat("With umask 062 permission should not be 755 since the " +
- "default permission is 777", new FsPermission("755"),
- is(not(finalPermission)));
- assertEquals(
- "With umask 062 we expect 715 since the default permission is 777",
- new FsPermission("715"), finalPermission);
- } finally {
- conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "022");
- cleanup(localfs, dir);
- cleanup(localfs, dir2);
- }
- }
-
static List<String> getGroups() throws IOException {
List<String> a = new ArrayList<String>();
String s = Shell.execCommand(Shell.getGroupsCommand());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] hadoop git commit: MAPREDUCE-6652. Add configuration property
to prevent JHS from loading jobs with a task count greater than X (haibochen
via rkanter)
Posted by su...@apache.org.
MAPREDUCE-6652. Add configuration property to prevent JHS from loading jobs with a task count greater than X (haibochen via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b390dd6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b390dd6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b390dd6e
Branch: refs/heads/YARN-2915
Commit: b390dd6e5ee57a4e4546713a2b8df2036be4a16e
Parents: 17c74ac
Author: Robert Kanter <rk...@apache.org>
Authored: Fri Jul 15 13:30:03 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:31 2016 -0700
----------------------------------------------------------------------
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 7 +
.../src/main/resources/mapred-default.xml | 9 +
.../mapreduce/v2/hs/HistoryFileManager.java | 35 ++-
.../hadoop/mapreduce/v2/hs/UnparsedJob.java | 211 +++++++++++++++
.../mapreduce/v2/hs/webapp/HsJobBlock.java | 16 +-
.../mapreduce/v2/hs/TestHistoryFileManager.java | 98 ++++++-
.../mapreduce/v2/hs/webapp/TestHsJobBlock.java | 268 +++++++++++++++++++
7 files changed, 634 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b390dd6e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index 1c79671..5097946 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -253,4 +253,11 @@ public class JHAdminConfig {
MR_HISTORY_PREFIX + "jhist.format";
public static final String DEFAULT_MR_HS_JHIST_FORMAT =
"binary";
+
+ /**
+ * The maximum number of tasks for a job to be loaded in Job History Server.
+ */
+ public static final String MR_HS_LOADED_JOBS_TASKS_MAX =
+ MR_HISTORY_PREFIX + "loadedjob.tasks.max";
+ public static final int DEFAULT_MR_HS_LOADED_JOBS_TASKS_MAX = -1;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b390dd6e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index ebc43aa..1741d88 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1923,4 +1923,13 @@
<value>SAMEORIGIN</value>
</property>
+<property>
+ <description>
+ The maximum number of tasks that a job can have so that the Job History
+ Server will fully parse its associated job history file and load it into
+ memory. A value of -1 (default) will allow all jobs to be loaded.
+ </description>
+ <name>mapreduce.jobhistory.loadedjob.tasks.max</name>
+ <value>-1</value>
+</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b390dd6e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index a1f4014..12e7a27 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -458,15 +458,23 @@ public class HistoryFileManager extends AbstractService {
/**
* Parse a job from the JobHistoryFile, if the underlying file is not going
- * to be deleted.
+ * to be deleted and the number of tasks associated with the job is not
+ * greater than maxTasksForLoadedJob.
*
- * @return the Job or null if the underlying file was deleted.
+ * @return null if the underlying job history file was deleted, or
+ * an {@link UnparsedJob} object representing a partially parsed job
+ * if the job tasks exceeds the configured maximum, or
+ * a {@link CompletedJob} representing a fully parsed job.
* @throws IOException
- * if there is an error trying to read the file.
+ * if there is an error trying to read the file if parsed.
*/
public synchronized Job loadJob() throws IOException {
- return new CompletedJob(conf, jobIndexInfo.getJobId(), historyFile,
- false, jobIndexInfo.getUser(), this, aclsMgr);
+ if(isOversized()) {
+ return new UnparsedJob(maxTasksForLoadedJob, jobIndexInfo, this);
+ } else {
+ return new CompletedJob(conf, jobIndexInfo.getJobId(), historyFile,
+ false, jobIndexInfo.getUser(), this, aclsMgr);
+ }
}
/**
@@ -504,6 +512,12 @@ public class HistoryFileManager extends AbstractService {
jobConf.addResource(fc.open(confFile), confFile.toString());
return jobConf;
}
+
+ private boolean isOversized() {
+ final int totalTasks = jobIndexInfo.getNumReduces() +
+ jobIndexInfo.getNumMaps();
+ return (maxTasksForLoadedJob > 0) && (totalTasks > maxTasksForLoadedJob);
+ }
}
private SerialNumberIndex serialNumberIndex = null;
@@ -536,7 +550,12 @@ public class HistoryFileManager extends AbstractService {
@VisibleForTesting
protected ThreadPoolExecutor moveToDoneExecutor = null;
private long maxHistoryAge = 0;
-
+
+ /**
+ * The maximum number of tasks allowed for a job to be loaded.
+ */
+ private int maxTasksForLoadedJob = -1;
+
public HistoryFileManager() {
super(HistoryFileManager.class.getName());
}
@@ -555,6 +574,10 @@ public class HistoryFileManager extends AbstractService {
JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
createHistoryDirs(SystemClock.getInstance(), 10 * 1000, maxFSWaitTime);
+ maxTasksForLoadedJob = conf.getInt(
+ JHAdminConfig.MR_HS_LOADED_JOBS_TASKS_MAX,
+ JHAdminConfig.DEFAULT_MR_HS_LOADED_JOBS_TASKS_MAX);
+
this.aclsMgr = new JobACLsManager(conf);
maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b390dd6e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java
new file mode 100644
index 0000000..cea336c
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.hs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.TaskCompletionEvent;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.JobACL;
+import org.apache.hadoop.mapreduce.v2.api.records.*;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.util.Records;
+
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * A job that has too many tasks associated with it, of which we do not parse
+ * its job history file, to prevent the Job History Server from hanging on
+ * parsing the file. It is meant to be used only by JHS to indicate if the
+ * history file of a job is fully parsed or not.
+ */
+public class UnparsedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job {
+ private final JobIndexInfo jobIndexInfo;
+ private final int maxTasksAllowed;
+ private JobReport jobReport;
+ private final HistoryFileManager.HistoryFileInfo jhfInfo;
+
+ public UnparsedJob(int maxTasksAllowed, JobIndexInfo jobIndexInfo,
+ HistoryFileManager.HistoryFileInfo jhfInfo) throws IOException {
+ this.jobIndexInfo = jobIndexInfo;
+ this.jhfInfo = jhfInfo;
+ this.maxTasksAllowed = maxTasksAllowed;
+ }
+
+ public int getMaxTasksAllowed() {
+ return maxTasksAllowed;
+ }
+
+ @Override
+ public JobId getID() {
+ return jobIndexInfo.getJobId();
+ }
+
+ @Override
+ public String getName() {
+ return jobIndexInfo.getJobName();
+ }
+
+ @Override
+ public JobState getState() {
+ return JobState.valueOf(jobIndexInfo.getJobStatus());
+ }
+
+ @Override
+ public synchronized JobReport getReport() {
+ if(jobReport == null) {
+ jobReport = constructJobReport();
+ }
+ return jobReport;
+ }
+
+ public JobReport constructJobReport() {
+ JobReport report = Records.newRecord(JobReport.class);
+ report.setJobId(getID());
+ report.setJobState(getState());
+ report.setSubmitTime(jobIndexInfo.getSubmitTime());
+ report.setStartTime(jobIndexInfo.getJobStartTime());
+ report.setFinishTime(jobIndexInfo.getFinishTime());
+ report.setJobName(jobIndexInfo.getJobName());
+ report.setUser(jobIndexInfo.getUser());
+ report.setJobFile(getConfFile().toString());
+ report.setHistoryFile(jhfInfo.getHistoryFile().toString());
+ return report;
+ }
+
+ @Override
+ public Counters getAllCounters() {
+ return new Counters();
+ }
+
+ @Override
+ public Map<TaskId, Task> getTasks() {
+ return new HashMap<>();
+ }
+
+ @Override
+ public Map<TaskId, Task> getTasks(TaskType taskType) {
+ return new HashMap<>();
+ }
+
+ @Override
+ public Task getTask(TaskId taskID) {
+ return null;
+ }
+
+ @Override
+ public List<String> getDiagnostics() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public int getTotalMaps() {
+ return jobIndexInfo.getNumMaps();
+ }
+
+ @Override
+ public int getTotalReduces() {
+ return jobIndexInfo.getNumReduces();
+ }
+
+ @Override
+ public int getCompletedMaps() {
+ return -1;
+ }
+
+ @Override
+ public int getCompletedReduces() {
+ return -1;
+ }
+
+ @Override
+ public float getProgress() {
+ return 1.0f;
+ }
+
+ @Override
+ public boolean isUber() {
+ return false;
+ }
+
+ @Override
+ public String getUserName() {
+ return jobIndexInfo.getUser();
+ }
+
+ @Override
+ public String getQueueName() {
+ return jobIndexInfo.getQueueName();
+ }
+
+ @Override
+ public Path getConfFile() {
+ return jhfInfo.getConfFile();
+ }
+
+ @Override
+ public Configuration loadConfFile() throws IOException {
+ return jhfInfo.loadConfFile();
+ }
+
+ @Override
+ public Map<JobACL, AccessControlList> getJobACLs() {
+ return new HashMap<>();
+ }
+
+ @Override
+ public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
+ int fromEventId, int maxEvents) {
+ return new TaskAttemptCompletionEvent[0];
+ }
+
+ @Override
+ public TaskCompletionEvent[] getMapAttemptCompletionEvents(
+ int startIndex, int maxEvents) {
+ return new TaskCompletionEvent[0];
+ }
+
+ @Override
+ public List<AMInfo> getAMInfos() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public boolean checkAccess(UserGroupInformation callerUGI,
+ JobACL jobOperation) {
+ return true;
+ }
+
+ @Override
+ public void setQueueName(String queueName) {
+ throw new UnsupportedOperationException("Can't set job's " +
+ "queue name in history");
+ }
+
+ @Override
+ public void setJobPriority(Priority priority) {
+ throw new UnsupportedOperationException(
+ "Can't set job's priority in history");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b390dd6e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
index 13774a8..0d5b03a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
@@ -33,8 +33,10 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.ConfEntryInfo;
+import org.apache.hadoop.mapreduce.v2.hs.UnparsedJob;
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.AMAttemptInfo;
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobInfo;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.mapreduce.v2.util.MRApps.TaskAttemptStateUI;
import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil;
@@ -73,8 +75,18 @@ public class HsJobBlock extends HtmlBlock {
JobId jobID = MRApps.toJobID(jid);
Job j = appContext.getJob(jobID);
if (j == null) {
- html.
- p()._("Sorry, ", jid, " not found.")._();
+ html.p()._("Sorry, ", jid, " not found.")._();
+ return;
+ }
+ if(j instanceof UnparsedJob) {
+ final int taskCount = j.getTotalMaps() + j.getTotalReduces();
+ UnparsedJob oversizedJob = (UnparsedJob) j;
+ html.p()._("The job has a total of " + taskCount + " tasks. ")
+ ._("Any job larger than " + oversizedJob.getMaxTasksAllowed() +
+ " will not be loaded.")._();
+ html.p()._("You can either use the CLI tool: 'mapred job -history'"
+ + " to view large jobs or adjust the property " +
+ JHAdminConfig.MR_HS_LOADED_JOBS_TASKS_MAX + ".")._();
return;
}
List<AMInfo> amInfos = j.getAMInfos();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b390dd6e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index aa2e979..b7a3672 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
@@ -25,6 +25,7 @@ import java.io.FileNotFoundException;
import java.util.UUID;
import java.util.List;
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -37,11 +38,9 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
-import org.apache.hadoop.test.CoreTestDriver;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.Clock;
@@ -86,6 +85,10 @@ public class TestHistoryFileManager {
@After
public void cleanTest() throws Exception {
new File(coreSitePath).delete();
+ dfsCluster.getFileSystem().setSafeMode(
+ HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+ dfsCluster2.getFileSystem().setSafeMode(
+ HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
}
private String getDoneDirNameForTest() {
@@ -247,6 +250,97 @@ public class TestHistoryFileManager {
Assert.assertFalse(info.didMoveFail());
}
+ @Test
+ public void testHistoryFileInfoLoadOversizedJobShouldReturnUnParsedJob()
+ throws Exception {
+ HistoryFileManagerTest hmTest = new HistoryFileManagerTest();
+
+ int allowedMaximumTasks = 5;
+ Configuration conf = dfsCluster.getConfiguration(0);
+ conf.setInt(JHAdminConfig.MR_HS_LOADED_JOBS_TASKS_MAX, allowedMaximumTasks);
+
+ hmTest.init(conf);
+
+ // set up a job of which the number of tasks is greater than maximum allowed
+ String jobId = "job_1410889000000_123456";
+ JobIndexInfo jobIndexInfo = new JobIndexInfo();
+ jobIndexInfo.setJobId(TypeConverter.toYarn(JobID.forName(jobId)));
+ jobIndexInfo.setNumMaps(allowedMaximumTasks);
+ jobIndexInfo.setNumReduces(allowedMaximumTasks);
+
+
+ HistoryFileInfo info = hmTest.getHistoryFileInfo(null, null, null,
+ jobIndexInfo, false);
+
+ Job job = info.loadJob();
+ Assert.assertTrue("Should return an instance of UnparsedJob to indicate" +
+ " the job history file is not parsed", job instanceof UnparsedJob);
+ }
+
+ @Test
+ public void testHistoryFileInfoLoadNormalSizedJobShouldReturnCompletedJob()
+ throws Exception {
+ HistoryFileManagerTest hmTest = new HistoryFileManagerTest();
+
+ final int numOfTasks = 100;
+ Configuration conf = dfsCluster.getConfiguration(0);
+ conf.setInt(JHAdminConfig.MR_HS_LOADED_JOBS_TASKS_MAX,
+ numOfTasks + numOfTasks + 1);
+
+ hmTest.init(conf);
+
+ // set up a job of which the number of tasks is smaller than the maximum
+ // allowed, and therefore will be fully loaded.
+ final String jobId = "job_1416424547277_0002";
+ JobIndexInfo jobIndexInfo = new JobIndexInfo();
+ jobIndexInfo.setJobId(TypeConverter.toYarn(JobID.forName(jobId)));
+ jobIndexInfo.setNumMaps(numOfTasks);
+ jobIndexInfo.setNumReduces(numOfTasks);
+
+
+ final String historyFile = getClass().getClassLoader().getResource(
+ "job_2.0.3-alpha-FAILED.jhist").getFile();
+ final Path historyFilePath = FileSystem.getLocal(conf).makeQualified(
+ new Path(historyFile));
+ HistoryFileInfo info = hmTest.getHistoryFileInfo(historyFilePath, null,
+ null, jobIndexInfo, false);
+
+ Job job = info.loadJob();
+ Assert.assertTrue("Should return an instance of CompletedJob as " +
+ "a result of parsing the job history file of the job",
+ job instanceof CompletedJob);
+ }
+
+ @Test
+ public void testHistoryFileInfoShouldReturnCompletedJobIfMaxNotConfiged()
+ throws Exception {
+ HistoryFileManagerTest hmTest = new HistoryFileManagerTest();
+
+ Configuration conf = dfsCluster.getConfiguration(0);
+ conf.setInt(JHAdminConfig.MR_HS_LOADED_JOBS_TASKS_MAX, -1);
+
+ hmTest.init(conf);
+
+ final String jobId = "job_1416424547277_0002";
+ JobIndexInfo jobIndexInfo = new JobIndexInfo();
+ jobIndexInfo.setJobId(TypeConverter.toYarn(JobID.forName(jobId)));
+ jobIndexInfo.setNumMaps(100);
+ jobIndexInfo.setNumReduces(100);
+
+ final String historyFile = getClass().getClassLoader().getResource(
+ "job_2.0.3-alpha-FAILED.jhist").getFile();
+ final Path historyFilePath = FileSystem.getLocal(conf).makeQualified(
+ new Path(historyFile));
+ HistoryFileInfo info = hmTest.getHistoryFileInfo(historyFilePath, null,
+ null, jobIndexInfo, false);
+
+ Job job = info.loadJob();
+ Assert.assertTrue("Should return an instance of CompletedJob as " +
+ "a result of parsing the job history file of the job",
+ job instanceof CompletedJob);
+
+ }
+
static class HistoryFileManagerTest extends HistoryFileManager {
public HistoryFileManagerTest() {
super();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b390dd6e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java
new file mode 100644
index 0000000..7fa238e
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.hs.webapp;
+
+import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.v2.api.records.*;
+import org.apache.hadoop.mapreduce.v2.api.records.impl.pb.JobIdPBImpl;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.app.webapp.AMParams;
+import org.apache.hadoop.mapreduce.v2.hs.CompletedJob;
+import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager;
+import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo;
+import org.apache.hadoop.mapreduce.v2.hs.JobHistory;
+import org.apache.hadoop.mapreduce.v2.hs.UnparsedJob;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.StringHelper;
+import org.apache.hadoop.yarn.webapp.ResponseInfo;
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.view.BlockForTest;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlockForTest;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test the HsJobBlock generated for oversized jobs in JHS.
+ */
+public class TestHsJobBlock {
+
+ @Test
+ public void testHsJobBlockForOversizeJobShouldDisplayWarningMessage() {
+ int maxAllowedTaskNum = 100;
+
+ Configuration config = new Configuration();
+ config.setInt(JHAdminConfig.MR_HS_LOADED_JOBS_TASKS_MAX, maxAllowedTaskNum);
+
+ JobHistory jobHistory =
+ new JobHistoryStubWithAllOversizeJobs(maxAllowedTaskNum);
+ jobHistory.init(config);
+
+ HsJobBlock jobBlock = new HsJobBlock(jobHistory) {
+ // override this so that job block can fetch a job id.
+ @Override
+ public Map<String, String> moreParams() {
+ Map<String, String> map = new HashMap<>();
+ map.put(AMParams.JOB_ID, "job_0000_0001");
+ return map;
+ }
+ };
+
+ // set up the test block to render HsJobBLock to
+ OutputStream outputStream = new ByteArrayOutputStream();
+ HtmlBlock.Block block = createBlockToCreateTo(outputStream);
+
+ jobBlock.render(block);
+
+ block.getWriter().flush();
+ String out = outputStream.toString();
+ Assert.assertTrue("Should display warning message for jobs that have too " +
+ "many tasks", out.contains("Any job larger than " + maxAllowedTaskNum +
+ " will not be loaded"));
+ }
+
+ @Test
+ public void testHsJobBlockForNormalSizeJobShouldNotDisplayWarningMessage() {
+
+ Configuration config = new Configuration();
+ config.setInt(JHAdminConfig.MR_HS_LOADED_JOBS_TASKS_MAX, -1);
+
+ JobHistory jobHistory = new JobHitoryStubWithAllNormalSizeJobs();
+ jobHistory.init(config);
+
+ HsJobBlock jobBlock = new HsJobBlock(jobHistory) {
+ // override this so that the job block can fetch a job id.
+ @Override
+ public Map<String, String> moreParams() {
+ Map<String, String> map = new HashMap<>();
+ map.put(AMParams.JOB_ID, "job_0000_0001");
+ return map;
+ }
+
+ // override this to avoid view context lookup in render()
+ @Override
+ public ResponseInfo info(String about) {
+ return new ResponseInfo().about(about);
+ }
+
+ // override this to avoid view context lookup in render()
+ @Override
+ public String url(String... parts) {
+ return StringHelper.ujoin("", parts);
+ }
+ };
+
+ // set up the test block to render HsJobBLock to
+ OutputStream outputStream = new ByteArrayOutputStream();
+ HtmlBlock.Block block = createBlockToCreateTo(outputStream);
+
+ jobBlock.render(block);
+
+ block.getWriter().flush();
+ String out = outputStream.toString();
+
+ Assert.assertTrue("Should display job overview for the job.",
+ out.contains("ApplicationMaster"));
+ }
+
+ private static HtmlBlock.Block createBlockToCreateTo(
+ OutputStream outputStream) {
+ PrintWriter printWriter = new PrintWriter(outputStream);
+ HtmlBlock html = new HtmlBlockForTest();
+ return new BlockForTest(html, printWriter, 10, false) {
+ @Override
+ protected void subView(Class<? extends SubView> cls) {
+ }
+ };
+ };
+
+ /**
+ * A JobHistory stub that treat all jobs as oversized and therefore will
+ * not parse their job history files but return a UnparseJob instance.
+ */
+ static class JobHistoryStubWithAllOversizeJobs extends JobHistory {
+ private final int maxAllowedTaskNum;
+
+ public JobHistoryStubWithAllOversizeJobs(int maxAllowedTaskNum) {
+ this.maxAllowedTaskNum = maxAllowedTaskNum;
+ }
+
+ @Override
+ protected HistoryFileManager createHistoryFileManager() {
+ HistoryFileManager historyFileManager;
+ try {
+ HistoryFileInfo historyFileInfo =
+ createUnparsedJobHistoryFileInfo(maxAllowedTaskNum);
+
+ historyFileManager = mock(HistoryFileManager.class);
+ when(historyFileManager.getFileInfo(any(JobId.class))).thenReturn(
+ historyFileInfo);
+ } catch (IOException ex) {
+ // this should never happen
+ historyFileManager = super.createHistoryFileManager();
+ }
+ return historyFileManager;
+ }
+
+ private static HistoryFileInfo createUnparsedJobHistoryFileInfo(
+ int maxAllowedTaskNum) throws IOException {
+ HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
+
+ // create an instance of UnparsedJob for a large job
+ UnparsedJob unparsedJob = mock(UnparsedJob.class);
+ when(unparsedJob.getMaxTasksAllowed()).thenReturn(maxAllowedTaskNum);
+ when(unparsedJob.getTotalMaps()).thenReturn(maxAllowedTaskNum);
+ when(unparsedJob.getTotalReduces()).thenReturn(maxAllowedTaskNum);
+
+ when(fileInfo.loadJob()).thenReturn(unparsedJob);
+
+ return fileInfo;
+ }
+ }
+
+ /**
+ * A JobHistory stub that treats all jobs as normal size and therefore will
+ * return a CompletedJob on HistoryFileInfo.loadJob().
+ */
+ static class JobHitoryStubWithAllNormalSizeJobs extends JobHistory {
+ @Override
+ public HistoryFileManager createHistoryFileManager() {
+ HistoryFileManager historyFileManager;
+ try {
+ HistoryFileInfo historyFileInfo = createParsedJobHistoryFileInfo();
+
+ historyFileManager = mock(HistoryFileManager.class);
+ when(historyFileManager.getFileInfo(any(JobId.class))).thenReturn(
+ historyFileInfo);
+ } catch (IOException ex) {
+ // this should never happen
+ historyFileManager = super.createHistoryFileManager();
+ }
+ return historyFileManager;
+
+ }
+
+ private static HistoryFileInfo createParsedJobHistoryFileInfo()
+ throws IOException {
+ HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
+ CompletedJob job = createFakeCompletedJob();
+ when(fileInfo.loadJob()).thenReturn(job);
+ return fileInfo;
+ }
+
+
+ private static CompletedJob createFakeCompletedJob() {
+ CompletedJob job = mock(CompletedJob.class);
+
+ when(job.getTotalMaps()).thenReturn(0);
+ when(job.getCompletedMaps()).thenReturn(0);
+ when(job.getTotalReduces()).thenReturn(0);
+ when(job.getCompletedReduces()).thenReturn(0);
+
+ JobId jobId = createFakeJobId();
+ when(job.getID()).thenReturn(jobId);
+
+ JobReport jobReport = mock(JobReport.class);
+ when(jobReport.getSubmitTime()).thenReturn(-1L);
+ when(jobReport.getStartTime()).thenReturn(-1L);
+ when(jobReport.getFinishTime()).thenReturn(-1L);
+ when(job.getReport()).thenReturn(jobReport);
+
+ when(job.getAMInfos()).thenReturn(new ArrayList<AMInfo>());
+ when(job.getDiagnostics()).thenReturn(new ArrayList<String>());
+ when(job.getName()).thenReturn("fake completed job");
+ when(job.getQueueName()).thenReturn("default");
+ when(job.getUserName()).thenReturn("junit");
+ when(job.getState()).thenReturn(JobState.ERROR);
+ when(job.getAllCounters()).thenReturn(new Counters());
+ when(job.getTasks()).thenReturn(new HashMap<TaskId, Task>());
+
+ return job;
+ }
+
+ private static JobId createFakeJobId() {
+ JobId jobId = new JobIdPBImpl();
+ jobId.setId(0);
+
+ ApplicationId appId = mock(ApplicationId.class);
+ when(appId.getClusterTimestamp()).thenReturn(0L);
+ when(appId.getId()).thenReturn(0);
+
+ jobId.setAppId(appId);
+
+ return jobId;
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] hadoop git commit: YARN-5383. Fix findbugs and checkstyle
issues in ContainerExecutor. (Vrushali C via Varun Saxena)
Posted by su...@apache.org.
YARN-5383. Fix findbugs and checkstyle issues in ContainerExecutor. (Vrushali C via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/672a52f1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/672a52f1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/672a52f1
Branch: refs/heads/YARN-2915
Commit: 672a52f13887a4c52122c5c082ba7f35cd1e9e9d
Parents: eee5643
Author: Varun Saxena <va...@apache.org>
Authored: Fri Jul 15 23:17:04 2016 +0530
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:31 2016 -0700
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 52 ++++++++++----------
1 file changed, 27 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/672a52f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 2689480..f08db5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -305,7 +305,7 @@ public abstract class ContainerExecutor implements Configurable {
List<String> command, Path logDir, String outFilename)
throws IOException {
ContainerLaunch.ShellScriptBuilder sb =
- ContainerLaunch.ShellScriptBuilder.create();
+ ContainerLaunch.ShellScriptBuilder.create();
Set<String> whitelist = new HashSet<>();
whitelist.add(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME);
@@ -316,7 +316,7 @@ public abstract class ContainerExecutor implements Configurable {
whitelist.add(ApplicationConstants.Environment.JAVA_HOME.name());
if (environment != null) {
- for (Map.Entry<String,String> env : environment.entrySet()) {
+ for (Map.Entry<String, String> env : environment.entrySet()) {
if (!whitelist.contains(env.getKey())) {
sb.env(env.getKey(), env.getValue());
} else {
@@ -326,19 +326,20 @@ public abstract class ContainerExecutor implements Configurable {
}
if (resources != null) {
- for (Path path: resources.keySet()) {
- for (String linkName: resources.get(path)) {
+ for (Map.Entry<Path, List<String>> resourceEntry :
+ resources.entrySet()) {
+ for (String linkName : resourceEntry.getValue()) {
if (new Path(linkName).getName().equals(WILDCARD)) {
// If this is a wildcarded path, link to everything in the
// directory from the working directory
- File directory = new File(path.toString());
+ File directory = new File(resourceEntry.getKey().toString());
for (File wildLink : directory.listFiles()) {
sb.symlink(new Path(wildLink.toString()),
new Path(wildLink.getName()));
}
} else {
- sb.symlink(path, new Path(linkName));
+ sb.symlink(resourceEntry.getKey(), new Path(linkName));
}
}
}
@@ -467,12 +468,12 @@ public abstract class ContainerExecutor implements Configurable {
* @param groupId the job owner's GID
* @param userName the job owner's username
* @param pidFile the path to the container's PID file
- * @param conf the configuration
+ * @param config the configuration
* @return the command line to execute
*/
protected String[] getRunCommand(String command, String groupId,
- String userName, Path pidFile, Configuration conf) {
- return getRunCommand(command, groupId, userName, pidFile, conf, null);
+ String userName, Path pidFile, Configuration config) {
+ return getRunCommand(command, groupId, userName, pidFile, config, null);
}
/**
@@ -488,18 +489,18 @@ public abstract class ContainerExecutor implements Configurable {
* systems it is ignored.
* @param pidFile the path to the container's PID file on Windows. On other
* operating systems it is ignored.
- * @param conf the configuration
+ * @param config the configuration
* @param resource on Windows this parameter controls memory and CPU limits.
* If null, no limits are set. On other operating systems it is ignored.
* @return the command line to execute
*/
protected String[] getRunCommand(String command, String groupId,
- String userName, Path pidFile, Configuration conf, Resource resource) {
+ String userName, Path pidFile, Configuration config, Resource resource) {
if (Shell.WINDOWS) {
return getRunCommandForWindows(command, groupId, userName, pidFile,
- conf, resource);
+ config, resource);
} else {
- return getRunCommandForOther(command, conf);
+ return getRunCommandForOther(command, config);
}
}
@@ -513,30 +514,31 @@ public abstract class ContainerExecutor implements Configurable {
* @param groupId the job owner's GID
* @param userName the job owner's username
* @param pidFile the path to the container's PID file
- * @param conf the configuration
+ * @param config the configuration
* @param resource this parameter controls memory and CPU limits.
* If null, no limits are set.
* @return the command line to execute
*/
protected String[] getRunCommandForWindows(String command, String groupId,
- String userName, Path pidFile, Configuration conf, Resource resource) {
+ String userName, Path pidFile, Configuration config, Resource resource) {
int cpuRate = -1;
int memory = -1;
if (resource != null) {
- if (conf.getBoolean(
+ if (config.getBoolean(
YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED,
- YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
- memory = (int)resource.getMemorySize();
+ YarnConfiguration.
+ DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
+ memory = (int) resource.getMemorySize();
}
- if (conf.getBoolean(
+ if (config.getBoolean(
YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED,
YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) {
int containerVCores = resource.getVirtualCores();
- int nodeVCores = NodeManagerHardwareUtils.getVCores(conf);
+ int nodeVCores = NodeManagerHardwareUtils.getVCores(config);
int nodeCpuPercentage =
- NodeManagerHardwareUtils.getNodeCpuPercentage(conf);
+ NodeManagerHardwareUtils.getNodeCpuPercentage(config);
float containerCpuPercentage =
(float)(nodeCpuPercentage * containerVCores) / nodeVCores;
@@ -564,20 +566,20 @@ public abstract class ContainerExecutor implements Configurable {
* Return a command line to execute the given command in the OS shell.
*
* @param command the command to execute
- * @param conf the configuration
+ * @param config the configuration
* @return the command line to execute
*/
protected String[] getRunCommandForOther(String command,
- Configuration conf) {
+ Configuration config) {
List<String> retCommand = new ArrayList<>();
boolean containerSchedPriorityIsSet = false;
int containerSchedPriorityAdjustment =
YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY;
- if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) !=
+ if (config.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) !=
null) {
containerSchedPriorityIsSet = true;
- containerSchedPriorityAdjustment = conf
+ containerSchedPriorityAdjustment = config
.getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY,
YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] hadoop git commit: YARN=5181. ClusterNodeTracker: add method
to get list of nodes matching a specific resourceName. (kasha via asuresh)
Posted by su...@apache.org.
YARN=5181. ClusterNodeTracker: add method to get list of nodes matching a specific resourceName. (kasha via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f878bd40
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f878bd40
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f878bd40
Branch: refs/heads/YARN-2915
Commit: f878bd4085a666a6956317a4037b1d27f930b2de
Parents: 4a483d6
Author: Arun Suresh <as...@apache.org>
Authored: Fri Jul 15 14:35:12 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
.../scheduler/ClusterNodeTracker.java | 56 ++++++++++++----
.../scheduler/TestClusterNodeTracker.java | 69 ++++++++++++++++++++
2 files changed, 114 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f878bd40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index 413bbdf..e487f69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -18,11 +18,13 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -50,7 +52,8 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
private Lock writeLock = readWriteLock.writeLock();
private HashMap<NodeId, N> nodes = new HashMap<>();
- private Map<String, Integer> nodesPerRack = new HashMap<>();
+ private Map<String, N> nodeNameToNodeMap = new HashMap<>();
+ private Map<String, List<N>> nodesPerRack = new HashMap<>();
private Resource clusterCapacity = Resources.clone(Resources.none());
private Resource staleClusterCapacity = null;
@@ -66,14 +69,16 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
writeLock.lock();
try {
nodes.put(node.getNodeID(), node);
+ nodeNameToNodeMap.put(node.getNodeName(), node);
// Update nodes per rack as well
String rackName = node.getRackName();
- Integer numNodes = nodesPerRack.get(rackName);
- if (numNodes == null) {
- numNodes = 0;
+ List<N> nodesList = nodesPerRack.get(rackName);
+ if (nodesList == null) {
+ nodesList = new ArrayList<>();
+ nodesPerRack.put(rackName, nodesList);
}
- nodesPerRack.put(rackName, ++numNodes);
+ nodesList.add(node);
// Update cluster capacity
Resources.addTo(clusterCapacity, node.getTotalResource());
@@ -126,8 +131,8 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
readLock.lock();
String rName = rackName == null ? "NULL" : rackName;
try {
- Integer nodeCount = nodesPerRack.get(rName);
- return nodeCount == null ? 0 : nodeCount;
+ List<N> nodesList = nodesPerRack.get(rName);
+ return nodesList == null ? 0 : nodesList.size();
} finally {
readLock.unlock();
}
@@ -154,14 +159,18 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
LOG.warn("Attempting to remove a non-existent node " + nodeId);
return null;
}
+ nodeNameToNodeMap.remove(node.getNodeName());
// Update nodes per rack as well
String rackName = node.getRackName();
- Integer numNodes = nodesPerRack.get(rackName);
- if (numNodes > 0) {
- nodesPerRack.put(rackName, --numNodes);
- } else {
+ List<N> nodesList = nodesPerRack.get(rackName);
+ if (nodesList == null) {
LOG.error("Attempting to remove node from an empty rack " + rackName);
+ } else {
+ nodesList.remove(node);
+ if (nodesList.isEmpty()) {
+ nodesPerRack.remove(rackName);
+ }
}
// Update cluster capacity
@@ -297,4 +306,29 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
Collections.sort(sortedList, comparator);
return sortedList;
}
+
+ /**
+ * Convenience method to return list of nodes corresponding to resourceName
+ * passed in the {@link ResourceRequest}.
+ *
+ * @param resourceName Host/rack name of the resource, or
+ * {@link ResourceRequest#ANY}
+ * @return list of nodes that match the resourceName
+ */
+ public List<N> getNodesByResourceName(final String resourceName) {
+ Preconditions.checkArgument(
+ resourceName != null && !resourceName.isEmpty());
+ List<N> retNodes = new ArrayList<>();
+ if (ResourceRequest.ANY.equals(resourceName)) {
+ return getAllNodes();
+ } else if (nodeNameToNodeMap.containsKey(resourceName)) {
+ retNodes.add(nodeNameToNodeMap.get(resourceName));
+ } else if (nodesPerRack.containsKey(resourceName)) {
+ return nodesPerRack.get(resourceName);
+ } else {
+ LOG.info(
+ "Could not find a node matching given resourceName " + resourceName);
+ }
+ return retNodes;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f878bd40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
new file mode 100644
index 0000000..7f527f1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSSchedulerNode;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test class to verify ClusterNodeTracker. Using FSSchedulerNode without
+ * loss of generality.
+ */
+public class TestClusterNodeTracker {
+ private ClusterNodeTracker<FSSchedulerNode> nodeTracker =
+ new ClusterNodeTracker<>();
+
+ @Before
+ public void setup() {
+ List<RMNode> rmNodes =
+ MockNodes.newNodes(2, 4, Resource.newInstance(4096, 4));
+ for (RMNode rmNode : rmNodes) {
+ nodeTracker.addNode(new FSSchedulerNode(rmNode, false));
+ }
+ }
+
+ @Test
+ public void testGetNodeCount() {
+ assertEquals("Incorrect number of nodes in the cluster",
+ 8, nodeTracker.nodeCount());
+
+ assertEquals("Incorrect number of nodes in each rack",
+ 4, nodeTracker.nodeCount("rack0"));
+ }
+
+ @Test
+ public void testGetNodesForResourceName() throws Exception {
+ assertEquals("Incorrect number of nodes matching ANY",
+ 8, nodeTracker.getNodesByResourceName(ResourceRequest.ANY).size());
+
+ assertEquals("Incorrect number of nodes matching rack",
+ 4, nodeTracker.getNodesByResourceName("rack0").size());
+
+ assertEquals("Incorrect number of nodes matching node",
+ 1, nodeTracker.getNodesByResourceName("host0").size());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] hadoop git commit: YARN-4484. Available Resource calculation
for a queue is not correct when used with labels. (Sunil G via wangda)
Posted by su...@apache.org.
YARN-4484. Available Resource calculation for a queue is not correct when used with labels. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80b2f931
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80b2f931
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80b2f931
Branch: refs/heads/YARN-2915
Commit: 80b2f931816aaeea127e03989c2b44f84ecab2fb
Parents: 4ac42f2
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Jul 15 11:40:12 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:31 2016 -0700
----------------------------------------------------------------------
.../scheduler/capacity/CSQueueUtils.java | 55 ++---
.../TestNodeLabelContainerAllocation.java | 212 +++++++++++++++++++
2 files changed, 242 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80b2f931/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 0166d83..d5cdb32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -227,24 +227,34 @@ class CSQueueUtils {
.setAbsoluteReservedCapacity(nodePartition, absoluteReservedCapacity);
}
- private static Resource getNonPartitionedMaxAvailableResourceToQueue(
- final ResourceCalculator rc, Resource totalNonPartitionedResource,
- CSQueue queue) {
- Resource queueLimit = Resources.none();
- Resource usedResources = queue.getUsedResources();
+ private static Resource getMaxAvailableResourceToQueue(
+ final ResourceCalculator rc, RMNodeLabelsManager nlm, CSQueue queue,
+ Resource cluster) {
+ Set<String> nodeLabels = queue.getNodeLabelsForQueue();
+ Resource totalAvailableResource = Resources.createResource(0, 0);
- if (Resources.greaterThan(rc, totalNonPartitionedResource,
- totalNonPartitionedResource, Resources.none())) {
- queueLimit =
- Resources.multiply(totalNonPartitionedResource,
- queue.getAbsoluteCapacity());
- }
+ for (String partition : nodeLabels) {
+ // Calculate guaranteed resource for a label in a queue by below logic.
+ // (total label resource) * (absolute capacity of label in that queue)
+ Resource queueGuranteedResource = Resources.multiply(nlm
+ .getResourceByLabel(partition, cluster), queue.getQueueCapacities()
+ .getAbsoluteCapacity(partition));
- Resource available = Resources.subtract(queueLimit, usedResources);
- return Resources.max(rc, totalNonPartitionedResource, available,
- Resources.none());
+ // Available resource in queue for a specific label will be calculated as
+ // {(guaranteed resource for a label in a queue) -
+ // (resource usage of that label in the queue)}
+ // Finally accumulate this available resource to get total.
+ Resource available = (Resources.greaterThan(rc, cluster,
+ queueGuranteedResource,
+ queue.getQueueResourceUsage().getUsed(partition))) ? Resources
+ .componentwiseMax(Resources.subtractFrom(queueGuranteedResource,
+ queue.getQueueResourceUsage().getUsed(partition)), Resources
+ .none()) : Resources.none();
+ Resources.addTo(totalAvailableResource, available);
+ }
+ return totalAvailableResource;
}
-
+
/**
* <p>
* Update Queue Statistics:
@@ -277,15 +287,10 @@ class CSQueueUtils {
updateUsedCapacity(rc, nlm.getResourceByLabel(nodePartition, cluster),
minimumAllocation, queueResourceUsage, queueCapacities, nodePartition);
}
-
- // Now in QueueMetrics, we only store available-resource-to-queue for
- // default partition.
- if (nodePartition == null
- || nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
- childQueue.getMetrics().setAvailableResourcesToQueue(
- getNonPartitionedMaxAvailableResourceToQueue(rc,
- nlm.getResourceByLabel(RMNodeLabelsManager.NO_LABEL, cluster),
- childQueue));
- }
+
+ // Update queue metrics w.r.t node labels. In a generic way, we can
+ // calculate available resource from all labels in cluster.
+ childQueue.getMetrics().setAvailableResourcesToQueue(
+ getMaxAvailableResourceToQueue(rc, nlm, childQueue, cluster));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80b2f931/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index cff1514..47fd534 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import static org.junit.Assert.assertEquals;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -47,6 +49,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
@@ -1863,4 +1866,213 @@ public class TestNodeLabelContainerAllocation {
checkNumOfContainersInAnAppOnGivenNode(1, nm1.getNodeId(),
cs.getApplicationAttempt(am2.getApplicationAttemptId()));
}
+
+ @Test
+ public void testQueueMetricsWithLabels() throws Exception {
+ /**
+ * Test case: have a following queue structure:
+ *
+ * <pre>
+ * root
+ * / \
+ * a b
+ * (x) (x)
+ * </pre>
+ *
+ * a/b can access x, both of them has max-capacity-on-x = 50
+ *
+ * When doing non-exclusive allocation, app in a (or b) can use 100% of x
+ * resource.
+ */
+
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
+ this.conf);
+
+ // Define top-level queues
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[] { "a", "b" });
+ csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+
+ final String queueA = CapacitySchedulerConfiguration.ROOT + ".a";
+ csConf.setCapacity(queueA, 25);
+ csConf.setAccessibleNodeLabels(queueA, toSet("x"));
+ csConf.setCapacityByLabel(queueA, "x", 50);
+ csConf.setMaximumCapacityByLabel(queueA, "x", 50);
+ final String queueB = CapacitySchedulerConfiguration.ROOT + ".b";
+ csConf.setCapacity(queueB, 75);
+ csConf.setAccessibleNodeLabels(queueB, toSet("x"));
+ csConf.setCapacityByLabel(queueB, "x", 50);
+ csConf.setMaximumCapacityByLabel(queueB, "x", 50);
+
+ // set node -> label
+ mgr.addToCluserNodeLabels(
+ ImmutableSet.of(NodeLabel.newInstance("x", false)));
+ mgr.addToCluserNodeLabels(
+ ImmutableSet.of(NodeLabel.newInstance("y", false)));
+ mgr.addLabelsToNode(
+ ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
+ mgr.addLabelsToNode(
+ ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
+
+ // inject node label manager
+ MockRM rm1 = new MockRM(csConf) {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+
+ rm1.getRMContext().setNodeLabelManager(mgr);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB); // label = x
+ MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB); // label = y
+ // app1 -> a
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a", "x");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ // app1 asks for 5 partition=x containers
+ am1.allocate("*", 1 * GB, 5, new ArrayList<ContainerId>(), "x");
+ // NM1 do 50 heartbeats
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+
+ SchedulerNode schedulerNode1 = cs.getSchedulerNode(nm1.getNodeId());
+
+ for (int i = 0; i < 50; i++) {
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ }
+
+ // app1 gets all resource in partition=x
+ Assert.assertEquals(5, schedulerNode1.getNumContainers());
+
+ SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
+ .getNodeReport(nm1.getNodeId());
+ Assert.assertEquals(5 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(5 * GB,
+ reportNm1.getAvailableResource().getMemorySize());
+
+ SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
+ .getNodeReport(nm2.getNodeId());
+ Assert.assertEquals(0 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm2.getAvailableResource().getMemorySize());
+
+ LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
+ assertEquals(0 * GB, leafQueue.getMetrics().getAvailableMB());
+ assertEquals(5 * GB, leafQueue.getMetrics().getAllocatedMB());
+ rm1.close();
+ }
+
+ @Test
+ public void testQueueMetricsWithLabelsOnDefaultLabelNode() throws Exception {
+ /**
+ * Test case: have a following queue structure:
+ *
+ * <pre>
+ * root
+ * / \
+ * a b
+ * (x) (x)
+ * </pre>
+ *
+ * a/b can access x, both of them has max-capacity-on-x = 50
+ *
+ * When doing non-exclusive allocation, app in a (or b) can use 100% of x
+ * resource.
+ */
+
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
+ this.conf);
+
+ // Define top-level queues
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[] { "a", "b" });
+ csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+
+ final String queueA = CapacitySchedulerConfiguration.ROOT + ".a";
+ csConf.setCapacity(queueA, 25);
+ csConf.setAccessibleNodeLabels(queueA, toSet("x"));
+ csConf.setCapacityByLabel(queueA, "x", 50);
+ csConf.setMaximumCapacityByLabel(queueA, "x", 50);
+ final String queueB = CapacitySchedulerConfiguration.ROOT + ".b";
+ csConf.setCapacity(queueB, 75);
+ csConf.setAccessibleNodeLabels(queueB, toSet("x"));
+ csConf.setCapacityByLabel(queueB, "x", 50);
+ csConf.setMaximumCapacityByLabel(queueB, "x", 50);
+
+ // set node -> label
+ mgr.addToCluserNodeLabels(
+ ImmutableSet.of(NodeLabel.newInstance("x", false)));
+ mgr.addLabelsToNode(
+ ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
+
+ // inject node label manager
+ MockRM rm1 = new MockRM(csConf) {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+
+ rm1.getRMContext().setNodeLabelManager(mgr);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB); // label = x
+ MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB); // label = <no_label>
+ // app1 -> a
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2);
+
+ // app1 asks for 3 partition= containers
+ am1.allocate("*", 1 * GB, 3, new ArrayList<ContainerId>());
+
+ // NM1 do 50 heartbeats
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+
+ SchedulerNode schedulerNode1 = cs.getSchedulerNode(nm1.getNodeId());
+ for (int i = 0; i < 50; i++) {
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ }
+
+ // app1 gets all resource in partition=x (non-exclusive)
+ Assert.assertEquals(3, schedulerNode1.getNumContainers());
+
+ SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
+ .getNodeReport(nm1.getNodeId());
+ Assert.assertEquals(3 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(7 * GB,
+ reportNm1.getAvailableResource().getMemorySize());
+
+ SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
+ .getNodeReport(nm2.getNodeId());
+ Assert.assertEquals(1 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(9 * GB,
+ reportNm2.getAvailableResource().getMemorySize());
+
+ LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
+ double delta = 0.0001;
+ // 3GB is used from label x quota. 1.5 GB is remaining from default label.
+ // 2GB is remaining from label x.
+ assertEquals(3.5 * GB, leafQueue.getMetrics().getAvailableMB(), delta);
+ assertEquals(4 * GB, leafQueue.getMetrics().getAllocatedMB());
+
+ // app1 asks for 1 default partition container
+ am1.allocate("*", 1 * GB, 5, new ArrayList<ContainerId>());
+
+ // NM2 do couple of heartbeats
+ RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId());
+
+ SchedulerNode schedulerNode2 = cs.getSchedulerNode(nm2.getNodeId());
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
+
+ // app1 gets all resource in default partition
+ Assert.assertEquals(2, schedulerNode2.getNumContainers());
+
+ // 3GB is used from label x quota. 2GB used from default label.
+ // So total 2.5 GB is remaining.
+ assertEquals(2.5 * GB, leafQueue.getMetrics().getAvailableMB(), delta);
+ assertEquals(5 * GB, leafQueue.getMetrics().getAllocatedMB());
+
+ rm1.close();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] hadoop git commit: HADOOP-13254. Create framework for
configurable disk checkers (yufeigu via rkanter)
Posted by su...@apache.org.
HADOOP-13254. Create framework for configurable disk checkers (yufeigu via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b105dae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b105dae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b105dae
Branch: refs/heads/YARN-2915
Commit: 8b105daee7a49853e719db7dc98039197dfbb4da
Parents: b390dd6
Author: Robert Kanter <rk...@apache.org>
Authored: Fri Jul 15 13:41:26 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/util/BasicDiskValidator.java | 34 ++++++++
.../org/apache/hadoop/util/DiskValidator.java | 41 +++++++++
.../hadoop/util/DiskValidatorFactory.java | 88 ++++++++++++++++++++
.../hadoop/util/TestBasicDiskValidator.java | 57 +++++++++++++
.../org/apache/hadoop/util/TestDiskChecker.java | 12 +--
.../hadoop/util/TestDiskValidatorFactory.java | 59 +++++++++++++
6 files changed, 285 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b105dae/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BasicDiskValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BasicDiskValidator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BasicDiskValidator.java
new file mode 100644
index 0000000..0366687
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BasicDiskValidator.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+import java.io.File;
+
+/**
+ * BasicDiskValidator is a wrapper around {@link DiskChecker}.
+ */
+public class BasicDiskValidator implements DiskValidator {
+ public static final String NAME = "basic";
+
+ @Override
+ public void checkStatus(File dir) throws DiskErrorException {
+ DiskChecker.checkDir(dir);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b105dae/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidator.java
new file mode 100644
index 0000000..fa4f589
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidator.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+import java.io.File;
+
+/**
+ * A interface for disk validators.
+ *
+ * The {@link #checkStatus(File)} operation checks status of a file/dir.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface DiskValidator {
+ /**
+ * Check the status of a file/dir.
+ * @param dir a file/dir
+ * @throws DiskErrorException if any disk error
+ */
+ void checkStatus(File dir) throws DiskErrorException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b105dae/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java
new file mode 100644
index 0000000..29ab2ad
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * The factory class to create instance of {@link DiskValidator}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class DiskValidatorFactory {
+ @VisibleForTesting
+ static final ConcurrentHashMap<Class<? extends DiskValidator>, DiskValidator>
+ INSTANCES = new ConcurrentHashMap<>();
+
+ private DiskValidatorFactory() {
+ }
+
+ /**
+ * Returns a {@link DiskValidator} instance corresponding to the passed clazz.
+ * @param clazz a class extends {@link DiskValidator}
+ */
+ public static DiskValidator
+ getInstance(Class<? extends DiskValidator> clazz) {
+ DiskValidator diskValidator;
+ if (INSTANCES.containsKey(clazz)) {
+ diskValidator = INSTANCES.get(clazz);
+ } else {
+ diskValidator = ReflectionUtils.newInstance(clazz, null);
+ // check the return of putIfAbsent() to see if any other thread have put
+ // the instance with the same key into INSTANCES
+ DiskValidator diskValidatorRet =
+ INSTANCES.putIfAbsent(clazz, diskValidator);
+ if (diskValidatorRet != null) {
+ diskValidator = diskValidatorRet;
+ }
+ }
+
+ return diskValidator;
+ }
+
+ /**
+ * Returns {@link DiskValidator} instance corresponding to its name.
+ * The diskValidator parameter can be "basic" for {@link BasicDiskValidator}.
+ * @param diskValidator canonical class name, for example, "basic"
+ * @throws DiskErrorException if the class cannot be located
+ */
+ @SuppressWarnings("unchecked")
+ public static DiskValidator getInstance(String diskValidator)
+ throws DiskErrorException {
+ @SuppressWarnings("rawtypes")
+ Class clazz;
+
+ if (diskValidator.equalsIgnoreCase(BasicDiskValidator.NAME)) {
+ clazz = BasicDiskValidator.class;
+ } else {
+ try {
+ clazz = Class.forName(diskValidator);
+ } catch (ClassNotFoundException cnfe) {
+ throw new DiskErrorException(diskValidator
+ + " DiskValidator class not found.", cnfe);
+ }
+ }
+
+ return getInstance(clazz);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b105dae/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestBasicDiskValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestBasicDiskValidator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestBasicDiskValidator.java
new file mode 100644
index 0000000..c961ee8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestBasicDiskValidator.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+import java.io.File;
+
+/**
+ * The class to test BasicDiskValidator.
+ */
+public class TestBasicDiskValidator extends TestDiskChecker {
+ @Override
+ protected void checkDirs(boolean isDir, String perm, boolean success)
+ throws Throwable {
+ File localDir = File.createTempFile("test", "tmp");
+ try {
+ if (isDir) {
+ // reuse the file path generated by File#createTempFile to create a dir
+ localDir.delete();
+ localDir.mkdir();
+ }
+
+ Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
+ localDir.getAbsolutePath()));
+
+ DiskValidatorFactory.getInstance(BasicDiskValidator.NAME).
+ checkStatus(localDir);
+ assertTrue("call to checkDir() succeeded.", success);
+ } catch (DiskErrorException e) {
+ // call to checkDir() succeeded even though it was expected to fail
+ // if success is false, otherwise throw the exception
+ if (success) {
+ throw e;
+ }
+ } finally {
+ localDir.delete();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b105dae/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
index de54735..b484940 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
@@ -138,30 +138,30 @@ public class TestDiskChecker {
@Test (timeout = 30000)
public void testCheckDir_normal_local() throws Throwable {
- _checkDirs(true, "755", true);
+ checkDirs(true, "755", true);
}
@Test (timeout = 30000)
public void testCheckDir_notDir_local() throws Throwable {
- _checkDirs(false, "000", false);
+ checkDirs(false, "000", false);
}
@Test (timeout = 30000)
public void testCheckDir_notReadable_local() throws Throwable {
- _checkDirs(true, "000", false);
+ checkDirs(true, "000", false);
}
@Test (timeout = 30000)
public void testCheckDir_notWritable_local() throws Throwable {
- _checkDirs(true, "444", false);
+ checkDirs(true, "444", false);
}
@Test (timeout = 30000)
public void testCheckDir_notListable_local() throws Throwable {
- _checkDirs(true, "666", false);
+ checkDirs(true, "666", false);
}
- private void _checkDirs(boolean isDir, String perm, boolean success)
+ protected void checkDirs(boolean isDir, String perm, boolean success)
throws Throwable {
File localDir = File.createTempFile("test", "tmp");
if (isDir) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b105dae/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskValidatorFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskValidatorFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskValidatorFactory.java
new file mode 100644
index 0000000..1c02b7a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskValidatorFactory.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+/**
+ * The class to test DiskValidatorFactory.
+ */
+public class TestDiskValidatorFactory {
+
+ /**
+ * Trivial tests that make sure
+ * {@link DiskValidatorFactory#getInstance(String)} works as expected.
+ *
+ * @throws DiskErrorException if fail to get the instance.
+ */
+ @Test
+ public void testGetInstance() throws DiskErrorException {
+ DiskValidator diskValidator = DiskValidatorFactory.getInstance("basic");
+ assertNotNull("Fail to get the instance.", diskValidator);
+
+ assertEquals("Fail to create the correct instance.",
+ diskValidator.getClass(), BasicDiskValidator.class);
+
+ assertNotNull("Fail to cache the object", DiskValidatorFactory.INSTANCES.
+ get(BasicDiskValidator.class));
+ }
+
+ /**
+ * To test whether an exception is threw out as expected if trying to create
+ * a non-exist class.
+ * @throws DiskErrorException if fail to get the instance.
+ */
+ @Test(expected = DiskErrorException.class)
+ public void testGetInstanceOfNonExistClass() throws DiskErrorException {
+ DiskValidatorFactory.getInstance("non-exist");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] hadoop git commit: HADOOP-13366. Fix dead link in
o.a.h.fs.CommonConfigurationKeysPublic javadoc. Contributed by Rakesh R.
Posted by su...@apache.org.
HADOOP-13366. Fix dead link in o.a.h.fs.CommonConfigurationKeysPublic javadoc. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/320221bd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/320221bd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/320221bd
Branch: refs/heads/YARN-2915
Commit: 320221bd141599672ac6cbfe2c4e6335f0beb239
Parents: b06353e
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Jul 13 13:02:20 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../hadoop/fs/CommonConfigurationKeys.java | 12 +-
.../fs/CommonConfigurationKeysPublic.java | 444 +++++++++++++++----
2 files changed, 380 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/320221bd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index e455201..76cf9cc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -231,12 +231,20 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
public static final boolean HADOOP_SECURITY_TOKEN_SERVICE_USE_IP_DEFAULT =
true;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml .</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_KEY =
"hadoop.security.dns.log-slow-lookups.enabled";
public static final boolean
HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_DEFAULT = false;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml .</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String
HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_KEY =
"hadoop.security.dns.log-slow-lookups.threshold.ms";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/320221bd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index dd80080..8781f24 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -37,26 +37,46 @@ import org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec;
public class CommonConfigurationKeysPublic {
// The Keys
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY =
"net.topology.script.number.args";
/** Default value for NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY */
public static final int NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_DEFAULT = 100;
//FS keys
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String FS_DEFAULT_NAME_KEY = "fs.defaultFS";
/** Default value for FS_DEFAULT_NAME_KEY */
public static final String FS_DEFAULT_NAME_DEFAULT = "file:///";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String FS_DF_INTERVAL_KEY = "fs.df.interval";
/** Default value for FS_DF_INTERVAL_KEY */
public static final long FS_DF_INTERVAL_DEFAULT = 60000;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String FS_DU_INTERVAL_KEY = "fs.du.interval";
/** Default value for FS_DU_INTERVAL_KEY */
public static final long FS_DU_INTERVAL_DEFAULT = 600000;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY =
"fs.client.resolve.remote.symlinks";
/** Default value for FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY */
@@ -64,22 +84,42 @@ public class CommonConfigurationKeysPublic {
//Defaults are not specified for following keys
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY =
"net.topology.script.file.name";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY =
"net.topology.node.switch.mapping.impl";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String NET_TOPOLOGY_IMPL_KEY =
"net.topology.impl";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY =
"net.topology.table.file.name";
public static final String NET_DEPENDENCY_SCRIPT_FILE_NAME_KEY =
"net.topology.dependency.script.file.name";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String FS_TRASH_CHECKPOINT_INTERVAL_KEY =
"fs.trash.checkpoint.interval";
/** Default value for FS_TRASH_CHECKPOINT_INTERVAL_KEY */
@@ -97,32 +137,64 @@ public class CommonConfigurationKeysPublic {
//
/** Not used anywhere, looks like default value for FS_LOCAL_BLOCK_SIZE */
public static final long FS_LOCAL_BLOCK_SIZE_DEFAULT = 32*1024*1024;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String FS_AUTOMATIC_CLOSE_KEY = "fs.automatic.close";
/** Default value for FS_AUTOMATIC_CLOSE_KEY */
public static final boolean FS_AUTOMATIC_CLOSE_DEFAULT = true;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String FS_FILE_IMPL_KEY = "fs.file.impl";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String FS_FTP_HOST_KEY = "fs.ftp.host";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String FS_FTP_HOST_PORT_KEY = "fs.ftp.host.port";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String FS_TRASH_INTERVAL_KEY = "fs.trash.interval";
/** Default value for FS_TRASH_INTERVAL_KEY */
public static final long FS_TRASH_INTERVAL_DEFAULT = 0;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a>. */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED =
"fs.client.resolve.topology.enabled";
/** Default value for FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED. */
public static final boolean FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED_DEFAULT =
false;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IO_MAPFILE_BLOOM_SIZE_KEY =
"io.mapfile.bloom.size";
/** Default value for IO_MAPFILE_BLOOM_SIZE_KEY */
public static final int IO_MAPFILE_BLOOM_SIZE_DEFAULT = 1024*1024;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IO_MAPFILE_BLOOM_ERROR_RATE_KEY =
"io.mapfile.bloom.error.rate" ;
/** Default value for IO_MAPFILE_BLOOM_ERROR_RATE_KEY */
@@ -130,26 +202,46 @@ public class CommonConfigurationKeysPublic {
/** Codec class that implements Lzo compression algorithm */
public static final String IO_COMPRESSION_CODEC_LZO_CLASS_KEY =
"io.compression.codec.lzo.class";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IO_MAP_INDEX_INTERVAL_KEY =
"io.map.index.interval";
/** Default value for IO_MAP_INDEX_INTERVAL_DEFAULT */
public static final int IO_MAP_INDEX_INTERVAL_DEFAULT = 128;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IO_MAP_INDEX_SKIP_KEY = "io.map.index.skip";
/** Default value for IO_MAP_INDEX_SKIP_KEY */
public static final int IO_MAP_INDEX_SKIP_DEFAULT = 0;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IO_SEQFILE_COMPRESS_BLOCKSIZE_KEY =
"io.seqfile.compress.blocksize";
/** Default value for IO_SEQFILE_COMPRESS_BLOCKSIZE_KEY */
public static final int IO_SEQFILE_COMPRESS_BLOCKSIZE_DEFAULT = 1000000;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IO_FILE_BUFFER_SIZE_KEY =
"io.file.buffer.size";
/** Default value for IO_FILE_BUFFER_SIZE_KEY */
public static final int IO_FILE_BUFFER_SIZE_DEFAULT = 4096;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IO_SKIP_CHECKSUM_ERRORS_KEY =
"io.skip.checksum.errors";
/** Default value for IO_SKIP_CHECKSUM_ERRORS_KEY */
@@ -170,19 +262,35 @@ public class CommonConfigurationKeysPublic {
public static final String IO_SORT_FACTOR_KEY = "io.sort.factor";
/** Default value for IO_SORT_FACTOR_DEFAULT */
public static final int IO_SORT_FACTOR_DEFAULT = 100;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IO_SERIALIZATIONS_KEY = "io.serializations";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String TFILE_IO_CHUNK_SIZE_KEY = "tfile.io.chunk.size";
/** Default value for TFILE_IO_CHUNK_SIZE_DEFAULT */
public static final int TFILE_IO_CHUNK_SIZE_DEFAULT = 1024*1024;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String TFILE_FS_INPUT_BUFFER_SIZE_KEY =
"tfile.fs.input.buffer.size";
/** Default value for TFILE_FS_INPUT_BUFFER_SIZE_KEY */
public static final int TFILE_FS_INPUT_BUFFER_SIZE_DEFAULT = 256*1024;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String TFILE_FS_OUTPUT_BUFFER_SIZE_KEY =
"tfile.fs.output.buffer.size";
/** Default value for TFILE_FS_OUTPUT_BUFFER_SIZE_KEY */
@@ -199,32 +307,56 @@ public class CommonConfigurationKeysPublic {
public static final int HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT =
40;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY =
"ipc.client.connection.maxidletime";
/** Default value for IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY */
public static final int IPC_CLIENT_CONNECTION_MAXIDLETIME_DEFAULT = 10000; // 10s
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IPC_CLIENT_CONNECT_TIMEOUT_KEY =
"ipc.client.connect.timeout";
/** Default value for IPC_CLIENT_CONNECT_TIMEOUT_KEY */
public static final int IPC_CLIENT_CONNECT_TIMEOUT_DEFAULT = 20000; // 20s
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IPC_CLIENT_CONNECT_MAX_RETRIES_KEY =
"ipc.client.connect.max.retries";
/** Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_KEY */
public static final int IPC_CLIENT_CONNECT_MAX_RETRIES_DEFAULT = 10;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY =
"ipc.client.connect.retry.interval";
/** Default value for IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY */
public static final int IPC_CLIENT_CONNECT_RETRY_INTERVAL_DEFAULT = 1000;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY =
"ipc.client.connect.max.retries.on.timeouts";
/** Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY */
public static final int IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT = 45;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IPC_CLIENT_TCPNODELAY_KEY =
"ipc.client.tcpnodelay";
/** Default value for IPC_CLIENT_TCPNODELAY_KEY */
@@ -233,26 +365,46 @@ public class CommonConfigurationKeysPublic {
public static final String IPC_CLIENT_LOW_LATENCY = "ipc.client.low-latency";
/** Default value of IPC_CLIENT_LOW_LATENCY */
public static final boolean IPC_CLIENT_LOW_LATENCY_DEFAULT = false;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IPC_SERVER_LISTEN_QUEUE_SIZE_KEY =
"ipc.server.listen.queue.size";
/** Default value for IPC_SERVER_LISTEN_QUEUE_SIZE_KEY */
public static final int IPC_SERVER_LISTEN_QUEUE_SIZE_DEFAULT = 128;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IPC_CLIENT_KILL_MAX_KEY = "ipc.client.kill.max";
/** Default value for IPC_CLIENT_KILL_MAX_KEY */
public static final int IPC_CLIENT_KILL_MAX_DEFAULT = 10;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IPC_CLIENT_IDLETHRESHOLD_KEY =
"ipc.client.idlethreshold";
/** Default value for IPC_CLIENT_IDLETHRESHOLD_DEFAULT */
public static final int IPC_CLIENT_IDLETHRESHOLD_DEFAULT = 4000;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IPC_SERVER_TCPNODELAY_KEY =
"ipc.server.tcpnodelay";
/** Default value for IPC_SERVER_TCPNODELAY_KEY */
public static final boolean IPC_SERVER_TCPNODELAY_DEFAULT = true;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String IPC_SERVER_MAX_CONNECTIONS_KEY =
"ipc.server.max.connections";
/** Default value for IPC_SERVER_MAX_CONNECTIONS_KEY */
@@ -263,83 +415,175 @@ public class CommonConfigurationKeysPublic {
"ipc.server.log.slow.rpc";
public static final boolean IPC_SERVER_LOG_SLOW_RPC_DEFAULT = false;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY =
"hadoop.rpc.socket.factory.class.default";
public static final String HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_DEFAULT =
"org.apache.hadoop.net.StandardSocketFactory";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SOCKS_SERVER_KEY = "hadoop.socks.server";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_UTIL_HASH_TYPE_KEY =
"hadoop.util.hash.type";
/** Default value for HADOOP_UTIL_HASH_TYPE_KEY */
public static final String HADOOP_UTIL_HASH_TYPE_DEFAULT = "murmur";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_GROUP_MAPPING =
"hadoop.security.group.mapping";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_GROUPS_CACHE_SECS =
"hadoop.security.groups.cache.secs";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final long HADOOP_SECURITY_GROUPS_CACHE_SECS_DEFAULT =
300;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS =
"hadoop.security.groups.negative-cache.secs";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final long HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS_DEFAULT =
30;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS =
"hadoop.security.groups.cache.warn.after.ms";
public static final long HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT =
5000;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD =
"hadoop.security.groups.cache.background.reload";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a>. */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final boolean
HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_DEFAULT = false;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a>. */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String
HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS =
"hadoop.security.groups.cache.background.reload.threads";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a>. */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final int
HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS_DEFAULT = 3;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a>.*/
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_AUTHENTICATION =
"hadoop.security.authentication";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_AUTHORIZATION =
"hadoop.security.authorization";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN =
"hadoop.security.instrumentation.requires.admin";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_SERVICE_USER_NAME_KEY =
"hadoop.security.service.user.name.key";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_AUTH_TO_LOCAL =
"hadoop.security.auth_to_local";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_DNS_INTERFACE_KEY =
"hadoop.security.dns.interface";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_DNS_NAMESERVER_KEY =
"hadoop.security.dns.nameserver";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_TOKEN_FILES =
"hadoop.token.files";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN =
"hadoop.kerberos.min.seconds.before.relogin";
/** Default value for HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN */
public static final int HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT =
60;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_RPC_PROTECTION =
"hadoop.rpc.protection";
/** Class to override Sasl Properties for a connection */
@@ -355,15 +599,27 @@ public class CommonConfigurationKeysPublic {
HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_DEFAULT =
OpensslAesCtrCryptoCodec.class.getName() + "," +
JceAesCtrCryptoCodec.class.getName();
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY =
"hadoop.security.crypto.cipher.suite";
public static final String HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT =
"AES/CTR/NoPadding";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY =
"hadoop.security.crypto.jce.provider";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY =
"hadoop.security.crypto.buffer.size";
/** Defalt value for HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY */
@@ -373,60 +629,100 @@ public class CommonConfigurationKeysPublic {
"hadoop.security.impersonation.provider.class";
// <!-- KMSClientProvider configurations -->
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String KMS_CLIENT_ENC_KEY_CACHE_SIZE =
"hadoop.security.kms.client.encrypted.key.cache.size";
/** Default value for KMS_CLIENT_ENC_KEY_CACHE_SIZE */
public static final int KMS_CLIENT_ENC_KEY_CACHE_SIZE_DEFAULT = 500;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK =
"hadoop.security.kms.client.encrypted.key.cache.low-watermark";
/** Default value for KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK */
public static final float KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK_DEFAULT =
0.3f;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS =
"hadoop.security.kms.client.encrypted.key.cache.num.refill.threads";
/** Default value for KMS_CLIENT_ENC_KEY_NUM_REFILL_THREADS */
public static final int KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT =
2;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_MS =
"hadoop.security.kms.client.encrypted.key.cache.expiry";
/** Default value for KMS_CLIENT_ENC_KEY_CACHE_EXPIRY (12 hrs)*/
public static final int KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_DEFAULT = 43200000;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY =
"hadoop.security.java.secure.random.algorithm";
/** Defalt value for HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY */
public static final String HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT =
"SHA1PRNG";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY =
"hadoop.security.secure.random.impl";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY =
"hadoop.security.random.device.file.path";
public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT =
"/dev/urandom";
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY =
"hadoop.shell.missing.defaultFs.warning";
public static final boolean HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT =
false;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES =
"hadoop.shell.safely.delete.limit.num.files";
public static final long HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES_DEFAULT =
100;
- /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_HTTP_LOGS_ENABLED =
"hadoop.http.logs.enabled";
/** Defalt value for HADOOP_HTTP_LOGS_ENABLED */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] hadoop git commit: HADOOP-11361. Fix a race condition in
MetricsSourceAdapter.updateJmxCache. Contributed by Vinayakumar B,
Yongjun Zhang, and Brahma Reddy Battula. (ozawa)
Posted by su...@apache.org.
HADOOP-11361. Fix a race condition in MetricsSourceAdapter.updateJmxCache. Contributed by Vinayakumar B, Yongjun Zhang, and Brahma Reddy Battula. (ozawa)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17893a23
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17893a23
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17893a23
Branch: refs/heads/YARN-2915
Commit: 17893a23734e3203446562635f7db54033305ccf
Parents: b71d5ed
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Wed Jul 13 21:28:04 2016 +0900
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:29 2016 -0700
----------------------------------------------------------------------
.../metrics2/impl/MetricsSourceAdapter.java | 33 +++++++++-----------
1 file changed, 14 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17893a23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java
index cbba014..3406ace 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java
@@ -31,6 +31,7 @@ import javax.management.ReflectionException;
import static com.google.common.base.Preconditions.*;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -59,7 +60,6 @@ class MetricsSourceAdapter implements DynamicMBean {
private final MBeanInfoBuilder infoBuilder;
private final Iterable<MetricsTag> injectedTags;
- private Iterable<MetricsRecordImpl> lastRecs;
private boolean lastRecsCleared;
private long jmxCacheTS = 0;
private long jmxCacheTTL;
@@ -175,18 +175,19 @@ class MetricsSourceAdapter implements DynamicMBean {
}
}
+ // HADOOP-11361: Release lock here for avoid deadlock between
+ // MetricsSystemImpl's lock and MetricsSourceAdapter's lock.
+ Iterable<MetricsRecordImpl> lastRecs = null;
if (getAllMetrics) {
- MetricsCollectorImpl builder = new MetricsCollectorImpl();
- getMetrics(builder, true);
+ lastRecs = getMetrics(new MetricsCollectorImpl(), true);
}
- synchronized(this) {
- updateAttrCache();
- if (getAllMetrics) {
- updateInfoCache();
+ synchronized (this) {
+ if (lastRecs != null) {
+ updateAttrCache(lastRecs);
+ updateInfoCache(lastRecs);
}
jmxCacheTS = Time.now();
- lastRecs = null; // in case regular interval update is not running
lastRecsCleared = true;
}
}
@@ -194,11 +195,6 @@ class MetricsSourceAdapter implements DynamicMBean {
Iterable<MetricsRecordImpl> getMetrics(MetricsCollectorImpl builder,
boolean all) {
builder.setRecordFilter(recordFilter).setMetricFilter(metricFilter);
- synchronized(this) {
- if (lastRecs == null && jmxCacheTS == 0) {
- all = true; // Get all the metrics to populate the sink caches
- }
- }
try {
source.getMetrics(builder, all);
} catch (Exception e) {
@@ -209,10 +205,7 @@ class MetricsSourceAdapter implements DynamicMBean {
rb.add(t);
}
}
- synchronized(this) {
- lastRecs = builder.getRecords();
- return lastRecs;
- }
+ return builder.getRecords();
}
synchronized void stop() {
@@ -246,13 +239,15 @@ class MetricsSourceAdapter implements DynamicMBean {
return jmxCacheTTL;
}
- private void updateInfoCache() {
+ private void updateInfoCache(Iterable<MetricsRecordImpl> lastRecs) {
+ Preconditions.checkNotNull(lastRecs, "LastRecs should not be null");
LOG.debug("Updating info cache...");
infoCache = infoBuilder.reset(lastRecs).get();
LOG.debug("Done");
}
- private int updateAttrCache() {
+ private int updateAttrCache(Iterable<MetricsRecordImpl> lastRecs) {
+ Preconditions.checkNotNull(lastRecs, "LastRecs should not be null");
LOG.debug("Updating attr cache...");
int recNo = 0;
int numMetrics = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] hadoop git commit: YARN-5379. TestHBaseTimelineStorage.
testWriteApplicationToHBase() fails intermittently (Vrushali C via sjlee)
Posted by su...@apache.org.
YARN-5379. TestHBaseTimelineStorage. testWriteApplicationToHBase() fails intermittently (Vrushali C via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d28d3baf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d28d3baf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d28d3baf
Branch: refs/heads/YARN-2915
Commit: d28d3bafb901f58b820d6ffe262faa49e84337d7
Parents: 9fbde75
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu Jul 14 15:19:03 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:31 2016 -0700
----------------------------------------------------------------------
.../timelineservice/storage/TestHBaseTimelineStorage.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d28d3baf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
index a8de759..e37865f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -613,8 +613,8 @@ public class TestHBaseTimelineStorage {
TimelineMetric aggMetric = new TimelineMetric();
aggMetric.setId("MEM_USAGE");
Map<Long, Number> aggMetricValues = new HashMap<Long, Number>();
- ts = System.currentTimeMillis();
- aggMetricValues.put(ts - 120000, 102400000L);
+ long aggTs = ts;
+ aggMetricValues.put(aggTs - 120000, 102400000L);
aggMetric.setType(Type.SINGLE_VALUE);
aggMetric.setRealtimeAggregationOp(TimelineMetricOperation.SUM);
aggMetric.setValues(aggMetricValues);
@@ -819,9 +819,9 @@ public class TestHBaseTimelineStorage {
metric.getValues().get(ts - 20000));
}
if (metric.getId().equals("MEM_USAGE")) {
- assertTrue(metric.getValues().containsKey(ts - 120000));
- assertEquals(aggMetricValues.get(ts - 120000),
- metric.getValues().get(ts - 120000));
+ assertTrue(metric.getValues().containsKey(aggTs - 120000));
+ assertEquals(aggMetricValues.get(aggTs - 120000),
+ metric.getValues().get(aggTs - 120000));
}
}
} finally {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] hadoop git commit: HDFS-10570. Remove classpath conflicts of
netty-all jar in hadoop-hdfs-client. Contributed by Vinayakumar B.
Posted by su...@apache.org.
HDFS-10570. Remove classpath conflicts of netty-all jar in hadoop-hdfs-client. Contributed by Vinayakumar B.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a97b0ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a97b0ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a97b0ff
Branch: refs/heads/YARN-2915
Commit: 4a97b0ff2cf7b0b5daaf591cc952d603b87b7c04
Parents: efdd81b
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jul 14 10:20:18 2016 +0100
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 31 +++++++++++++++++++++
1 file changed, 31 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a97b0ff/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index 51e7cee..1de9421 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -66,9 +66,40 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.mock-server</groupId>
<artifactId>mockserver-netty</artifactId>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] hadoop git commit: YARN-5298. Mount usercache and NM
filecache directories into Docker container. Contributed by Sidharta
Seethana.
Posted by su...@apache.org.
YARN-5298. Mount usercache and NM filecache directories into Docker container. Contributed by Sidharta Seethana.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f9f64f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f9f64f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f9f64f4
Branch: refs/heads/YARN-2915
Commit: 5f9f64f47bf8d8249cc20bd9d56269ad6cacea8a
Parents: 1bec8a5
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu Jul 14 16:11:07 2016 +0530
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/api/ApplicationConstants.java | 6 +++
.../nodemanager/LinuxContainerExecutor.java | 5 +-
.../launcher/ContainerLaunch.java | 43 +++++++++++++++-
.../runtime/DockerLinuxContainerRuntime.java | 7 +++
.../runtime/LinuxContainerRuntimeConstants.java | 4 ++
.../executor/ContainerStartContext.java | 29 ++++++++++-
.../TestLinuxContainerExecutorWithMocks.java | 8 +++
.../launcher/TestContainerLaunch.java | 9 ++--
.../runtime/TestDockerContainerRuntime.java | 52 +++++++++++++-------
9 files changed, 136 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f9f64f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index fa475e7..9442c53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -230,6 +230,12 @@ public interface ApplicationConstants {
LOCAL_DIRS("LOCAL_DIRS"),
/**
+ * $LOCAL_USER_DIRS
+ * Final, exported by NodeManager and non-modifiable by users.
+ */
+ LOCAL_USER_DIRS("LOCAL_USER_DIRS"),
+
+ /**
* $LOG_DIRS
* Final, exported by NodeManager and non-modifiable by users.
* Comma separate list of directories that the container should use for
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f9f64f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 1072b5a..be5d078 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -52,7 +52,6 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler;
import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler;
import org.apache.hadoop.yarn.server.nodemanager.util.LCEResourcesHandler;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import java.io.File;
import java.io.IOException;
@@ -311,6 +310,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
Path containerWorkDir = ctx.getContainerWorkDir();
List<String> localDirs = ctx.getLocalDirs();
List<String> logDirs = ctx.getLogDirs();
+ List<String> filecacheDirs = ctx.getFilecacheDirs();
+ List<String> userLocalDirs = ctx.getUserLocalDirs();
List<String> containerLocalDirs = ctx.getContainerLocalDirs();
List<String> containerLogDirs = ctx.getContainerLogDirs();
Map<Path, List<String>> localizedResources = ctx.getLocalizedResources();
@@ -397,6 +398,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
.setExecutionAttribute(PID_FILE_PATH, pidFilePath)
.setExecutionAttribute(LOCAL_DIRS, localDirs)
.setExecutionAttribute(LOG_DIRS, logDirs)
+ .setExecutionAttribute(FILECACHE_DIRS, filecacheDirs)
+ .setExecutionAttribute(USER_LOCAL_DIRS, userLocalDirs)
.setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
.setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
.setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f9f64f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 7e9030c..dffe120 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -230,6 +230,8 @@ public class ContainerLaunch implements Callable<Integer> {
pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
List<String> localDirs = dirsHandler.getLocalDirs();
List<String> logDirs = dirsHandler.getLogDirs();
+ List<String> filecacheDirs = getNMFilecacheDirs(localDirs);
+ List<String> userLocalDirs = getUserLocalDirs(localDirs);
List<String> containerLocalDirs = getContainerLocalDirs(localDirs);
List<String> containerLogDirs = getContainerLogDirs(logDirs);
@@ -242,6 +244,7 @@ public class ContainerLaunch implements Callable<Integer> {
try {
// /////////// Write out the container-script in the nmPrivate space.
List<Path> appDirs = new ArrayList<Path>(localDirs.size());
+
for (String localDir : localDirs) {
Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
Path userdir = new Path(usersdir, user);
@@ -258,7 +261,8 @@ public class ContainerLaunch implements Callable<Integer> {
new Path(containerWorkDir,
FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
// Sanitize the container's environment
- sanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs,
+ sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs,
+ containerLogDirs,
localResources, nmPrivateClasspathJarDir);
// Write out the environment
@@ -288,6 +292,8 @@ public class ContainerLaunch implements Callable<Integer> {
.setContainerWorkDir(containerWorkDir)
.setLocalDirs(localDirs)
.setLogDirs(logDirs)
+ .setFilecacheDirs(filecacheDirs)
+ .setUserLocalDirs(userLocalDirs)
.setContainerLocalDirs(containerLocalDirs)
.setContainerLogDirs(containerLogDirs)
.build());
@@ -353,6 +359,35 @@ public class ContainerLaunch implements Callable<Integer> {
return containerLocalDirs;
}
+ protected List<String> getUserLocalDirs(List<String> localDirs) {
+ List<String> userLocalDirs = new ArrayList<>(localDirs.size());
+ String user = container.getUser();
+
+ for (String localDir : localDirs) {
+ String userLocalDir = localDir + Path.SEPARATOR +
+ ContainerLocalizer.USERCACHE + Path.SEPARATOR + user
+ + Path.SEPARATOR;
+
+ userLocalDirs.add(userLocalDir);
+ }
+
+ return userLocalDirs;
+ }
+
+ protected List<String> getNMFilecacheDirs(List<String> localDirs) {
+ List<String> filecacheDirs = new ArrayList<>(localDirs.size());
+
+ for (String localDir : localDirs) {
+ String filecacheDir = localDir + Path.SEPARATOR +
+ ContainerLocalizer.FILECACHE;
+
+ filecacheDirs.add(filecacheDir);
+ }
+
+ return filecacheDirs;
+ }
+
+
protected Map<Path, List<String>> getLocalizedResources()
throws YarnException {
Map<Path, List<String>> localResources = container.getLocalizedResources();
@@ -990,7 +1025,8 @@ public class ContainerLaunch implements Callable<Integer> {
}
public void sanitizeEnv(Map<String, String> environment, Path pwd,
- List<Path> appDirs, List<String> containerLogDirs,
+ List<Path> appDirs, List<String> userLocalDirs, List<String>
+ containerLogDirs,
Map<Path, List<String>> resources,
Path nmPrivateClasspathJarDir) throws IOException {
/**
@@ -1012,6 +1048,9 @@ public class ContainerLaunch implements Callable<Integer> {
environment.put(Environment.LOCAL_DIRS.name(),
StringUtils.join(",", appDirs));
+ environment.put(Environment.LOCAL_USER_DIRS.name(), StringUtils.join(",",
+ userLocalDirs));
+
environment.put(Environment.LOG_DIRS.name(),
StringUtils.join(",", containerLogDirs));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f9f64f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index e1a162e..a1606f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -318,6 +318,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@SuppressWarnings("unchecked")
List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
@SuppressWarnings("unchecked")
+ List<String> filecacheDirs = ctx.getExecutionAttribute(FILECACHE_DIRS);
+ @SuppressWarnings("unchecked")
List<String> containerLocalDirs = ctx.getExecutionAttribute(
CONTAINER_LOCAL_DIRS);
@SuppressWarnings("unchecked")
@@ -326,6 +328,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@SuppressWarnings("unchecked")
Map<Path, List<String>> localizedResources = ctx.getExecutionAttribute(
LOCALIZED_RESOURCES);
+ @SuppressWarnings("unchecked")
+ List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
+
Set<String> capabilities = new HashSet<>(Arrays.asList(conf.getStrings(
YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
@@ -340,8 +345,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
.addMountLocation("/etc/passwd", "/etc/password:ro");
List<String> allDirs = new ArrayList<>(containerLocalDirs);
+ allDirs.addAll(filecacheDirs);
allDirs.add(containerWorkDir.toString());
allDirs.addAll(containerLogDirs);
+ allDirs.addAll(userLocalDirs);
for (String dir: allDirs) {
runCommand.addMountLocation(dir, dir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f9f64f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
index 96ff40d..0c1ec3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
@@ -55,6 +55,10 @@ public final class LinuxContainerRuntimeConstants {
List.class, "local_dirs");
public static final Attribute<List> LOG_DIRS = Attribute.attribute(
List.class, "log_dirs");
+ public static final Attribute<List> FILECACHE_DIRS = Attribute.attribute(
+ List.class, "filecache_dirs");
+ public static final Attribute<List> USER_LOCAL_DIRS = Attribute.attribute(
+ List.class, "user_local_dirs");
public static final Attribute<List> CONTAINER_LOCAL_DIRS = Attribute
.attribute(List.class, "container_local_dirs");
public static final Attribute<List> CONTAINER_LOG_DIRS = Attribute.attribute(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f9f64f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
index b14a520..9c5f3a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
@@ -45,6 +45,8 @@ public final class ContainerStartContext {
private final Path containerWorkDir;
private final List<String> localDirs;
private final List<String> logDirs;
+ private final List<String> filecacheDirs;
+ private final List<String> userLocalDirs;
private final List<String> containerLocalDirs;
private final List<String> containerLogDirs;
@@ -58,6 +60,8 @@ public final class ContainerStartContext {
private Path containerWorkDir;
private List<String> localDirs;
private List<String> logDirs;
+ private List<String> filecacheDirs;
+ private List<String> userLocalDirs;
private List<String> containerLocalDirs;
private List<String> containerLogDirs;
@@ -111,6 +115,16 @@ public final class ContainerStartContext {
return this;
}
+ public Builder setFilecacheDirs(List<String> filecacheDirs) {
+ this.filecacheDirs = filecacheDirs;
+ return this;
+ }
+
+ public Builder setUserLocalDirs(List<String> userLocalDirs) {
+ this.userLocalDirs = userLocalDirs;
+ return this;
+ }
+
public Builder setContainerLocalDirs(List<String> containerLocalDirs) {
this.containerLocalDirs = containerLocalDirs;
return this;
@@ -136,6 +150,8 @@ public final class ContainerStartContext {
this.containerWorkDir = builder.containerWorkDir;
this.localDirs = builder.localDirs;
this.logDirs = builder.logDirs;
+ this.filecacheDirs = builder.filecacheDirs;
+ this.userLocalDirs = builder.userLocalDirs;
this.containerLocalDirs = builder.containerLocalDirs;
this.containerLogDirs = builder.containerLogDirs;
}
@@ -180,11 +196,20 @@ public final class ContainerStartContext {
return Collections.unmodifiableList(this.logDirs);
}
+ public List<String> getFilecacheDirs() {
+ return Collections.unmodifiableList(this.filecacheDirs);
+ }
+
+ public List<String> getUserLocalDirs() {
+ return Collections.unmodifiableList(this.userLocalDirs);
+ }
+
public List<String> getContainerLocalDirs() {
- return this.containerLocalDirs;
+ return Collections.unmodifiableList(this.containerLocalDirs);
}
public List<String> getContainerLogDirs() {
- return this.containerLogDirs;
+ return Collections.unmodifiableList(this
+ .containerLogDirs);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f9f64f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index 0ef788b..aae0037 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -180,6 +180,10 @@ public class TestLinuxContainerExecutorWithMocks {
.setContainerWorkDir(workDir)
.setLocalDirs(dirsHandler.getLocalDirs())
.setLogDirs(dirsHandler.getLogDirs())
+ .setFilecacheDirs(new ArrayList<>())
+ .setUserLocalDirs(new ArrayList<>())
+ .setContainerLocalDirs(new ArrayList<>())
+ .setContainerLogDirs(new ArrayList<>())
.build());
assertEquals(0, ret);
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
@@ -345,6 +349,10 @@ public class TestLinuxContainerExecutorWithMocks {
.setContainerWorkDir(workDir)
.setLocalDirs(dirsHandler.getLocalDirs())
.setLogDirs(dirsHandler.getLogDirs())
+ .setFilecacheDirs(new ArrayList<>())
+ .setUserLocalDirs(new ArrayList<>())
+ .setContainerLocalDirs(new ArrayList<>())
+ .setContainerLogDirs(new ArrayList<>())
.build());
Assert.assertNotSame(0, ret);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f9f64f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index cc9e662..6b32dd9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -455,6 +455,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
"target/test-dir");
Path pwd = new Path(testDir);
List<Path> appDirs = new ArrayList<Path>();
+ List<String> userLocalDirs = new ArrayList<>();
List<String> containerLogs = new ArrayList<String>();
Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
@@ -465,8 +466,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
Path nmp = new Path(testDir);
- launch.sanitizeEnv(
- userSetEnv, pwd, appDirs, containerLogs, resources, nmp);
+ launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
+ resources, nmp);
List<String> result =
getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name()));
@@ -484,8 +485,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
launch = new ContainerLaunch(distContext, conf,
dispatcher, exec, null, container, dirsHandler, containerManager);
- launch.sanitizeEnv(
- userSetEnv, pwd, appDirs, containerLogs, resources, nmp);
+ launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
+ resources, nmp);
result =
getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f9f64f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 3c0b67d..bb3f0f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -82,6 +82,8 @@ public class TestDockerContainerRuntime {
private Path pidFilePath;
private List<String> localDirs;
private List<String> logDirs;
+ private List<String> filecacheDirs;
+ private List<String> userLocalDirs;
private List<String> containerLocalDirs;
private List<String> containerLogDirs;
private Map<Path, List<String>> localizedResources;
@@ -126,13 +128,17 @@ public class TestDockerContainerRuntime {
pidFilePath = new Path("/test_pid_file_path");
localDirs = new ArrayList<>();
logDirs = new ArrayList<>();
+ filecacheDirs = new ArrayList<>();
resourcesOptions = "cgroups=none";
+ userLocalDirs = new ArrayList<>();
containerLocalDirs = new ArrayList<>();
containerLogDirs = new ArrayList<>();
localizedResources = new HashMap<>();
localDirs.add("/test_local_dir");
logDirs.add("/test_log_dir");
+ filecacheDirs.add("/test_filecache_dir");
+ userLocalDirs.add("/test_user_local_dir");
containerLocalDirs.add("/test_container_local_dir");
containerLogDirs.add("/test_container_log_dir");
localizedResources.put(new Path("/test_local_dir/test_resource_file"),
@@ -156,6 +162,8 @@ public class TestDockerContainerRuntime {
.setExecutionAttribute(PID_FILE_PATH, pidFilePath)
.setExecutionAttribute(LOCAL_DIRS, localDirs)
.setExecutionAttribute(LOG_DIRS, logDirs)
+ .setExecutionAttribute(FILECACHE_DIRS, filecacheDirs)
+ .setExecutionAttribute(USER_LOCAL_DIRS, userLocalDirs)
.setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
.setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
.setExecutionAttribute(LOCALIZED_RESOURCES, localizedResources)
@@ -262,12 +270,15 @@ public class TestDockerContainerRuntime {
.append("-v %4$s:%4$s ")
.append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ")
- .append("%7$s ")
- .append("bash %8$s/launch_container.sh");
+ .append("-v %7$s:%7$s ")
+ .append("-v %8$s:%8$s ").append("%9$s ")
+ .append("bash %10$s/launch_container.sh");
- String expectedCommand = String.format(expectedCommandTemplate.toString(),
- containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), image, containerWorkDir);
+ String expectedCommand = String
+ .format(expectedCommandTemplate.toString(), containerId, runAsUser,
+ containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
+ containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
+ image, containerWorkDir);
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
@@ -363,13 +374,15 @@ public class TestDockerContainerRuntime {
.append(getExpectedTestCapabilitiesArgumentString())
.append("-v /etc/passwd:/etc/password:ro ")
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ").append("%7$s ")
- .append("bash %8$s/launch_container.sh");
+ .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
+ .append("-v %8$s:%8$s ").append("%9$s ")
+ .append("bash %10$s/launch_container.sh");
String expectedCommand = String
.format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), containerWorkDir,
- containerLogDirs.get(0), image, containerWorkDir);
+ containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
+ containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
+ image, containerWorkDir);
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
@@ -416,13 +429,15 @@ public class TestDockerContainerRuntime {
.append(getExpectedTestCapabilitiesArgumentString())
.append("-v /etc/passwd:/etc/password:ro ")
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ").append("%7$s ")
- .append("bash %8$s/launch_container.sh");
+ .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
+ .append("-v %8$s:%8$s ").append("%9$s ")
+ .append("bash %10$s/launch_container.sh");
String expectedCommand = String
.format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), containerWorkDir,
- containerLogDirs.get(0), image, containerWorkDir);
+ containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
+ containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
+ image, containerWorkDir);
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
@@ -451,14 +466,15 @@ public class TestDockerContainerRuntime {
.append(getExpectedTestCapabilitiesArgumentString())
.append("-v /etc/passwd:/etc/password:ro ")
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ").append("%7$s ")
- .append("bash %8$s/launch_container.sh");
+ .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
+ .append("-v %8$s:%8$s ").append("%9$s ")
+ .append("bash %10$s/launch_container.sh");
expectedCommand = String
.format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), containerWorkDir,
- containerLogDirs.get(0), image, containerWorkDir);
-
+ containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
+ containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
+ image, containerWorkDir);
dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] hadoop git commit: HDFS-10628. Log HDFS Balancer exit message
to its own log. Contributed by Jiayi Zhou.
Posted by su...@apache.org.
HDFS-10628. Log HDFS Balancer exit message to its own log. Contributed by Jiayi Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17c74ac5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17c74ac5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17c74ac5
Branch: refs/heads/YARN-2915
Commit: 17c74ac5f6e58445323e2a9b6095d67faed16b1c
Parents: 80b2f93
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Jul 15 11:42:44 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:31 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17c74ac5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 6ebd6b0..8d94ed5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -609,6 +609,8 @@ public class Balancer {
if (!runDuringUpgrade && nnc.isUpgrading()) {
System.err.println("Balancer exiting as upgrade is not finalized, "
+ "please finalize the HDFS upgrade before running the balancer.");
+ LOG.error("Balancer exiting as upgrade is not finalized, "
+ + "please finalize the HDFS upgrade before running the balancer.");
return newResult(ExitStatus.UNFINALIZED_UPGRADE, bytesLeftToMove, -1);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] hadoop git commit: YARN-5272. Handle queue names consistently
in FairScheduler. (Wilfred Spiegelenburg via rchiang)
Posted by su...@apache.org.
YARN-5272. Handle queue names consistently in FairScheduler. (Wilfred Spiegelenburg via rchiang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46e39856
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46e39856
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46e39856
Branch: refs/heads/YARN-2915
Commit: 46e39856204ad4db42050de191139a233b74a921
Parents: f878bd4
Author: Ray Chiang <rc...@apache.org>
Authored: Fri Jul 15 14:35:50 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
.../fair/AllocationFileLoaderService.java | 4 ++-
.../scheduler/fair/QueueManager.java | 6 ++++-
.../fair/TestAllocationFileLoaderService.java | 28 ++++++++++++++++++++
.../scheduler/fair/TestFairScheduler.java | 11 ++++++++
.../scheduler/fair/TestQueueManager.java | 3 +++
5 files changed, 50 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e39856/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index 661caa7..fab536d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -53,6 +53,7 @@ import org.w3c.dom.NodeList;
import org.w3c.dom.Text;
import org.xml.sax.SAXException;
+import com.google.common.base.CharMatcher;
import com.google.common.annotations.VisibleForTesting;
@Public
@@ -445,7 +446,8 @@ public class AllocationFileLoaderService extends AbstractService {
Set<String> reservableQueues,
Set<String> nonPreemptableQueues)
throws AllocationConfigurationException {
- String queueName = element.getAttribute("name").trim();
+ String queueName = CharMatcher.WHITESPACE.trimFrom(
+ element.getAttribute("name"));
if (queueName.contains(".")) {
throw new AllocationConfigurationException("Bad fair scheduler config "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e39856/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index aeadcf6..8d308dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.xml.sax.SAXException;
+import com.google.common.base.CharMatcher;
import com.google.common.annotations.VisibleForTesting;
/**
* Maintains a list of queues as well as scheduling parameters for each queue,
@@ -457,6 +458,9 @@ public class QueueManager {
*/
@VisibleForTesting
boolean isQueueNameValid(String node) {
- return !node.isEmpty() && node.equals(node.trim());
+ // use the same white space trim as in QueueMetrics() otherwise things fail
+ // guava uses a different definition for whitespace than java.
+ return !node.isEmpty() &&
+ node.equals(CharMatcher.WHITESPACE.trimFrom(node));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e39856/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index cc91ef9..11d4981 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -20,8 +20,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import static org.junit.Assert.*;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.FileWriter;
+import java.io.OutputStreamWriter;
import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -581,6 +584,31 @@ public class TestAllocationFileLoaderService {
}
/**
+ * Verify that you can't have the queue name with just a non breaking
+ * whitespace in the allocations file.
+ */
+ @Test (expected = AllocationConfigurationException.class)
+ public void testQueueNameContainingNBWhitespace() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ PrintWriter out = new PrintWriter(new OutputStreamWriter(
+ new FileOutputStream(ALLOC_FILE), StandardCharsets.UTF_8));
+ out.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"\u00a0\">");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
+ allocLoader.init(conf);
+ ReloadListener confHolder = new ReloadListener();
+ allocLoader.setReloadListener(confHolder);
+ allocLoader.reloadAllocations();
+ }
+
+ /**
* Verify that defaultQueueSchedulingMode can't accept FIFO as a value.
*/
@Test (expected = AllocationConfigurationException.class)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e39856/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 43ebe53..f92af35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -4362,6 +4362,17 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals(1, scheduler.getQueueManager().getLeafQueue("B.C", true)
.getNumRunnableApps());
assertNotNull(scheduler.getSchedulerApp(appAttemptId3));
+
+ // submit app with queue name "A\u00a0" (non-breaking space)
+ ApplicationAttemptId appAttemptId4 = createAppAttemptId(4, 1);
+ AppAddedSchedulerEvent appAddedEvent4 = new AppAddedSchedulerEvent(
+ appAttemptId4.getApplicationId(), "A\u00a0", "user1");
+ scheduler.handle(appAddedEvent4);
+ // submission rejected
+ assertEquals(3, scheduler.getQueueManager().getLeafQueues().size());
+ assertNull(scheduler.getSchedulerApplications().get(appAttemptId4.
+ getApplicationId()));
+ assertNull(scheduler.getSchedulerApp(appAttemptId4));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e39856/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
index a9b27a1..33d4419 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
@@ -131,6 +131,9 @@ public class TestQueueManager {
assertFalse(queueManager.isQueueNameValid(" a"));
assertFalse(queueManager.isQueueNameValid("a "));
assertFalse(queueManager.isQueueNameValid(" a "));
+ assertFalse(queueManager.isQueueNameValid("\u00a0"));
+ assertFalse(queueManager.isQueueNameValid("a\u00a0"));
+ assertFalse(queueManager.isQueueNameValid("\u00a0a\u00a0"));
assertTrue(queueManager.isQueueNameValid("a b"));
assertTrue(queueManager.isQueueNameValid("a"));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] hadoop git commit: Preparing for 3.0.0-alpha2 development
Posted by su...@apache.org.
Preparing for 3.0.0-alpha2 development
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e677d720
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e677d720
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e677d720
Branch: refs/heads/YARN-2915
Commit: e677d7206ccf91b41b658dd2fa93318b6040bbd8
Parents: 84501bf
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Jul 15 19:04:17 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
hadoop-assemblies/pom.xml | 4 ++--
hadoop-build-tools/pom.xml | 2 +-
hadoop-client/pom.xml | 4 ++--
hadoop-common-project/hadoop-annotations/pom.xml | 4 ++--
hadoop-common-project/hadoop-auth-examples/pom.xml | 4 ++--
hadoop-common-project/hadoop-auth/pom.xml | 4 ++--
hadoop-common-project/hadoop-common/pom.xml | 4 ++--
hadoop-common-project/hadoop-kms/pom.xml | 4 ++--
hadoop-common-project/hadoop-minikdc/pom.xml | 4 ++--
hadoop-common-project/hadoop-nfs/pom.xml | 4 ++--
hadoop-common-project/pom.xml | 4 ++--
hadoop-dist/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml | 4 ++--
hadoop-hdfs-project/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml | 4 ++--
.../hadoop-mapreduce-client-common/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml | 4 ++--
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml | 4 ++--
.../hadoop-mapreduce-client-jobclient/pom.xml | 4 ++--
.../hadoop-mapreduce-client-nativetask/pom.xml | 4 ++--
.../hadoop-mapreduce-client-shuffle/pom.xml | 4 ++--
hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 4 ++--
hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml | 4 ++--
hadoop-mapreduce-project/pom.xml | 4 ++--
hadoop-maven-plugins/pom.xml | 2 +-
hadoop-minicluster/pom.xml | 4 ++--
hadoop-project-dist/pom.xml | 4 ++--
hadoop-project/pom.xml | 4 ++--
hadoop-tools/hadoop-archive-logs/pom.xml | 4 ++--
hadoop-tools/hadoop-archives/pom.xml | 4 ++--
hadoop-tools/hadoop-aws/pom.xml | 4 ++--
hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
hadoop-tools/hadoop-azure/pom.xml | 2 +-
hadoop-tools/hadoop-datajoin/pom.xml | 4 ++--
hadoop-tools/hadoop-distcp/pom.xml | 4 ++--
hadoop-tools/hadoop-extras/pom.xml | 4 ++--
hadoop-tools/hadoop-gridmix/pom.xml | 4 ++--
hadoop-tools/hadoop-kafka/pom.xml | 4 ++--
hadoop-tools/hadoop-openstack/pom.xml | 4 ++--
hadoop-tools/hadoop-pipes/pom.xml | 4 ++--
hadoop-tools/hadoop-rumen/pom.xml | 4 ++--
hadoop-tools/hadoop-sls/pom.xml | 4 ++--
hadoop-tools/hadoop-streaming/pom.xml | 4 ++--
hadoop-tools/hadoop-tools-dist/pom.xml | 4 ++--
hadoop-tools/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml | 4 ++--
.../hadoop-yarn-applications-distributedshell/pom.xml | 4 ++--
.../hadoop-yarn-applications-unmanaged-am-launcher/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 ++--
.../hadoop-yarn-server-applicationhistoryservice/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-common/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml | 4 ++--
.../hadoop-yarn-server-resourcemanager/pom.xml | 4 ++--
.../hadoop-yarn-server-sharedcachemanager/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml | 4 ++--
.../hadoop-yarn-server-timeline-pluginstorage/pom.xml | 4 ++--
.../hadoop-yarn-server-timelineservice-hbase-tests/pom.xml | 4 ++--
.../hadoop-yarn-server-timelineservice/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/pom.xml | 4 ++--
hadoop-yarn-project/pom.xml | 4 ++--
pom.xml | 2 +-
73 files changed, 141 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-assemblies/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/pom.xml b/hadoop-assemblies/pom.xml
index 1643ff5..4ff11f6 100644
--- a/hadoop-assemblies/pom.xml
+++ b/hadoop-assemblies/pom.xml
@@ -23,12 +23,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-assemblies</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop Assemblies</name>
<description>Apache Hadoop Assemblies</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-build-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-build-tools/pom.xml b/hadoop-build-tools/pom.xml
index b80cf57..26c8aa6 100644
--- a/hadoop-build-tools/pom.xml
+++ b/hadoop-build-tools/pom.xml
@@ -18,7 +18,7 @@
<parent>
<artifactId>hadoop-main</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-build-tools</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml
index 0e923fe..b330a3d 100644
--- a/hadoop-client/pom.xml
+++ b/hadoop-client/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Apache Hadoop Client</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-common-project/hadoop-annotations/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml
index 21f7b1e..0afa7e3 100644
--- a/hadoop-common-project/hadoop-annotations/pom.xml
+++ b/hadoop-common-project/hadoop-annotations/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Annotations</description>
<name>Apache Hadoop Annotations</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-common-project/hadoop-auth-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth-examples/pom.xml b/hadoop-common-project/hadoop-auth-examples/pom.xml
index c304a74..c36c157 100644
--- a/hadoop-common-project/hadoop-auth-examples/pom.xml
+++ b/hadoop-common-project/hadoop-auth-examples/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth-examples</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>war</packaging>
<name>Apache Hadoop Auth Examples</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-common-project/hadoop-auth/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index 93dceb3..4cbdc49 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Hadoop Auth</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index c21b81e..398bb84 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Common</description>
<name>Apache Hadoop Common</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index aa1b842..f65e94a 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -22,12 +22,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-kms</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>war</packaging>
<name>Apache Hadoop KMS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-common-project/hadoop-minikdc/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-minikdc/pom.xml b/hadoop-common-project/hadoop-minikdc/pom.xml
index 3075cad..e75bc7f 100644
--- a/hadoop-common-project/hadoop-minikdc/pom.xml
+++ b/hadoop-common-project/hadoop-minikdc/pom.xml
@@ -18,13 +18,13 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop MiniKDC</description>
<name>Apache Hadoop MiniKDC</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-common-project/hadoop-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml
index 9ca537e..c943f3c 100644
--- a/hadoop-common-project/hadoop-nfs/pom.xml
+++ b/hadoop-common-project/hadoop-nfs/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-nfs</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Hadoop NFS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-common-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/pom.xml b/hadoop-common-project/pom.xml
index cc0d20c..db54ae5 100644
--- a/hadoop-common-project/pom.xml
+++ b/hadoop-common-project/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Common Project</description>
<name>Apache Hadoop Common Project</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 249790b..4a7feaa 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-dist</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Distribution</description>
<name>Apache Hadoop Distribution</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index 1de9421..b8b531b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop HDFS Client</description>
<name>Apache Hadoop HDFS Client</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index 80ee0ae..df1d63b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -22,12 +22,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-httpfs</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>war</packaging>
<name>Apache Hadoop HttpFS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
index 36fdb35..fa38719 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-native-client</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop HDFS Native Client</description>
<name>Apache Hadoop HDFS Native Client</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index 1827bc80..7bbfe87 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-nfs</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop HDFS-NFS</description>
<name>Apache Hadoop HDFS-NFS</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index e8397e1..b06cd4c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop HDFS</description>
<name>Apache Hadoop HDFS</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
index 2e6b5cc..7fb6c24 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
@@ -20,13 +20,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../../../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop.contrib</groupId>
<artifactId>hadoop-hdfs-bkjournal</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop HDFS BookKeeper Journal</description>
<name>Apache Hadoop HDFS BookKeeper Journal</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-hdfs-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml
index bd2af42..4bb0545 100644
--- a/hadoop-hdfs-project/pom.xml
+++ b/hadoop-hdfs-project/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop HDFS Project</description>
<name>Apache Hadoop HDFS Project</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index a586479..ef0ed6e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce App</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index 0cbfa86..2d969f5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Common</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index 3aaf247..b6b356d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Core</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
index e1bd9b3..eca7571 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-hs-plugins</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce HistoryServer Plugins</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
index 6166e80..1bcaf73 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-hs</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce HistoryServer</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
index d19b075..bbbe1b2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce JobClient</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 19dedbc..0520245 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-nativetask</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce NativeTask</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
index abf5f5b..856d074 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Shuffle</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
index 9a05f22..7eabd40 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Client</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
index 9694dc8..3fbd487 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-examples</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop MapReduce Examples</description>
<name>Apache Hadoop MapReduce Examples</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-mapreduce-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 4a705f5..5fd346d 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Hadoop MapReduce</name>
<url>http://hadoop.apache.org/mapreduce/</url>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-maven-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index bff2a84..48ee5d5 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-minicluster/pom.xml b/hadoop-minicluster/pom.xml
index 47e8598..473ee35 100644
--- a/hadoop-minicluster/pom.xml
+++ b/hadoop-minicluster/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Apache Hadoop Mini-Cluster</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index ba4c0a8..0357269 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Project Dist POM</description>
<name>Apache Hadoop Project Dist POM</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 0a7a16d..df13d4a 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-main</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Project POM</description>
<name>Apache Hadoop Project POM</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-archive-logs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archive-logs/pom.xml b/hadoop-tools/hadoop-archive-logs/pom.xml
index c4c2dbc..714f0d9 100644
--- a/hadoop-tools/hadoop-archive-logs/pom.xml
+++ b/hadoop-tools/hadoop-archive-logs/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-archive-logs</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Archive Logs</description>
<name>Apache Hadoop Archive Logs</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-archives/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml
index 40d64bd..d7def7b 100644
--- a/hadoop-tools/hadoop-archives/pom.xml
+++ b/hadoop-tools/hadoop-archives/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-archives</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Archives</description>
<name>Apache Hadoop Archives</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 7c25e60..8147d37 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-aws</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop Amazon Web Services support</name>
<description>
This module contains code to support integration with Amazon Web Services.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-azure-datalake/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
index d2161c7..c07a1d7 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-azure/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index 02a1240..48f9043 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-azure</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-datajoin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml
index 36e84bc..40b165d 100644
--- a/hadoop-tools/hadoop-datajoin/pom.xml
+++ b/hadoop-tools/hadoop-datajoin/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-datajoin</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Data Join</description>
<name>Apache Hadoop Data Join</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-distcp/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml
index 2cec22f..a07406b 100644
--- a/hadoop-tools/hadoop-distcp/pom.xml
+++ b/hadoop-tools/hadoop-distcp/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Distributed Copy</description>
<name>Apache Hadoop Distributed Copy</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-extras/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml
index 8cd95f9..fbb75a7 100644
--- a/hadoop-tools/hadoop-extras/pom.xml
+++ b/hadoop-tools/hadoop-extras/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-extras</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Extras</description>
<name>Apache Hadoop Extras</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-gridmix/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml
index 5699796..a1a8b0c 100644
--- a/hadoop-tools/hadoop-gridmix/pom.xml
+++ b/hadoop-tools/hadoop-gridmix/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-gridmix</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Gridmix</description>
<name>Apache Hadoop Gridmix</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-kafka/pom.xml b/hadoop-tools/hadoop-kafka/pom.xml
index 839d134..ea77979 100644
--- a/hadoop-tools/hadoop-kafka/pom.xml
+++ b/hadoop-tools/hadoop-kafka/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-kafka</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop Kafka Library support</name>
<description>
This module contains code to support integration with Kafka.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-openstack/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml
index 86417c7..7e7e4fd 100644
--- a/hadoop-tools/hadoop-openstack/pom.xml
+++ b/hadoop-tools/hadoop-openstack/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-openstack</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop OpenStack support</name>
<description>
This module contains code to support integration with OpenStack.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-pipes/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml
index 1f427dd..5be72e7 100644
--- a/hadoop-tools/hadoop-pipes/pom.xml
+++ b/hadoop-tools/hadoop-pipes/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-pipes</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Pipes</description>
<name>Apache Hadoop Pipes</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-rumen/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml
index 622a302..f5acf4e 100644
--- a/hadoop-tools/hadoop-rumen/pom.xml
+++ b/hadoop-tools/hadoop-rumen/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-rumen</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Rumen</description>
<name>Apache Hadoop Rumen</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-sls/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml
index f58997d..e361ba3 100644
--- a/hadoop-tools/hadoop-sls/pom.xml
+++ b/hadoop-tools/hadoop-sls/pom.xml
@@ -19,12 +19,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-sls</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Scheduler Load Simulator</description>
<name>Apache Hadoop Scheduler Load Simulator</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml
index 3cde3ae..b0eb5df 100644
--- a/hadoop-tools/hadoop-streaming/pom.xml
+++ b/hadoop-tools/hadoop-streaming/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-streaming</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop MapReduce Streaming</description>
<name>Apache Hadoop MapReduce Streaming</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 9485741..899a945 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-tools-dist</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Tools Dist</description>
<name>Apache Hadoop Tools Dist</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index a2c441b..db002f4 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-tools</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Tools</description>
<name>Apache Hadoop Tools</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
index e4b445d..52e276b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN API</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
index e8cb138..0f696d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications-distributedshell</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN DistributedShell</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
index e0eadfc..ea3f294 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications-unmanaged-am-launcher</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Unmanaged Am Launcher</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index d58607d..a1ff36d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Applications</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
index df15c7c..668c50d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
@@ -17,11 +17,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Client</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index b060d10..4b031c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Common</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
index 46d8a59..886c98f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-registry</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Registry</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
index cc66b0a..f772c77 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
@@ -22,12 +22,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN ApplicationHistoryService</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 95bb89b..f8330e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Server Common</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index 4a69c9c..a2f81d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN NodeManager</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index 834dc3d..56fb075 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN ResourceManager</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
index 69e7a55..4918276 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
@@ -17,11 +17,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-sharedcachemanager</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN SharedCacheManager</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
index 6bd54c2..58bed0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Server Tests</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
index 8512571..1b8b7f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
@@ -22,12 +22,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timeline-pluginstorage</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Timeline Plugin Storage</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
index 904badd..a7b22a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
@@ -22,12 +22,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice-hbase-tests</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Timeline Service HBase tests</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index ab3e962..004972a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -22,12 +22,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Timeline Service</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
index 2f0985c..844d8d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Web Proxy</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
index 6e1f461..416c511 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Server</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
index 5e3367d..6436109 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-site</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<name>Apache Hadoop YARN Site</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/hadoop-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index d367f83..b276801 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -16,12 +16,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Hadoop YARN</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/hadoop-yarn-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index d3d0815..380b515 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-project</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Hadoop YARN Project</name>
<url>http://hadoop.apache.org/yarn/</url>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e677d720/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ac3bec6..fb50419 100644
--- a/pom.xml
+++ b/pom.xml
@@ -18,7 +18,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-main</artifactId>
- <version>3.0.0-alpha1-SNAPSHOT</version>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
<description>Apache Hadoop Main</description>
<name>Apache Hadoop Main</name>
<packaging>pom</packaging>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] hadoop git commit: YARN-5362.
TestRMRestart#testFinishedAppRemovalAfterRMRestart can fail. Contributed by
sandflee.
Posted by su...@apache.org.
YARN-5362. TestRMRestart#testFinishedAppRemovalAfterRMRestart can fail. Contributed by sandflee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2956e8d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2956e8d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2956e8d
Branch: refs/heads/YARN-2915
Commit: f2956e8da31f7fd84e514715a538a4d5522f6532
Parents: 17893a2
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed Jul 13 19:12:16 2016 +0530
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:29 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2956e8d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 1a79479..b271b37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -1648,6 +1648,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
RMApp app1 = rm2.submitApp(200);
MockAM am1 = launchAM(app1, rm2, nm1);
finishApplicationMaster(app1, rm2, nm1, am1);
+ rm2.drainEvents();
// the first app0 get kicked out from both rmContext and state store
Assert.assertNull(rm2.getRMContext().getRMApps()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] hadoop git commit: HADOOP-13289. Remove unused variables in
TestFairCallQueue. Contributed by Ye Zhou.
Posted by su...@apache.org.
HADOOP-13289. Remove unused variables in TestFairCallQueue. Contributed by Ye Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/16872b3c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/16872b3c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/16872b3c
Branch: refs/heads/YARN-2915
Commit: 16872b3cd5361710ce9bdfc19c03c831d4a0f5bb
Parents: d28d3ba
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Jul 14 23:13:55 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:31 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/ipc/TestFairCallQueue.java | 10 ----------
1 file changed, 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16872b3c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
index d5e5f90..96dea80 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
@@ -32,7 +32,6 @@ import java.util.concurrent.BlockingQueue;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.conf.Configuration;
-import org.mockito.Matchers;
public class TestFairCallQueue extends TestCase {
private FairCallQueue<Schedulable> fcq;
@@ -52,14 +51,6 @@ public class TestFairCallQueue extends TestCase {
return mockCall(id, 0);
}
- // A scheduler which always schedules into priority zero
- private RpcScheduler alwaysZeroScheduler;
- {
- RpcScheduler sched = mock(RpcScheduler.class);
- when(sched.getPriorityLevel(Matchers.<Schedulable>any())).thenReturn(0); // always queue 0
- alwaysZeroScheduler = sched;
- }
-
@SuppressWarnings("deprecation")
public void setUp() {
Configuration conf = new Configuration();
@@ -124,7 +115,6 @@ public class TestFairCallQueue extends TestCase {
public void testOfferSucceedsWhenScheduledLowPriority() {
// Scheduler will schedule into queue 0 x 5, then queue 1
- RpcScheduler sched = mock(RpcScheduler.class);
int mockedPriorities[] = {0, 0, 0, 0, 0, 1, 0};
for (int i = 0; i < 5; i++) { assertTrue(fcq.offer(mockCall("c", mockedPriorities[i]))); }
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] hadoop git commit: YARN-5364. timelineservice modules have
indirect dependencies on mapreduce artifacts. Contributed by Sangjin Lee
Posted by su...@apache.org.
YARN-5364. timelineservice modules have indirect dependencies on mapreduce artifacts. Contributed by Sangjin Lee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0eaa937
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0eaa937
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0eaa937
Branch: refs/heads/YARN-2915
Commit: e0eaa93781314889c785de7e705182f46c7d01a4
Parents: 320221b
Author: Naganarasimha <na...@apache.org>
Authored: Thu Jul 14 01:41:35 2016 +0530
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../pom.xml | 56 ++++++++++++++++++++
.../hadoop-yarn-server-timelineservice/pom.xml | 36 +++++++++++++
2 files changed, 92 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0eaa937/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
index 920edac..904badd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
@@ -164,6 +164,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -180,6 +184,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -204,6 +212,14 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -217,6 +233,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -242,6 +262,14 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -265,6 +293,14 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -285,6 +321,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -306,6 +346,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -359,6 +403,18 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0eaa937/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index dd15f44..ab3e962 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -134,21 +134,57 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-core</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] hadoop git commit: YARN-4996. Make
TestNMReconnect.testCompareRMNodeAfterReconnect() scheduler agnostic (Kai
Sasaki via Varun Saxena)
Posted by su...@apache.org.
YARN-4996. Make TestNMReconnect.testCompareRMNodeAfterReconnect() scheduler agnostic (Kai Sasaki via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82c4d7dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82c4d7dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82c4d7dc
Branch: refs/heads/YARN-2915
Commit: 82c4d7dcd41afe87088cbd3fdd2ca59b71dc9cd8
Parents: 5cab56c
Author: Varun Saxena <va...@apache.org>
Authored: Tue Jul 19 16:03:28 2016 +0530
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
.../ParameterizedSchedulerTestBase.java | 20 ++++++++++++++++++++
.../resourcetracker/TestNMReconnect.java | 13 +++++++++----
2 files changed, 29 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82c4d7dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
index 3f2bfc9..c29e703 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
@@ -38,6 +39,7 @@ public abstract class ParameterizedSchedulerTestBase {
private SchedulerType schedulerType;
private YarnConfiguration conf = null;
+ private AbstractYarnScheduler scheduler = null;
public enum SchedulerType {
CAPACITY, FAIR
@@ -58,8 +60,11 @@ public abstract class ParameterizedSchedulerTestBase {
if (schedulerClass == FairScheduler.class) {
schedulerType = SchedulerType.FAIR;
configureFairScheduler(conf);
+ scheduler = new FairScheduler();
} else if (schedulerClass == CapacityScheduler.class) {
schedulerType = SchedulerType.CAPACITY;
+ scheduler = new CapacityScheduler();
+ ((CapacityScheduler)scheduler).setConf(conf);
}
}
@@ -87,4 +92,19 @@ public abstract class ParameterizedSchedulerTestBase {
public SchedulerType getSchedulerType() {
return schedulerType;
}
+
+ /**
+ * Return a scheduler configured by {@code YarnConfiguration.RM_SCHEDULER}
+ *
+ * <p>The scheduler is configured by {@link #configureScheduler()}.
+ * Client test code can obtain the scheduler with this getter method.
+ * Schedulers supported by this class are {@link FairScheduler} or
+ * {@link CapacityScheduler}. </p>
+ *
+ * @return The scheduler configured by
+ * {@code YarnConfiguration.RM_SCHEDULER}
+ */
+ public AbstractYarnScheduler getScheduler() {
+ return scheduler;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82c4d7dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
index cc31f62..2c926d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase;
import org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManager;
@@ -47,7 +48,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
@@ -57,7 +58,12 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-public class TestNMReconnect {
+/**
+ * TestNMReconnect run tests against the scheduler set by
+ * {@link ParameterizedSchedulerTestBase} which is configured
+ * in {@link YarnConfiguration}.
+ */
+public class TestNMReconnect extends ParameterizedSchedulerTestBase {
private static final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
@@ -147,9 +153,8 @@ public class TestNMReconnect {
@Test
public void testCompareRMNodeAfterReconnect() throws Exception {
+ AbstractYarnScheduler scheduler = getScheduler();
Configuration yarnConf = new YarnConfiguration();
- CapacityScheduler scheduler = new CapacityScheduler();
- scheduler.setConf(yarnConf);
ConfigurationProvider configurationProvider =
ConfigurationProviderFactory.getConfigurationProvider(yarnConf);
configurationProvider.init(yarnConf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] hadoop git commit: HADOOP-13368.
DFSOpsCountStatistics$OpType#fromSymbol and s3a.Statistic#fromSymbol should
be O(1) operation. Contributed by Mingliang Liu.
Posted by su...@apache.org.
HADOOP-13368. DFSOpsCountStatistics$OpType#fromSymbol and s3a.Statistic#fromSymbol should be O(1) operation. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a483d63
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a483d63
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a483d63
Branch: refs/heads/YARN-2915
Commit: 4a483d63087fb3b0e446b42473c981a197765029
Parents: 8b105da
Author: Jitendra Pandey <ji...@apache.org>
Authored: Fri Jul 15 14:28:53 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/DFSOpsCountStatistics.java | 18 ++++++++++--------
.../hadoop/hdfs/TestDFSOpsCountStatistics.java | 2 ++
.../org/apache/hadoop/fs/s3a/Statistic.java | 20 ++++++++++++--------
3 files changed, 24 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a483d63/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
index 83d880a..d631dd4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import java.util.EnumMap;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
@@ -88,6 +89,14 @@ public class DFSOpsCountStatistics extends StorageStatistics {
TRUNCATE(CommonStatisticNames.OP_TRUNCATE),
UNSET_STORAGE_POLICY("op_unset_storage_policy");
+ private static final Map<String, OpType> SYMBOL_MAP =
+ new HashMap<>(OpType.values().length);
+ static {
+ for (OpType opType : values()) {
+ SYMBOL_MAP.put(opType.getSymbol(), opType);
+ }
+ }
+
private final String symbol;
OpType(String symbol) {
@@ -99,14 +108,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
}
public static OpType fromSymbol(String symbol) {
- if (symbol != null) {
- for (OpType opType : values()) {
- if (opType.getSymbol().equals(symbol)) {
- return opType;
- }
- }
- }
- return null;
+ return SYMBOL_MAP.get(symbol);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a483d63/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java
index d63ef10..5ccee3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java
@@ -111,12 +111,14 @@ public class TestDFSOpsCountStatistics {
@Test
public void testGetLong() {
+ assertNull(statistics.getLong(null));
assertNull(statistics.getLong(NO_SUCH_OP));
verifyStatistics();
}
@Test
public void testIsTracked() {
+ assertFalse(statistics.isTracked(null));
assertFalse(statistics.isTracked(NO_SUCH_OP));
final Iterator<LongStatistic> iter = statistics.getLongStatistics();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a483d63/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index 3c205f3..36d163c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.fs.s3a;
import org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames;
+import java.util.HashMap;
+import java.util.Map;
+
/**
* Statistic which are collected in S3A.
* These statistics are available at a low level in {@link S3AStorageStatistics}
@@ -105,6 +108,14 @@ public enum Statistic {
STREAM_ABORT_BYTES_DISCARDED("stream_bytes_discarded_in_abort",
"Count of bytes discarded by aborting the stream");
+ private static final Map<String, Statistic> SYMBOL_MAP =
+ new HashMap<>(Statistic.values().length);
+ static {
+ for (Statistic stat : values()) {
+ SYMBOL_MAP.put(stat.getSymbol(), stat);
+ }
+ }
+
Statistic(String symbol, String description) {
this.symbol = symbol;
this.description = description;
@@ -123,14 +134,7 @@ public enum Statistic {
* @return the value or null.
*/
public static Statistic fromSymbol(String symbol) {
- if (symbol != null) {
- for (Statistic opType : values()) {
- if (opType.getSymbol().equals(symbol)) {
- return opType;
- }
- }
- }
- return null;
+ return SYMBOL_MAP.get(symbol);
}
public String getDescription() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] hadoop git commit: YARN-5299. Log Docker run command when
container fails. Contributed by Varun Vasudev.
Posted by su...@apache.org.
YARN-5299. Log Docker run command when container fails. Contributed by Varun Vasudev.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efdd81b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efdd81b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efdd81b1
Branch: refs/heads/YARN-2915
Commit: efdd81b17c67357225ac380b7532f538a86c8bf3
Parents: 288eade
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Thu Jul 14 11:48:10 2016 +0530
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../linux/runtime/DockerLinuxContainerRuntime.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efdd81b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 4398e54..e1a162e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -416,6 +416,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
if (tcCommandFile != null) {
launchOp.appendArgs(tcCommandFile);
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Launching container with cmd: " + runCommand
+ .getCommandWithArguments());
+ }
try {
privilegedOperationExecutor.executePrivilegedOperation(null,
@@ -423,6 +427,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
false, false);
} catch (PrivilegedOperationException e) {
LOG.warn("Launch container failed. Exception: ", e);
+ LOG.info("Docker command used: " + runCommand.getCommandWithArguments());
throw new ContainerExecutionException("Launch container failed", e
.getExitCode(), e.getOutput(), e.getErrorOutput());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] hadoop git commit: YARN-4759. Fix signal handling for docker
containers. Contributed by Shane Kumpf.
Posted by su...@apache.org.
YARN-4759. Fix signal handling for docker containers. Contributed by Shane Kumpf.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aedeabca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aedeabca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aedeabca
Branch: refs/heads/YARN-2915
Commit: aedeabcaf91f98176a887ce373f8312d8336f3f6
Parents: 5f9f64f
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu Jul 14 19:27:16 2016 +0530
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 47 ++++++----
.../linux/runtime/docker/DockerStopCommand.java | 39 ++++++++
.../impl/container-executor.c | 35 +++++--
.../runtime/TestDockerContainerRuntime.java | 99 +++++++++++++++++++-
.../runtime/docker/DockerStopCommandTest.java | 55 +++++++++++
5 files changed, 250 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aedeabca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index a1606f6..0cfdd05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
@@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerClient;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStopCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
@@ -445,26 +447,39 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
public void signalContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
Container container = ctx.getContainer();
- PrivilegedOperation signalOp = new PrivilegedOperation(
- PrivilegedOperation.OperationType.SIGNAL_CONTAINER);
+ ContainerExecutor.Signal signal = ctx.getExecutionAttribute(SIGNAL);
+
+ PrivilegedOperation privOp = null;
+ // Handle liveliness checks, send null signal to pid
+ if(ContainerExecutor.Signal.NULL.equals(signal)) {
+ privOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.SIGNAL_CONTAINER);
+ privOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER),
+ ctx.getExecutionAttribute(USER),
+ Integer.toString(PrivilegedOperation.RunAsUserCommand
+ .SIGNAL_CONTAINER.getValue()),
+ ctx.getExecutionAttribute(PID),
+ Integer.toString(ctx.getExecutionAttribute(SIGNAL).getValue()));
+
+ // All other signals handled as docker stop
+ } else {
+ String containerId = ctx.getContainer().getContainerId().toString();
+ DockerStopCommand stopCommand = new DockerStopCommand(containerId);
+ String commandFile = dockerClient.writeCommandToTempFile(stopCommand,
+ containerId);
+ privOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.RUN_DOCKER_CMD);
+ privOp.appendArgs(commandFile);
+ }
- signalOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER),
- ctx.getExecutionAttribute(USER),
- Integer.toString(PrivilegedOperation
- .RunAsUserCommand.SIGNAL_CONTAINER.getValue()),
- ctx.getExecutionAttribute(PID),
- Integer.toString(ctx.getExecutionAttribute(SIGNAL).getValue()));
+ //Some failures here are acceptable. Let the calling executor decide.
+ privOp.disableFailureLogging();
try {
- PrivilegedOperationExecutor executor = PrivilegedOperationExecutor
- .getInstance(conf);
-
- executor.executePrivilegedOperation(null,
- signalOp, null, container.getLaunchContext().getEnvironment(),
- false, true);
+ privilegedOperationExecutor.executePrivilegedOperation(null,
+ privOp, null, container.getLaunchContext().getEnvironment(),
+ false, false);
} catch (PrivilegedOperationException e) {
- LOG.warn("Signal container failed. Exception: ", e);
-
throw new ContainerExecutionException("Signal container failed", e
.getExitCode(), e.getOutput(), e.getErrorOutput());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aedeabca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
new file mode 100644
index 0000000..e9d6c43
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
@@ -0,0 +1,39 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+
+/**
+ * Encapsulates the docker stop command and its command
+ * line arguments.
+ */
+public class DockerStopCommand extends DockerCommand {
+ private static final String STOP_COMMAND = "stop";
+
+ public DockerStopCommand(String containerName) {
+ super(STOP_COMMAND);
+ super.addCommandArguments(containerName);
+ }
+
+ public DockerStopCommand setGracePeriod(int value) {
+ super.addCommandArguments("--time=" + Integer.toString(value));
+ return this;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aedeabca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 14dd15e..e40101c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -283,7 +283,14 @@ static int write_pid_to_file_as_nm(const char* pid_file, pid_t pid) {
* Write the exit code of the container into the exit code file
* exit_code_file: Path to exit code file where exit code needs to be written
*/
-static int write_exit_code_file(const char* exit_code_file, int exit_code) {
+static int write_exit_code_file_as_nm(const char* exit_code_file, int exit_code) {
+ uid_t user = geteuid();
+ gid_t group = getegid();
+ if (change_effective_user(nm_uid, nm_gid) != 0) {
+ fprintf(ERRORFILE, "Could not change to effective users %d, %d\n", nm_uid, nm_gid);
+ fflush(ERRORFILE);
+ return -1;
+ }
char *tmp_ecode_file = concatenate("%s.tmp", "exit_code_path", 1,
exit_code_file);
if (tmp_ecode_file == NULL) {
@@ -320,6 +327,13 @@ static int write_exit_code_file(const char* exit_code_file, int exit_code) {
return -1;
}
+ // always change back
+ if (change_effective_user(user, group) != 0) {
+ fprintf(ERRORFILE,
+ "Could not change to effective users %d, %d\n", user, group);
+ fflush(ERRORFILE);
+ return -1;
+ }
free(tmp_ecode_file);
return 0;
}
@@ -357,11 +371,8 @@ static int wait_and_get_exit_code(pid_t pid) {
static int wait_and_write_exit_code(pid_t pid, const char* exit_code_file) {
int exit_code = -1;
- if (change_effective_user(nm_uid, nm_gid) != 0) {
- return -1;
- }
exit_code = wait_and_get_exit_code(pid);
- if (write_exit_code_file(exit_code_file, exit_code) < 0) {
+ if (write_exit_code_file_as_nm(exit_code_file, exit_code) < 0) {
return -1;
}
@@ -1240,6 +1251,9 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
int BUFFER_SIZE = 4096;
char buffer[BUFFER_SIZE];
+ gid_t user_gid = getegid();
+ uid_t prev_uid = geteuid();
+
char *docker_command = parse_docker_command_file(command_file);
char *docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg);
if (docker_binary == NULL) {
@@ -1255,7 +1269,6 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
fflush(ERRORFILE);
goto cleanup;
}
- gid_t user_gid = getegid();
fprintf(LOGFILE, "Creating local dirs...\n");
exit_code = create_local_dirs(user, app_id, container_id,
@@ -1400,11 +1413,19 @@ cleanup:
//clean up docker command file
unlink(command_file);
- if (exit_code_file != NULL && write_exit_code_file(exit_code_file, exit_code) < 0) {
+ if (exit_code_file != NULL && write_exit_code_file_as_nm(exit_code_file, exit_code) < 0) {
fprintf (ERRORFILE,
"Could not write exit code to file %s.\n", exit_code_file);
fflush(ERRORFILE);
}
+
+ // Drop root privileges
+ if (change_effective_user(prev_uid, user_gid) != 0) {
+ fprintf(ERRORFILE,
+ "Could not change to effective users %d, %d\n", prev_uid, user_gid);
+ fflush(ERRORFILE);
+ }
+
#if HAVE_FCLOSEALL
fcloseall();
#else
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aedeabca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index bb3f0f8..a29b174 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -23,10 +23,12 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
@@ -92,6 +94,7 @@ public class TestDockerContainerRuntime {
private final String submittingUser = "anakin";
private final String whitelistedUser = "yoda";
private String[] testCapabilities;
+ private final String signalPid = "1234";
@Before
public void setup() {
@@ -187,7 +190,7 @@ public class TestDockerContainerRuntime {
}
@SuppressWarnings("unchecked")
- private PrivilegedOperation capturePrivilegedOperationAndVerifyArgs()
+ private PrivilegedOperation capturePrivilegedOperation()
throws PrivilegedOperationException {
ArgumentCaptor<PrivilegedOperation> opCaptor = ArgumentCaptor.forClass(
PrivilegedOperation.class);
@@ -203,7 +206,14 @@ public class TestDockerContainerRuntime {
// hence, reset mock here
Mockito.reset(mockExecutor);
- PrivilegedOperation op = opCaptor.getValue();
+ return opCaptor.getValue();
+ }
+
+ @SuppressWarnings("unchecked")
+ private PrivilegedOperation capturePrivilegedOperationAndVerifyArgs()
+ throws PrivilegedOperationException {
+
+ PrivilegedOperation op = capturePrivilegedOperation();
Assert.assertEquals(PrivilegedOperation.OperationType
.LAUNCH_DOCKER_CONTAINER, op.getOperationType());
@@ -784,4 +794,89 @@ public class TestDockerContainerRuntime {
":ro "));
}
+ @Test
+ public void testContainerLivelinessCheck()
+ throws ContainerExecutionException, PrivilegedOperationException {
+
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ builder.setExecutionAttribute(RUN_AS_USER, runAsUser)
+ .setExecutionAttribute(USER, user)
+ .setExecutionAttribute(PID, signalPid)
+ .setExecutionAttribute(SIGNAL, ContainerExecutor.Signal.NULL);
+ runtime.initialize(getConfigurationWithMockContainerExecutor());
+ runtime.signalContainer(builder.build());
+
+ PrivilegedOperation op = capturePrivilegedOperation();
+ Assert.assertEquals(op.getOperationType(),
+ PrivilegedOperation.OperationType.SIGNAL_CONTAINER);
+ Assert.assertEquals("run_as_user", op.getArguments().get(0));
+ Assert.assertEquals("user", op.getArguments().get(1));
+ Assert.assertEquals("2", op.getArguments().get(2));
+ Assert.assertEquals("1234", op.getArguments().get(3));
+ Assert.assertEquals("0", op.getArguments().get(4));
+ }
+
+ @Test
+ public void testDockerStopOnTermSignal()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException {
+ List<String> dockerCommands = getDockerCommandsForSignal(
+ ContainerExecutor.Signal.TERM);
+ Assert.assertEquals(1, dockerCommands.size());
+ Assert.assertEquals("stop container_id", dockerCommands.get(0));
+ }
+
+ @Test
+ public void testDockerStopOnKillSignal()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException {
+ List<String> dockerCommands = getDockerCommandsForSignal(
+ ContainerExecutor.Signal.KILL);
+ Assert.assertEquals(1, dockerCommands.size());
+ Assert.assertEquals("stop container_id", dockerCommands.get(0));
+ }
+
+ @Test
+ public void testDockerStopOnQuitSignal()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException {
+ List<String> dockerCommands = getDockerCommandsForSignal(
+ ContainerExecutor.Signal.QUIT);
+ Assert.assertEquals(1, dockerCommands.size());
+ Assert.assertEquals("stop container_id", dockerCommands.get(0));
+ }
+
+ private List<String> getDockerCommandsForSignal(
+ ContainerExecutor.Signal signal)
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException {
+
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ builder.setExecutionAttribute(RUN_AS_USER, runAsUser)
+ .setExecutionAttribute(USER, user)
+ .setExecutionAttribute(PID, signalPid)
+ .setExecutionAttribute(SIGNAL, signal);
+ runtime.initialize(getConfigurationWithMockContainerExecutor());
+ runtime.signalContainer(builder.build());
+
+ PrivilegedOperation op = capturePrivilegedOperation();
+ Assert.assertEquals(op.getOperationType(),
+ PrivilegedOperation.OperationType.RUN_DOCKER_CMD);
+ String dockerCommandFile = op.getArguments().get(0);
+ return Files.readAllLines(Paths.get(dockerCommandFile),
+ Charset.forName("UTF-8"));
+ }
+
+ private Configuration getConfigurationWithMockContainerExecutor() {
+ File f = new File("./src/test/resources/mock-container-executor");
+ if(!FileUtil.canExecute(f)) {
+ FileUtil.setExecutable(f, true);
+ }
+ String executorPath = f.getAbsolutePath();
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
+ return conf;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aedeabca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommandTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommandTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommandTest.java
new file mode 100644
index 0000000..b53e58b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommandTest.java
@@ -0,0 +1,55 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+
+import static org.junit.Assert.*;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests the docker stop command and its command
+ * line arguments.
+ */
+public class DockerStopCommandTest {
+
+ private DockerStopCommand dockerStopCommand;
+
+ private static final int GRACE_PERIOD = 10;
+ private static final String CONTAINER_NAME = "foo";
+
+ @Before
+ public void setup() {
+ dockerStopCommand = new DockerStopCommand(CONTAINER_NAME);
+ }
+
+ @Test
+ public void testGetCommandOption() {
+ assertEquals("stop", dockerStopCommand.getCommandOption());
+ }
+
+ @Test
+ public void testSetGracePeriod() throws Exception {
+ dockerStopCommand.setGracePeriod(GRACE_PERIOD);
+ assertEquals("stop foo --time=10",
+ dockerStopCommand.getCommandWithArguments());
+
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] hadoop git commit: HADOOP-13298. Fix the leftover L&N files
in hadoop-build-tools/src/main/resources/META-INF/. (ozawa)
Posted by su...@apache.org.
HADOOP-13298. Fix the leftover L&N files in hadoop-build-tools/src/main/resources/META-INF/. (ozawa)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84501bf2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84501bf2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84501bf2
Branch: refs/heads/YARN-2915
Commit: 84501bf29c4107522e0ec78d37f8883df1b88d99
Parents: 46e3985
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Sat Jul 16 08:01:16 2016 +0900
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
hadoop-build-tools/pom.xml | 44 ++++++++++++++++++++++++++++++++++++++++-
pom.xml | 26 ------------------------
2 files changed, 43 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84501bf2/hadoop-build-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-build-tools/pom.xml b/hadoop-build-tools/pom.xml
index e84c94f..b80cf57 100644
--- a/hadoop-build-tools/pom.xml
+++ b/hadoop-build-tools/pom.xml
@@ -29,6 +29,19 @@
<failIfNoTests>false</failIfNoTests>
</properties>
<build>
+ <resources>
+ <resource>
+ <directory>${project.basedir}/target/extra-resources</directory>
+ <targetPath>META-INF</targetPath>
+ <includes>
+ <include>LICENSE.txt</include>
+ <include>NOTICE.txt</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${project.basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -37,17 +50,46 @@
<skip>true</skip>
</configuration>
</plugin>
+ <!-- copy L&N files to target/extra-resources -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.basedir}/target/extra-resources</outputDirectory>
+ <resources>
+ <resource>
+ <directory>../</directory>
+ <includes>
+ <include>LICENSE.txt</include>
+ <include>NOTICE.txt</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- add entries for L&N files to remote-resources.xml in jar file -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-remote-resources-plugin</artifactId>
<executions>
<execution>
+ <phase>process-resources</phase>
<goals>
<goal>bundle</goal>
</goals>
</execution>
</executions>
<configuration>
+ <resourcesDirectory>${project.build.outputDirectory}</resourcesDirectory>
<includes>
<include>META-INF/LICENSE.txt</include>
<include>META-INF/NOTICE.txt</include>
@@ -69,4 +111,4 @@
</plugin>
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84501bf2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6e66f8b..ac3bec6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -413,32 +413,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<artifactId>dependency-check-maven</artifactId>
<version>${dependency-check-maven.version}</version>
</plugin>
- <plugin>
- <!-- Copy license and notice files into hadoop-resource-bundle's
- metadata, which will then be bundled into jars.
- -->
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-files</id>
- <phase>process-resources</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <copy todir="${project.build.directory}/../hadoop-build-tools/src/main/resources/META-INF/">
- <fileset dir="${project.build.directory}/../">
- <include name="LICENSE.txt"/>
- <include name="NOTICE.txt"/>
- </fileset>
- </copy>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] hadoop git commit: HDFS-10387.
DataTransferProtocol#writeBlock missing some javadocs (John Zhuge via lei)
Posted by su...@apache.org.
HDFS-10387. DataTransferProtocol#writeBlock missing some javadocs (John Zhuge via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/329490ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/329490ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/329490ab
Branch: refs/heads/YARN-2915
Commit: 329490ab9372c4914b7e097040b9c86bade13c94
Parents: ef14fa6
Author: Lei Xu <le...@apache.org>
Authored: Tue Jul 12 16:16:03 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:29 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/329490ab/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
index 94f8906..6c5883c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
@@ -94,6 +94,11 @@ public interface DataTransferProtocol {
* @param minBytesRcvd minimum number of bytes received.
* @param maxBytesRcvd maximum number of bytes received.
* @param latestGenerationStamp the latest generation stamp of the block.
+ * @param requestedChecksum the requested checksum mechanism
+ * @param cachingStrategy the caching strategy
+ * @param allowLazyPersist hint to the DataNode that the block can be
+ * allocated on transient storage i.e. memory and
+ * written to disk lazily
* @param pinning whether to pin the block, so Balancer won't move it.
* @param targetPinnings whether to pin the block on target datanode
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] hadoop git commit: HADOOP-13315. FileContext#umask is not
initialized properly. (John Zhuge via lei)
Posted by su...@apache.org.
HADOOP-13315. FileContext#umask is not initialized properly. (John Zhuge via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bc7bfe0f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bc7bfe0f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bc7bfe0f
Branch: refs/heads/YARN-2915
Commit: bc7bfe0f7fe5c50a579d3c8a026e580b3b44aacb
Parents: 329490a
Author: Lei Xu <le...@apache.org>
Authored: Tue Jul 12 16:49:32 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:29 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/FileContext.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc7bfe0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index d1afd41..7832186 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -228,7 +228,7 @@ public class FileContext {
private FileContext(final AbstractFileSystem defFs,
final FsPermission theUmask, final Configuration aConf) {
defaultFS = defFs;
- umask = FsPermission.getUMask(aConf);
+ umask = theUmask;
conf = aConf;
tracer = FsTracer.get(aConf);
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] hadoop git commit: HADOOP-13073 RawLocalFileSystem does not
react on changing umask. Contributed by Andras Bokor
Posted by su...@apache.org.
HADOOP-13073 RawLocalFileSystem does not react on changing umask. Contributed by Andras Bokor
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/825310a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/825310a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/825310a8
Branch: refs/heads/YARN-2915
Commit: 825310a820e3fa479afa9e562fa21e161e3193af
Parents: 5ff6461
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jul 18 14:38:35 2016 +0100
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/RawLocalFileSystem.java | 7 +-
.../fs/TestLocalFileSystemPermission.java | 102 ++++++++++++++-----
2 files changed, 78 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/825310a8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index cc41f4a..0fcddcf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -64,8 +64,6 @@ public class RawLocalFileSystem extends FileSystem {
// Temporary workaround for HADOOP-9652.
private static boolean useDeprecatedFileStatus = true;
- private FsPermission umask;
-
@VisibleForTesting
public static void useStatIfAvailable() {
useDeprecatedFileStatus = !Stat.isAvailable();
@@ -99,7 +97,6 @@ public class RawLocalFileSystem extends FileSystem {
public void initialize(URI uri, Configuration conf) throws IOException {
super.initialize(uri, conf);
setConf(conf);
- umask = FsPermission.getUMask(conf);
}
/*******************************************************
@@ -233,7 +230,7 @@ public class RawLocalFileSystem extends FileSystem {
if (permission == null) {
this.fos = new FileOutputStream(file, append);
} else {
- permission = permission.applyUMask(umask);
+ permission = permission.applyUMask(FsPermission.getUMask(getConf()));
if (Shell.WINDOWS && NativeIO.isAvailable()) {
this.fos = NativeIO.Windows.createFileOutputStreamWithMode(file,
append, permission.toShort());
@@ -510,7 +507,7 @@ public class RawLocalFileSystem extends FileSystem {
if (permission == null) {
permission = FsPermission.getDirDefault();
}
- permission = permission.applyUMask(umask);
+ permission = permission.applyUMask(FsPermission.getUMask(getConf()));
if (Shell.WINDOWS && NativeIO.isAvailable()) {
try {
NativeIO.Windows.createDirectoryWithMode(p2f, permission.toShort());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/825310a8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
index e37de19..11e94a7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.fs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.*;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Level;
import org.apache.hadoop.util.Shell;
@@ -28,11 +27,21 @@ import java.io.*;
import java.util.*;
import junit.framework.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
/**
* This class tests the local file system via the FileSystem abstraction.
*/
public class TestLocalFileSystemPermission extends TestCase {
+
+ public static final Logger LOGGER =
+ LoggerFactory.getLogger(TestFcLocalFsPermission.class);
+
static final String TEST_PATH_PREFIX = GenericTestUtils.getTempPath(
TestLocalFileSystemPermission.class.getSimpleName());
@@ -64,12 +73,12 @@ public class TestLocalFileSystemPermission extends TestCase {
public void testLocalFSDirsetPermission() throws IOException {
if (Path.WINDOWS) {
- System.out.println("Cannot run test for Windows");
+ LOGGER.info("Cannot run test for Windows");
return;
}
- Configuration conf = new Configuration();
+ LocalFileSystem localfs = FileSystem.getLocal(new Configuration());
+ Configuration conf = localfs.getConf();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "044");
- LocalFileSystem localfs = FileSystem.getLocal(conf);
Path dir = new Path(TEST_PATH_PREFIX + "dir");
localfs.mkdirs(dir);
try {
@@ -78,8 +87,7 @@ public class TestLocalFileSystemPermission extends TestCase {
FsPermission.getDirDefault().applyUMask(FsPermission.getUMask(conf)),
initialPermission);
} catch(Exception e) {
- System.out.println(StringUtils.stringifyException(e));
- System.out.println("Cannot run test");
+ LOGGER.error("Cannot run test", e);
return;
}
@@ -90,8 +98,7 @@ public class TestLocalFileSystemPermission extends TestCase {
FsPermission initialPermission = getPermission(localfs, dir1);
assertEquals(perm.applyUMask(FsPermission.getUMask(conf)), initialPermission);
} catch(Exception e) {
- System.out.println(StringUtils.stringifyException(e));
- System.out.println("Cannot run test");
+ LOGGER.error("Cannot run test", e);
return;
}
@@ -105,8 +112,7 @@ public class TestLocalFileSystemPermission extends TestCase {
assertEquals(copyPermission, initialPermission);
dir2 = copyPath;
} catch (Exception e) {
- System.out.println(StringUtils.stringifyException(e));
- System.out.println("Cannot run test");
+ LOGGER.error("Cannot run test", e);
return;
} finally {
cleanup(localfs, dir);
@@ -120,7 +126,7 @@ public class TestLocalFileSystemPermission extends TestCase {
/** Test LocalFileSystem.setPermission */
public void testLocalFSsetPermission() throws IOException {
if (Path.WINDOWS) {
- System.out.println("Cannot run test for Windows");
+ LOGGER.info("Cannot run test for Windows");
return;
}
Configuration conf = new Configuration();
@@ -134,8 +140,7 @@ public class TestLocalFileSystemPermission extends TestCase {
FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(conf)),
initialPermission);
} catch(Exception e) {
- System.out.println(StringUtils.stringifyException(e));
- System.out.println("Cannot run test");
+ LOGGER.error("Cannot run test", e);
return;
}
@@ -147,8 +152,7 @@ public class TestLocalFileSystemPermission extends TestCase {
assertEquals(
perm.applyUMask(FsPermission.getUMask(conf)), initialPermission);
} catch(Exception e) {
- System.out.println(StringUtils.stringifyException(e));
- System.out.println("Cannot run test");
+ LOGGER.error("Cannot run test", e);
return;
}
@@ -162,8 +166,7 @@ public class TestLocalFileSystemPermission extends TestCase {
assertEquals(copyPermission, initialPermission);
f2 = copyPath;
} catch (Exception e) {
- System.out.println(StringUtils.stringifyException(e));
- System.out.println("Cannot run test");
+ LOGGER.error("Cannot run test", e);
return;
}
@@ -191,10 +194,10 @@ public class TestLocalFileSystemPermission extends TestCase {
return fs.getFileStatus(p).getPermission();
}
- /** Test LocalFileSystem.setOwner */
+ /** Test LocalFileSystem.setOwner. */
public void testLocalFSsetOwner() throws IOException {
if (Path.WINDOWS) {
- System.out.println("Cannot run test for Windows");
+ LOGGER.info("Cannot run test for Windows");
return;
}
@@ -206,16 +209,15 @@ public class TestLocalFileSystemPermission extends TestCase {
List<String> groups = null;
try {
groups = getGroups();
- System.out.println(filename + ": " + getPermission(localfs, f));
+ LOGGER.info("{}: {}", filename, getPermission(localfs, f));
}
catch(IOException e) {
- System.out.println(StringUtils.stringifyException(e));
- System.out.println("Cannot run test");
+ LOGGER.error("Cannot run test", e);
return;
}
if (groups == null || groups.size() < 1) {
- System.out.println("Cannot run test: need at least one group. groups="
- + groups);
+ LOGGER.error("Cannot run test: need at least one group. groups={}",
+ groups);
return;
}
@@ -230,13 +232,61 @@ public class TestLocalFileSystemPermission extends TestCase {
localfs.setOwner(f, null, g1);
assertEquals(g1, getGroup(localfs, f));
} else {
- System.out.println("Not testing changing the group since user " +
- "belongs to only one group.");
+ LOGGER.info("Not testing changing the group since user " +
+ "belongs to only one group.");
}
}
finally {cleanup(localfs, f);}
}
+ /**
+ * Steps:
+ * 1. Create a directory with default permissions: 777 with umask 022
+ * 2. Check the directory has good permissions: 755
+ * 3. Set the umask to 062.
+ * 4. Create a new directory with default permissions.
+ * 5. For this directory we expect 715 as permission not 755
+ * @throws Exception we can throw away all the exception.
+ */
+ public void testSetUmaskInRealTime() throws Exception {
+ if (Path.WINDOWS) {
+ LOGGER.info("Cannot run test for Windows");
+ return;
+ }
+
+ LocalFileSystem localfs = FileSystem.getLocal(new Configuration());
+ Configuration conf = localfs.getConf();
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "022");
+ LOGGER.info("Current umask is {}",
+ conf.get(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY));
+ Path dir = new Path(TEST_PATH_PREFIX + "dir");
+ Path dir2 = new Path(TEST_PATH_PREFIX + "dir2");
+ try {
+ assertTrue(localfs.mkdirs(dir));
+ FsPermission initialPermission = getPermission(localfs, dir);
+ assertEquals(
+ "With umask 022 permission should be 755 since the default " +
+ "permission is 777", new FsPermission("755"), initialPermission);
+
+ // Modify umask and create a new directory
+ // and check if new umask is applied
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "062");
+ assertTrue(localfs.mkdirs(dir2));
+ FsPermission finalPermission = localfs.getFileStatus(dir2)
+ .getPermission();
+ assertThat("With umask 062 permission should not be 755 since the " +
+ "default permission is 777", new FsPermission("755"),
+ is(not(finalPermission)));
+ assertEquals(
+ "With umask 062 we expect 715 since the default permission is 777",
+ new FsPermission("715"), finalPermission);
+ } finally {
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "022");
+ cleanup(localfs, dir);
+ cleanup(localfs, dir2);
+ }
+ }
+
static List<String> getGroups() throws IOException {
List<String> a = new ArrayList<String>();
String s = Shell.execCommand(Shell.getGroupsCommand());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] hadoop git commit: HDFS-10603. Fix flaky tests in
org.apache.hadoop.hdfs.server.namenode.snapshot.TestOpenFilesWithSnapshot.
Contributed by Yiqun Lin.
Posted by su...@apache.org.
HDFS-10603. Fix flaky tests in org.apache.hadoop.hdfs.server.namenode.snapshot.TestOpenFilesWithSnapshot. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85f4b862
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85f4b862
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85f4b862
Branch: refs/heads/YARN-2915
Commit: 85f4b86233158ada5f71b77b7783ce0604e0b3ad
Parents: ebc4a5d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Jul 19 13:49:24 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:33 2016 -0700
----------------------------------------------------------------------
.../snapshot/TestOpenFilesWithSnapshot.java | 21 +++++++++-----------
1 file changed, 9 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85f4b862/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
index 812bcc5..7b7c34a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
@@ -69,7 +69,7 @@ public class TestOpenFilesWithSnapshot {
// delete files separately
fs.delete(new Path("/test/test/test2"), true);
fs.delete(new Path("/test/test/test3"), true);
- cluster.restartNameNode();
+ restartNameNode();
}
@Test
@@ -79,7 +79,7 @@ public class TestOpenFilesWithSnapshot {
// delete parent directory
fs.delete(new Path("/test/test"), true);
- cluster.restartNameNode();
+ restartNameNode();
}
@Test
@@ -87,11 +87,7 @@ public class TestOpenFilesWithSnapshot {
Path path = new Path("/test");
doWriteAndAbort(fs, path);
fs.delete(new Path("/test/test"), true);
- NameNode nameNode = cluster.getNameNode();
- NameNodeAdapter.enterSafeMode(nameNode, false);
- NameNodeAdapter.saveNamespace(nameNode);
- NameNodeAdapter.leaveSafeMode(nameNode);
- cluster.restartNameNode(true);
+ restartNameNode();
// read snapshot file after restart
String test2snapshotPath = Snapshot.getSnapshotPath(path.toString(),
@@ -108,11 +104,7 @@ public class TestOpenFilesWithSnapshot {
doWriteAndAbort(fs, path);
fs.delete(new Path("/test/test/test2"), true);
fs.delete(new Path("/test/test/test3"), true);
- NameNode nameNode = cluster.getNameNode();
- NameNodeAdapter.enterSafeMode(nameNode, false);
- NameNodeAdapter.saveNamespace(nameNode);
- NameNodeAdapter.leaveSafeMode(nameNode);
- cluster.restartNameNode(true);
+ restartNameNode();
// read snapshot file after restart
String test2snapshotPath = Snapshot.getSnapshotPath(path.toString(),
@@ -203,6 +195,11 @@ public class TestOpenFilesWithSnapshot {
fs.rename(new Path("/test/test"), new Path("/test/test-renamed"));
fs.delete(new Path("/test/test-renamed"), true);
+ restartNameNode();
+ }
+
+ private void restartNameNode() throws Exception {
+ cluster.triggerBlockReports();
NameNode nameNode = cluster.getNameNode();
NameNodeAdapter.enterSafeMode(nameNode, false);
NameNodeAdapter.saveNamespace(nameNode);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] hadoop git commit: YARN-5181. ClusterNodeTracker: add method
to get list of nodes matching a specific resourceName. (kasha via asuresh)
Posted by su...@apache.org.
YARN-5181. ClusterNodeTracker: add method to get list of nodes matching a specific resourceName. (kasha via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebc4a5db
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebc4a5db
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebc4a5db
Branch: refs/heads/YARN-2915
Commit: ebc4a5db450062be9e258bb0b729c95639808869
Parents: 8a71a1a
Author: Arun Suresh <as...@apache.org>
Authored: Tue Jul 19 10:43:37 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:33 2016 -0700
----------------------------------------------------------------------
.../scheduler/ClusterNodeTracker.java | 56 ++++++++++++----
.../scheduler/TestClusterNodeTracker.java | 69 ++++++++++++++++++++
2 files changed, 114 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebc4a5db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index 413bbdf..e487f69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -18,11 +18,13 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -50,7 +52,8 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
private Lock writeLock = readWriteLock.writeLock();
private HashMap<NodeId, N> nodes = new HashMap<>();
- private Map<String, Integer> nodesPerRack = new HashMap<>();
+ private Map<String, N> nodeNameToNodeMap = new HashMap<>();
+ private Map<String, List<N>> nodesPerRack = new HashMap<>();
private Resource clusterCapacity = Resources.clone(Resources.none());
private Resource staleClusterCapacity = null;
@@ -66,14 +69,16 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
writeLock.lock();
try {
nodes.put(node.getNodeID(), node);
+ nodeNameToNodeMap.put(node.getNodeName(), node);
// Update nodes per rack as well
String rackName = node.getRackName();
- Integer numNodes = nodesPerRack.get(rackName);
- if (numNodes == null) {
- numNodes = 0;
+ List<N> nodesList = nodesPerRack.get(rackName);
+ if (nodesList == null) {
+ nodesList = new ArrayList<>();
+ nodesPerRack.put(rackName, nodesList);
}
- nodesPerRack.put(rackName, ++numNodes);
+ nodesList.add(node);
// Update cluster capacity
Resources.addTo(clusterCapacity, node.getTotalResource());
@@ -126,8 +131,8 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
readLock.lock();
String rName = rackName == null ? "NULL" : rackName;
try {
- Integer nodeCount = nodesPerRack.get(rName);
- return nodeCount == null ? 0 : nodeCount;
+ List<N> nodesList = nodesPerRack.get(rName);
+ return nodesList == null ? 0 : nodesList.size();
} finally {
readLock.unlock();
}
@@ -154,14 +159,18 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
LOG.warn("Attempting to remove a non-existent node " + nodeId);
return null;
}
+ nodeNameToNodeMap.remove(node.getNodeName());
// Update nodes per rack as well
String rackName = node.getRackName();
- Integer numNodes = nodesPerRack.get(rackName);
- if (numNodes > 0) {
- nodesPerRack.put(rackName, --numNodes);
- } else {
+ List<N> nodesList = nodesPerRack.get(rackName);
+ if (nodesList == null) {
LOG.error("Attempting to remove node from an empty rack " + rackName);
+ } else {
+ nodesList.remove(node);
+ if (nodesList.isEmpty()) {
+ nodesPerRack.remove(rackName);
+ }
}
// Update cluster capacity
@@ -297,4 +306,29 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
Collections.sort(sortedList, comparator);
return sortedList;
}
+
+ /**
+ * Convenience method to return list of nodes corresponding to resourceName
+ * passed in the {@link ResourceRequest}.
+ *
+ * @param resourceName Host/rack name of the resource, or
+ * {@link ResourceRequest#ANY}
+ * @return list of nodes that match the resourceName
+ */
+ public List<N> getNodesByResourceName(final String resourceName) {
+ Preconditions.checkArgument(
+ resourceName != null && !resourceName.isEmpty());
+ List<N> retNodes = new ArrayList<>();
+ if (ResourceRequest.ANY.equals(resourceName)) {
+ return getAllNodes();
+ } else if (nodeNameToNodeMap.containsKey(resourceName)) {
+ retNodes.add(nodeNameToNodeMap.get(resourceName));
+ } else if (nodesPerRack.containsKey(resourceName)) {
+ return nodesPerRack.get(resourceName);
+ } else {
+ LOG.info(
+ "Could not find a node matching given resourceName " + resourceName);
+ }
+ return retNodes;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebc4a5db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
new file mode 100644
index 0000000..7f527f1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSSchedulerNode;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test class to verify ClusterNodeTracker. Using FSSchedulerNode without
+ * loss of generality.
+ */
+public class TestClusterNodeTracker {
+ private ClusterNodeTracker<FSSchedulerNode> nodeTracker =
+ new ClusterNodeTracker<>();
+
+ @Before
+ public void setup() {
+ List<RMNode> rmNodes =
+ MockNodes.newNodes(2, 4, Resource.newInstance(4096, 4));
+ for (RMNode rmNode : rmNodes) {
+ nodeTracker.addNode(new FSSchedulerNode(rmNode, false));
+ }
+ }
+
+ @Test
+ public void testGetNodeCount() {
+ assertEquals("Incorrect number of nodes in the cluster",
+ 8, nodeTracker.nodeCount());
+
+ assertEquals("Incorrect number of nodes in each rack",
+ 4, nodeTracker.nodeCount("rack0"));
+ }
+
+ @Test
+ public void testGetNodesForResourceName() throws Exception {
+ assertEquals("Incorrect number of nodes matching ANY",
+ 8, nodeTracker.getNodesByResourceName(ResourceRequest.ANY).size());
+
+ assertEquals("Incorrect number of nodes matching rack",
+ 4, nodeTracker.getNodesByResourceName("rack0").size());
+
+ assertEquals("Incorrect number of nodes matching node",
+ 1, nodeTracker.getNodesByResourceName("host0").size());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] hadoop git commit: HADOOP-13351. TestDFSClientSocketSize
buffer size tests are flaky. Contributed by Aaron Fabbri and Mingliang Liu.
Posted by su...@apache.org.
HADOOP-13351. TestDFSClientSocketSize buffer size tests are flaky. Contributed by Aaron Fabbri and Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9fbde751
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9fbde751
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9fbde751
Branch: refs/heads/YARN-2915
Commit: 9fbde751b0c0c88ed4c904327198066c5131048f
Parents: 35a57c9
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Jul 14 14:40:58 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:31 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/TestDFSClientSocketSize.java | 80 +++++++++++---------
1 file changed, 44 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fbde751/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java
index 2376576..a2a7afc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
-import org.junit.After;
import org.junit.Test;
import org.slf4j.Logger;
@@ -33,7 +32,6 @@ import java.net.Socket;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class TestDFSClientSocketSize {
@@ -43,55 +41,65 @@ public class TestDFSClientSocketSize {
GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.ALL);
}
- private final Configuration conf = new Configuration();
- private MiniDFSCluster cluster;
- private Socket socket;
-
+ /**
+ * The setting of socket send buffer size in
+ * {@link java.net.Socket#setSendBufferSize(int)} is only a hint. Actual
+ * value may differ. We just sanity check that it is somewhere close.
+ */
@Test
public void testDefaultSendBufferSize() throws IOException {
- socket = createSocket();
- assertEquals("Send buffer size should be the default value.",
- DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT,
- socket.getSendBufferSize());
+ assertTrue("Send buffer size should be somewhere near default.",
+ getSendBufferSize(new Configuration()) >=
+ DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT / 2);
}
+ /**
+ * Note that {@link java.net.Socket#setSendBufferSize(int)} is only a hint.
+ * If this test is flaky it should be ignored. See HADOOP-13351.
+ */
@Test
public void testSpecifiedSendBufferSize() throws IOException {
- final int mySendBufferSize = 64 * 1024; // 64 KB
- conf.setInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY, mySendBufferSize);
- socket = createSocket();
- assertEquals("Send buffer size should be the customized value.",
- mySendBufferSize, socket.getSendBufferSize());
+ final Configuration conf1 = new Configuration();
+ conf1.setInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY, 256 * 1024); // 256 KB
+ final int sendBufferSize1 = getSendBufferSize(conf1);
+
+ final Configuration conf2 = new Configuration();
+ conf2.setInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY, 1024); // 1 KB
+ final int sendBufferSize2 = getSendBufferSize(conf2);
+
+ LOG.info("Large buf size is {}, small is {}",
+ sendBufferSize1, sendBufferSize2);
+ assertTrue("Larger specified send buffer should have effect",
+ sendBufferSize1 > sendBufferSize2);
}
@Test
public void testAutoTuningSendBufferSize() throws IOException {
+ final Configuration conf = new Configuration();
conf.setInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY, 0);
- socket = createSocket();
- LOG.info("The auto tuned send buffer size is: {}",
- socket.getSendBufferSize());
+ final int sendBufferSize = getSendBufferSize(conf);
+ LOG.info("The auto tuned send buffer size is: {}", sendBufferSize);
assertTrue("Send buffer size should be non-negative value which is " +
- "determined by system (kernel).", socket.getSendBufferSize() > 0);
+ "determined by system (kernel).", sendBufferSize > 0);
}
- @After
- public void tearDown() throws Exception {
- if (socket != null) {
- LOG.info("Closing the DFSClient socket.");
- }
- if (cluster != null) {
- LOG.info("Shutting down MiniDFSCluster.");
- cluster.shutdown();
- cluster = null;
+ private int getSendBufferSize(Configuration conf) throws IOException {
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(1)
+ .build();
+ try {
+ cluster.waitActive();
+ LOG.info("MiniDFSCluster started.");
+ try (Socket socket = DataStreamer.createSocketForPipeline(
+ new DatanodeInfo(cluster.dataNodes.get(0).datanode.getDatanodeId()),
+ 1, cluster.getFileSystem().getClient())) {
+ return socket.getSendBufferSize();
+ }
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
- private Socket createSocket() throws IOException {
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- LOG.info("MiniDFSCluster started.");
- return DataStreamer.createSocketForPipeline(
- new DatanodeInfo(cluster.dataNodes.get(0).datanode.getDatanodeId()),
- 1, cluster.getFileSystem().getClient());
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] hadoop git commit: HDFS-10590: Fix
TestReconstructStripedBlocks.testCountLiveReplicas test failures. Contributed
by Rakesh R
Posted by su...@apache.org.
HDFS-10590: Fix TestReconstructStripedBlocks.testCountLiveReplicas test failures. Contributed by Rakesh R
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b71d5ed1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b71d5ed1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b71d5ed1
Branch: refs/heads/YARN-2915
Commit: b71d5ed14f8dc1377c92f1811497df1f1801c519
Parents: e1ef53d
Author: Uma Maheswara Rao G <um...@intel.com>
Authored: Wed Jul 13 00:07:11 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:29 2016 -0700
----------------------------------------------------------------------
.../src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java | 2 ++
.../hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java | 2 ++
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71d5ed1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
index 4c3afdd..0b036fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
@@ -514,6 +514,8 @@ public class StripedFileTestUtil {
public static LocatedBlocks waitForReconstructionFinished(Path file,
DistributedFileSystem fs, int groupSize)
throws Exception {
+ LOG.info("Waiting for reconstruction to be finished for the file:" + file
+ + ", groupSize:" + groupSize);
final int attempts = 60;
for (int i = 0; i < attempts; i++) {
LocatedBlocks locatedBlocks = getLocatedBlocks(file, fs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71d5ed1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
index 6069924..c38b6b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
@@ -303,6 +303,8 @@ public class TestReconstructStripedBlocks {
// check if NN can detect the missing internal block and finish the
// reconstruction
+ StripedFileTestUtil.waitForReconstructionFinished(filePath, fs,
+ GROUP_SIZE);
boolean reconstructed = false;
for (int i = 0; i < 5; i++) {
NumberReplicas num = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] hadoop git commit: YARN-5200. Enhanced "yarn logs" to be able
to get a list of containers whose logs are aggregated via a
"show_container_log_info" option. Contributed by Xuan Gong.
Posted by su...@apache.org.
YARN-5200. Enhanced "yarn logs" to be able to get a list of containers whose logs are aggregated via a "show_container_log_info" option. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b06353ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b06353ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b06353ec
Branch: refs/heads/YARN-2915
Commit: b06353ec3d19e0d14581ad17b8e3b691d107d63f
Parents: ee086e7
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Wed Jul 13 10:47:43 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 321 ++++++++++++++-----
.../hadoop/yarn/client/cli/TestLogsCLI.java | 85 +++--
.../logaggregation/AggregatedLogFormat.java | 13 +-
.../yarn/logaggregation/LogCLIHelpers.java | 97 +++++-
4 files changed, 392 insertions(+), 124 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b06353ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 79401b1..31395be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -78,14 +78,30 @@ public class LogsCLI extends Configured implements Tool {
private static final String APP_OWNER_OPTION = "appOwner";
private static final String AM_CONTAINER_OPTION = "am";
private static final String CONTAINER_LOG_FILES = "logFiles";
- private static final String SHOW_META_INFO = "show_meta_info";
private static final String LIST_NODES_OPTION = "list_nodes";
+ private static final String SHOW_APPLICATION_LOG_INFO
+ = "show_application_log_info";
+ private static final String SHOW_CONTAINER_LOG_INFO
+ = "show_container_log_info";
private static final String OUT_OPTION = "out";
private static final String SIZE_OPTION = "size";
public static final String HELP_CMD = "help";
+ private PrintStream outStream = System.out;
+ private YarnClient yarnClient = null;
@Override
public int run(String[] args) throws Exception {
+ try {
+ yarnClient = createYarnClient();
+ return runCommand(args);
+ } finally {
+ if (yarnClient != null) {
+ yarnClient.close();
+ }
+ }
+ }
+
+ private int runCommand(String[] args) throws Exception {
Options opts = createCommandOpts();
Options printOpts = createPrintOpts(opts);
if (args.length < 1) {
@@ -102,8 +118,9 @@ public class LogsCLI extends Configured implements Tool {
String nodeAddress = null;
String appOwner = null;
boolean getAMContainerLogs = false;
- boolean showMetaInfo = false;
boolean nodesList = false;
+ boolean showApplicationLogInfo = false;
+ boolean showContainerLogInfo = false;
String[] logFiles = null;
List<String> amContainersList = new ArrayList<String>();
String localDir = null;
@@ -115,9 +132,11 @@ public class LogsCLI extends Configured implements Tool {
nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION);
appOwner = commandLine.getOptionValue(APP_OWNER_OPTION);
getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION);
- showMetaInfo = commandLine.hasOption(SHOW_META_INFO);
nodesList = commandLine.hasOption(LIST_NODES_OPTION);
localDir = commandLine.getOptionValue(OUT_OPTION);
+ showApplicationLogInfo = commandLine.hasOption(
+ SHOW_APPLICATION_LOG_INFO);
+ showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO);
if (getAMContainerLogs) {
try {
amContainersList = parseAMContainer(commandLine, printOpts);
@@ -172,6 +191,12 @@ public class LogsCLI extends Configured implements Tool {
}
}
+ if (showApplicationLogInfo && showContainerLogInfo) {
+ System.err.println("Invalid options. Can only accept one of "
+ + "show_application_log_info/show_container_log_info.");
+ return -1;
+ }
+
LogCLIHelpers logCliHelper = new LogCLIHelpers();
logCliHelper.setConf(getConf());
@@ -215,14 +240,17 @@ public class LogsCLI extends Configured implements Tool {
isApplicationFinished(appState), appOwner, nodeAddress, null,
containerIdStr, localDir, logs, bytes);
- if (showMetaInfo) {
- return showMetaInfo(request, logCliHelper);
+ if (showContainerLogInfo) {
+ return showContainerLogInfo(request, logCliHelper);
}
if (nodesList) {
return showNodeLists(request, logCliHelper);
}
+ if (showApplicationLogInfo) {
+ return showApplicationLogInfo(request, logCliHelper);
+ }
// To get am logs
if (getAMContainerLogs) {
return fetchAMContainerLogs(request, amContainersList,
@@ -246,21 +274,15 @@ public class LogsCLI extends Configured implements Tool {
private ApplicationReport getApplicationReport(ApplicationId appId)
throws IOException, YarnException {
- YarnClient yarnClient = createYarnClient();
-
- try {
- return yarnClient.getApplicationReport(appId);
- } finally {
- yarnClient.close();
- }
+ return yarnClient.getApplicationReport(appId);
}
@VisibleForTesting
protected YarnClient createYarnClient() {
- YarnClient yarnClient = YarnClient.createYarnClient();
- yarnClient.init(getConf());
- yarnClient.start();
- return yarnClient;
+ YarnClient client = YarnClient.createYarnClient();
+ client.init(getConf());
+ client.start();
+ return client;
}
public static void main(String[] args) throws Exception {
@@ -272,7 +294,7 @@ public class LogsCLI extends Configured implements Tool {
}
private void printHelpMessage(Options options) {
- System.out.println("Retrieve logs for completed YARN applications.");
+ outStream.println("Retrieve logs for YARN applications.");
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("yarn logs -applicationId <application ID> [OPTIONS]",
new Options());
@@ -336,9 +358,9 @@ public class LogsCLI extends Configured implements Tool {
return false;
}
- private List<String> getContainerLogFiles(Configuration conf,
+ private List<PerLogFileInfo> getContainerLogFiles(Configuration conf,
String containerIdStr, String nodeHttpAddress) throws IOException {
- List<String> logFiles = new ArrayList<>();
+ List<PerLogFileInfo> logFileInfos = new ArrayList<>();
Client webServiceClient = Client.create();
try {
WebResource webResource = webServiceClient
@@ -354,7 +376,9 @@ public class LogsCLI extends Configured implements Tool {
JSONObject json = response.getEntity(JSONObject.class);
JSONArray array = json.getJSONArray("containerLogInfo");
for (int i = 0; i < array.length(); i++) {
- logFiles.add(array.getJSONObject(i).getString("fileName"));
+ String fileName = array.getJSONObject(i).getString("fileName");
+ String fileSize = array.getJSONObject(i).getString("fileSize");
+ logFileInfos.add(new PerLogFileInfo(fileName, fileSize));
}
} catch (Exception e) {
System.err.println("Unable to parse json from webservice. Error:");
@@ -367,7 +391,7 @@ public class LogsCLI extends Configured implements Tool {
System.err.println("Unable to fetch log files list");
throw new IOException(ex);
}
- return logFiles;
+ return logFileInfos;
}
@Private
@@ -378,15 +402,26 @@ public class LogsCLI extends Configured implements Tool {
String containerIdStr = request.getContainerId().toString();
String localDir = request.getOutputLocalDir();
String nodeHttpAddress = request.getNodeHttpAddress();
+ if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
+ System.err.println("Can not get the logs for the container: "
+ + containerIdStr);
+ System.err.println("The node http address is required to get container "
+ + "logs for the Running application.");
+ return -1;
+ }
String nodeId = request.getNodeId();
PrintStream out = logCliHelper.createPrintStream(localDir, nodeId,
containerIdStr);
try {
// fetch all the log files for the container
// filter the log files based on the given --logFiles pattern
- List<String> allLogs=
+ List<PerLogFileInfo> allLogFileInfos=
getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress);
- List<String> matchedFiles = getMatchedLogFiles(request, allLogs);
+ List<String> fileNames = new ArrayList<String>();
+ for (PerLogFileInfo fileInfo : allLogFileInfos) {
+ fileNames.add(fileInfo.getFileName());
+ }
+ List<String> matchedFiles = getMatchedLogFiles(request, fileNames);
if (matchedFiles.isEmpty()) {
System.err.println("Can not find any log file matching the pattern: "
+ request.getLogTypes() + " for the container: " + containerIdStr
@@ -397,8 +432,8 @@ public class LogsCLI extends Configured implements Tool {
newOptions.setLogTypes(matchedFiles);
Client webServiceClient = Client.create();
- String containerString = "\n\nContainer: " + containerIdStr + " on "
- + nodeId;
+ String containerString = String.format(
+ LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerIdStr, nodeId);
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
boolean foundAnyLogs = false;
@@ -477,13 +512,8 @@ public class LogsCLI extends Configured implements Tool {
@VisibleForTesting
public ContainerReport getContainerReport(String containerIdStr)
throws YarnException, IOException {
- YarnClient yarnClient = createYarnClient();
- try {
- return yarnClient.getContainerReport(
- ContainerId.fromString(containerIdStr));
- } finally {
- yarnClient.close();
- }
+ return yarnClient.getContainerReport(
+ ContainerId.fromString(containerIdStr));
}
private boolean isApplicationFinished(YarnApplicationState appState) {
@@ -508,8 +538,10 @@ public class LogsCLI extends Configured implements Tool {
for (JSONObject amContainer : amContainersList) {
ContainerLogsRequest amRequest = new ContainerLogsRequest(request);
amRequest.setContainerId(amContainer.getString("containerId"));
- amRequest.setNodeHttpAddress(
- amContainer.getString("nodeHttpAddress"));
+ String httpAddress = amContainer.getString("nodeHttpAddress");
+ if (httpAddress != null && !httpAddress.isEmpty()) {
+ amRequest.setNodeHttpAddress(httpAddress);
+ }
amRequest.setNodeId(amContainer.getString("nodeId"));
requests.add(amRequest);
}
@@ -545,8 +577,8 @@ public class LogsCLI extends Configured implements Tool {
for (ContainerLogsRequest amRequest : requests) {
outputAMContainerLogs(amRequest, conf, logCliHelper);
}
- System.out.println();
- System.out.println("Specified ALL for -am option. "
+ outStream.println();
+ outStream.println("Specified ALL for -am option. "
+ "Printed logs for all am containers.");
} else {
for (String amContainer : amContainers) {
@@ -602,15 +634,13 @@ public class LogsCLI extends Configured implements Tool {
}
}
- private int showMetaInfo(ContainerLogsRequest request,
- LogCLIHelpers logCliHelper) throws IOException {
+ private int showContainerLogInfo(ContainerLogsRequest request,
+ LogCLIHelpers logCliHelper) throws IOException, YarnException {
if (!request.isAppFinished()) {
- System.err.println("The -show_meta_info command can be only used "
- + "with finished applications");
- return -1;
+ return printContainerInfoFromRunningApplication(request);
} else {
- logCliHelper.printLogMetadata(request, System.out, System.err);
- return 0;
+ return logCliHelper.printAContainerLogMetadata(
+ request, System.out, System.err);
}
}
@@ -626,6 +656,33 @@ public class LogsCLI extends Configured implements Tool {
}
}
+ private int showApplicationLogInfo(ContainerLogsRequest request,
+ LogCLIHelpers logCliHelper) throws IOException, YarnException {
+ String appState = "Application State: "
+ + (request.isAppFinished() ? "Completed." : "Running.");
+ if (!request.isAppFinished()) {
+ List<ContainerReport> reports =
+ getContainerReportsFromRunningApplication(request);
+ List<ContainerReport> filterReports = filterContainersInfo(
+ request, reports);
+ if (filterReports.isEmpty()) {
+ System.err.println("Can not find any containers for the application:"
+ + request.getAppId() + ".");
+ return -1;
+ }
+ outStream.println(appState);
+ for (ContainerReport report : filterReports) {
+ outStream.println(String.format(LogCLIHelpers.CONTAINER_ON_NODE_PATTERN,
+ report.getContainerId(), report.getAssignedNode()));
+ }
+ return 0;
+ } else {
+ outStream.println(appState);
+ logCliHelper.printContainersList(request, System.out, System.err);
+ return 0;
+ }
+ }
+
private Options createCommandOpts() {
Options opts = new Options();
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
@@ -661,13 +718,16 @@ public class LogsCLI extends Configured implements Tool {
logFileOpt.setArgs(Option.UNLIMITED_VALUES);
logFileOpt.setArgName("Log File Name");
opts.addOption(logFileOpt);
- opts.addOption(SHOW_META_INFO, false, "Show the log metadata, "
+ opts.addOption(SHOW_CONTAINER_LOG_INFO, false,
+ "Show the container log metadata, "
+ "including log-file names, the size of the log files. "
+ "You can combine this with --containerId to get log metadata for "
+ "the specific container, or with --nodeAddress to get log metadata "
- + "for all the containers on the specific NodeManager. "
- + "Currently, this option can only be used for finished "
- + "applications.");
+ + "for all the containers on the specific NodeManager.");
+ opts.addOption(SHOW_APPLICATION_LOG_INFO, false, "Show the "
+ + "containerIds which belong to the specific Application. "
+ + "You can combine this with --nodeAddress to get containerIds "
+ + "for all the containers on the specific NodeManager.");
opts.addOption(LIST_NODES_OPTION, false,
"Show the list of nodes that successfully aggregated logs. "
+ "This option can only be used with finished applications.");
@@ -695,8 +755,9 @@ public class LogsCLI extends Configured implements Tool {
printOpts.addOption(commandOpts.getOption(APP_OWNER_OPTION));
printOpts.addOption(commandOpts.getOption(AM_CONTAINER_OPTION));
printOpts.addOption(commandOpts.getOption(CONTAINER_LOG_FILES));
- printOpts.addOption(commandOpts.getOption(SHOW_META_INFO));
printOpts.addOption(commandOpts.getOption(LIST_NODES_OPTION));
+ printOpts.addOption(commandOpts.getOption(SHOW_APPLICATION_LOG_INFO));
+ printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO));
printOpts.addOption(commandOpts.getOption(OUT_OPTION));
printOpts.addOption(commandOpts.getOption(SIZE_OPTION));
return printOpts;
@@ -801,12 +862,14 @@ public class LogsCLI extends Configured implements Tool {
// the ContainerReport. In the containerReport, we could get
// nodeAddress and nodeHttpAddress
ContainerReport report = getContainerReport(containerIdStr);
- nodeHttpAddress =
- report.getNodeHttpAddress().replaceFirst(
- WebAppUtils.getHttpSchemePrefix(getConf()), "");
+ nodeHttpAddress = report.getNodeHttpAddress();
+ if (nodeHttpAddress != null && !nodeHttpAddress.isEmpty()) {
+ nodeHttpAddress = nodeHttpAddress.replaceFirst(
+ WebAppUtils.getHttpSchemePrefix(getConf()), "");
+ request.setNodeHttpAddress(nodeHttpAddress);
+ }
nodeId = report.getAssignedNode().toString();
request.setNodeId(nodeId);
- request.setNodeHttpAddress(nodeHttpAddress);
} catch (IOException | YarnException ex) {
if (isAppFinished) {
return printContainerLogsForFinishedApplicationWithoutNodeId(
@@ -942,32 +1005,140 @@ public class LogsCLI extends Configured implements Tool {
ContainerLogsRequest options) throws YarnException, IOException {
List<ContainerLogsRequest> newOptionsList =
new ArrayList<ContainerLogsRequest>();
- YarnClient yarnClient = createYarnClient();
- try {
- List<ApplicationAttemptReport> attempts =
- yarnClient.getApplicationAttempts(options.getAppId());
- for (ApplicationAttemptReport attempt : attempts) {
- List<ContainerReport> containers = yarnClient.getContainers(
- attempt.getApplicationAttemptId());
- for (ContainerReport container : containers) {
- ContainerLogsRequest newOptions = new ContainerLogsRequest(options);
- newOptions.setContainerId(container.getContainerId().toString());
- newOptions.setNodeId(container.getAssignedNode().toString());
- newOptions.setNodeHttpAddress(container.getNodeHttpAddress()
- .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
- // if we do not specify the value for CONTAINER_LOG_FILES option,
- // we will only output syslog
- List<String> logFiles = newOptions.getLogTypes();
- if (logFiles == null || logFiles.isEmpty()) {
- logFiles = Arrays.asList("syslog");
- newOptions.setLogTypes(logFiles);
+ List<ContainerReport> reports =
+ getContainerReportsFromRunningApplication(options);
+ for (ContainerReport container : reports) {
+ ContainerLogsRequest newOptions = new ContainerLogsRequest(options);
+ newOptions.setContainerId(container.getContainerId().toString());
+ newOptions.setNodeId(container.getAssignedNode().toString());
+ String httpAddress = container.getNodeHttpAddress();
+ if (httpAddress != null && !httpAddress.isEmpty()) {
+ newOptions.setNodeHttpAddress(httpAddress
+ .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
+ }
+ // if we do not specify the value for CONTAINER_LOG_FILES option,
+ // we will only output syslog
+ List<String> logFiles = newOptions.getLogTypes();
+ if (logFiles == null || logFiles.isEmpty()) {
+ logFiles = Arrays.asList("syslog");
+ newOptions.setLogTypes(logFiles);
+ }
+ newOptionsList.add(newOptions);
+ }
+ return newOptionsList;
+ }
+
+ private List<ContainerReport> getContainerReportsFromRunningApplication(
+ ContainerLogsRequest options) throws YarnException, IOException {
+ List<ContainerReport> reports = new ArrayList<ContainerReport>();
+ List<ApplicationAttemptReport> attempts =
+ yarnClient.getApplicationAttempts(options.getAppId());
+ for (ApplicationAttemptReport attempt : attempts) {
+ List<ContainerReport> containers = yarnClient.getContainers(
+ attempt.getApplicationAttemptId());
+ reports.addAll(containers);
+ }
+ return reports;
+ }
+
+ // filter the containerReports based on the nodeId and ContainerId
+ private List<ContainerReport> filterContainersInfo(
+ ContainerLogsRequest options, List<ContainerReport> containers) {
+ List<ContainerReport> filterReports = new ArrayList<ContainerReport>(
+ containers);
+ String nodeId = options.getNodeId();
+ boolean filterBasedOnNodeId = (nodeId != null && !nodeId.isEmpty());
+ String containerId = options.getContainerId();
+ boolean filterBasedOnContainerId = (containerId != null
+ && !containerId.isEmpty());
+
+ if (filterBasedOnNodeId || filterBasedOnContainerId) {
+ // filter the reports based on the containerId and.or nodeId
+ for(ContainerReport report : containers) {
+ if (filterBasedOnContainerId) {
+ if (!report.getContainerId().toString()
+ .equalsIgnoreCase(containerId)) {
+ filterReports.remove(report);
+ }
+ }
+
+ if (filterBasedOnNodeId) {
+ if (!report.getAssignedNode().toString().equalsIgnoreCase(nodeId)) {
+ filterReports.remove(report);
}
- newOptionsList.add(newOptions);
}
}
- return newOptionsList;
- } finally {
- yarnClient.close();
+ }
+ return filterReports;
+ }
+
+ private int printContainerInfoFromRunningApplication(
+ ContainerLogsRequest options) throws YarnException, IOException {
+ String containerIdStr = options.getContainerId();
+ String nodeIdStr = options.getNodeId();
+ List<ContainerReport> reports =
+ getContainerReportsFromRunningApplication(options);
+ List<ContainerReport> filteredReports = filterContainersInfo(
+ options, reports);
+ if (filteredReports.isEmpty()) {
+ StringBuilder sb = new StringBuilder();
+ if (containerIdStr != null && !containerIdStr.isEmpty()) {
+ sb.append("Trying to get container with ContainerId: "
+ + containerIdStr + "\n");
+ }
+ if (nodeIdStr != null && !nodeIdStr.isEmpty()) {
+ sb.append("Trying to get container from NodeManager: "
+ + nodeIdStr + "\n");
+ }
+ sb.append("Can not find any matched containers for the application: "
+ + options.getAppId());
+ System.err.println(sb.toString());
+ return -1;
+ }
+ for (ContainerReport report : filteredReports) {
+ String nodeId = report.getAssignedNode().toString();
+ String nodeHttpAddress = report.getNodeHttpAddress().replaceFirst(
+ WebAppUtils.getHttpSchemePrefix(getConf()), "");
+ String containerId = report.getContainerId().toString();
+ String containerString = String.format(
+ LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerId, nodeId);
+ outStream.println(containerString);
+ outStream.println(StringUtils.repeat("=", containerString.length()));
+ outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN,
+ "LogType", "LogLength");
+ outStream.println(StringUtils.repeat("=", containerString.length()));
+ List<PerLogFileInfo> infos = getContainerLogFiles(
+ getConf(), containerId, nodeHttpAddress);
+ for (PerLogFileInfo info : infos) {
+ outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN,
+ info.getFileName(), info.getFileLength());
+ }
+ }
+ return 0;
+ }
+
+ private static class PerLogFileInfo {
+ private String fileName;
+ private String fileLength;
+ public PerLogFileInfo(String fileName, String fileLength) {
+ setFileName(fileName);
+ setFileLength(fileLength);
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public void setFileName(String fileName) {
+ this.fileName = fileName;
+ }
+
+ public String getFileLength() {
+ return fileLength;
+ }
+
+ public void setFileLength(String fileLength) {
+ this.fileLength = fileLength;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b06353ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 34369ef..cb9a663 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -182,7 +182,7 @@ public class TestLogsCLI {
assertTrue(exitCode == -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
- pw.println("Retrieve logs for completed YARN applications.");
+ pw.println("Retrieve logs for YARN applications.");
pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]");
pw.println();
pw.println("general options are:");
@@ -217,14 +217,18 @@ public class TestLogsCLI {
pw.println(" container logs. The container logs will");
pw.println(" be stored based on the node the container");
pw.println(" ran on.");
- pw.println(" -show_meta_info Show the log metadata, including log-file");
- pw.println(" names, the size of the log files. You can");
- pw.println(" combine this with --containerId to get");
- pw.println(" log metadata for the specific container,");
- pw.println(" or with --nodeAddress to get log metadata");
- pw.println(" for all the containers on the specific");
- pw.println(" NodeManager. Currently, this option can");
- pw.println(" only be used for finished applications.");
+ pw.println(" -show_application_log_info Show the containerIds which belong to the");
+ pw.println(" specific Application. You can combine");
+ pw.println(" this with --nodeAddress to get");
+ pw.println(" containerIds for all the containers on");
+ pw.println(" the specific NodeManager.");
+ pw.println(" -show_container_log_info Show the container log metadata,");
+ pw.println(" including log-file names, the size of the");
+ pw.println(" log files. You can combine this with");
+ pw.println(" --containerId to get log metadata for the");
+ pw.println(" specific container, or with --nodeAddress");
+ pw.println(" to get log metadata for all the");
+ pw.println(" containers on the specific NodeManager.");
pw.println(" -size <size> Prints the log file's first 'n' bytes or");
pw.println(" the last 'n' bytes. Use negative values");
pw.println(" as bytes to read from the end and");
@@ -698,9 +702,8 @@ public class TestLogsCLI {
"-applicationId", appTest.toString()});
assertTrue(exitCode == -1);
assertTrue(sysErrStream.toString().contains(
- "Guessed logs' owner is " + priorityUser + " and current user "
- + UserGroupInformation.getCurrentUser().getUserName()
- + " does not have permission to access"));
+ "Can not find the logs for the application: "
+ + appTest.toString()));
sysErrStream.reset();
} finally {
fs.delete(new Path(remoteLogRootDir), true);
@@ -842,49 +845,77 @@ public class TestLogsCLI {
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration);
- cli.run(new String[] { "-applicationId", appId.toString(),
- "-show_meta_info" });
+ int result = cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_container_log_info", "-show_application_log_info"});
+ assertTrue(result == -1);
+ assertTrue(sysErrStream.toString().contains("Invalid options. "
+ + "Can only accept one of show_application_log_info/"
+ + "show_container_log_info."));
+
+ cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_container_log_info"});
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000001 on localhost_"));
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000002 on localhost_"));
assertTrue(sysOutStream.toString().contains(
- "LogType:syslog"));
+ "syslog"));
assertTrue(sysOutStream.toString().contains(
- "LogLength:43"));
+ "43"));
sysOutStream.reset();
- cli.run(new String[] { "-applicationId", appId.toString(),
- "-show_meta_info", "-containerId", "container_0_0001_01_000001" });
+ cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_container_log_info", "-containerId",
+ "container_0_0001_01_000001"});
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000001 on localhost_"));
assertFalse(sysOutStream.toString().contains(
"Container: container_0_0001_01_000002 on localhost_"));
assertTrue(sysOutStream.toString().contains(
- "LogType:syslog"));
+ "syslog"));
assertTrue(sysOutStream.toString().contains(
- "LogLength:43"));
+ "43"));
sysOutStream.reset();
- cli.run(new String[] { "-applicationId", appId.toString(),
- "-show_meta_info", "-nodeAddress", "localhost" });
+ cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_container_log_info", "-nodeAddress", "localhost"});
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000001 on localhost_"));
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000002 on localhost_"));
assertTrue(sysOutStream.toString().contains(
- "LogType:syslog"));
+ "syslog"));
assertTrue(sysOutStream.toString().contains(
- "LogLength:43"));
+ "43"));
sysOutStream.reset();
- cli.run(new String[] { "-applicationId", appId.toString(),
- "-show_meta_info", "-nodeAddress", "localhost", "-containerId",
- "container_1234" });
+ cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_container_log_info", "-nodeAddress", "localhost",
+ "-containerId", "container_1234"});
assertTrue(sysErrStream.toString().contains(
"Invalid ContainerId specified"));
sysErrStream.reset();
+ cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_application_log_info"});
+ assertTrue(sysOutStream.toString().contains(
+ "Application State: Completed."));
+ assertTrue(sysOutStream.toString().contains(
+ "container_0_0001_01_000001 on localhost"));
+ assertTrue(sysOutStream.toString().contains(
+ "container_0_0001_01_000002 on localhost"));
+ sysOutStream.reset();
+
+ cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_application_log_info", "-nodeAddress", "localhost"});
+ assertTrue(sysOutStream.toString().contains(
+ "Application State: Completed."));
+ assertTrue(sysOutStream.toString().contains(
+ "container_0_0001_01_000001 on localhost"));
+ assertTrue(sysOutStream.toString().contains(
+ "container_0_0001_01_000002 on localhost"));
+ sysOutStream.reset();
+
fs.delete(new Path(remoteLogRootDir), true);
fs.delete(new Path(rootLogDir), true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b06353ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 7508dd5..3c4f835 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -49,6 +49,7 @@ import org.apache.commons.io.input.BoundedInputStream;
import org.apache.commons.io.output.WriterOutputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -959,25 +960,21 @@ public class AggregatedLogFormat {
}
@Private
- public static String readContainerMetaDataAndSkipData(
+ public static Pair<String, String> readContainerMetaDataAndSkipData(
DataInputStream valueStream, PrintStream out) throws IOException {
String fileType = valueStream.readUTF();
String fileLengthStr = valueStream.readUTF();
long fileLength = Long.parseLong(fileLengthStr);
- if (out != null) {
- out.print("LogType:");
- out.println(fileType);
- out.print("LogLength:");
- out.println(fileLengthStr);
- }
+ Pair<String, String> logMeta = new Pair<String, String>(
+ fileType, fileLengthStr);
long totalSkipped = 0;
long currSkipped = 0;
while (currSkipped != -1 && totalSkipped < fileLength) {
currSkipped = valueStream.skip(fileLength - totalSkipped);
totalSkipped += currSkipped;
}
- return fileType;
+ return logMeta;
}
public void close() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b06353ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index 4de11a5..ef56504 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -50,6 +51,11 @@ import com.google.common.annotations.VisibleForTesting;
public class LogCLIHelpers implements Configurable {
+ public static final String PER_LOG_FILE_INFO_PATTERN =
+ "%20s\t%20s" + System.getProperty("line.separator");
+ public static final String CONTAINER_ON_NODE_PATTERN =
+ "Container: %s on %s";
+
private Configuration conf;
@Private
@@ -151,8 +157,8 @@ public class LogCLIHelpers implements Configurable {
AggregatedLogFormat.LogReader reader = null;
PrintStream out = createPrintStream(localDir, fileName, containerId);
try {
- String containerString = "\n\nContainer: " + containerId + " on "
- + thisNodeFile.getPath().getName();
+ String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
+ containerId, thisNodeFile.getPath().getName());
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
reader =
@@ -219,8 +225,10 @@ public class LogCLIHelpers implements Configurable {
thisNodeFile.getPath());
out = createPrintStream(localDir, thisNodeFile.getPath().getName(),
containerId);
- out.println(containerId + " on " + thisNodeFile.getPath().getName());
- out.println(StringUtils.repeat("=", containerId.length()));
+ String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
+ containerId, thisNodeFile.getPath().getName());
+ out.println(containerString);
+ out.println(StringUtils.repeat("=", containerString.length()));
if (logType == null || logType.isEmpty()) {
if (dumpAContainerLogs(containerId, reader, out,
thisNodeFile.getModificationTime(), options.getBytes()) > -1) {
@@ -355,9 +363,9 @@ public class LogCLIHelpers implements Configurable {
PrintStream out = createPrintStream(localDir,
thisNodeFile.getPath().getName(), key.toString());
try {
- String containerString =
- "\n\nContainer: " + key + " on "
- + thisNodeFile.getPath().getName();
+ String containerString = String.format(
+ CONTAINER_ON_NODE_PATTERN, key,
+ thisNodeFile.getPath().getName());
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
while (true) {
@@ -400,7 +408,7 @@ public class LogCLIHelpers implements Configurable {
}
@Private
- public void printLogMetadata(ContainerLogsRequest options,
+ public int printAContainerLogMetadata(ContainerLogsRequest options,
PrintStream out, PrintStream err)
throws IOException {
ApplicationId appId = options.getAppId();
@@ -413,7 +421,7 @@ public class LogCLIHelpers implements Configurable {
RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
appId, appOwner);
if (nodeFiles == null) {
- return;
+ return -1;
}
boolean foundAnyLogs = false;
while (nodeFiles.hasNext()) {
@@ -434,16 +442,21 @@ public class LogCLIHelpers implements Configurable {
valueStream = reader.next(key);
while (valueStream != null) {
if (getAllContainers || (key.toString().equals(containerIdStr))) {
- String containerString =
- "\n\nContainer: " + key + " on "
- + thisNodeFile.getPath().getName();
+ String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
+ key, thisNodeFile.getPath().getName());
out.println(containerString);
out.println("Log Upload Time:"
+ thisNodeFile.getModificationTime());
out.println(StringUtils.repeat("=", containerString.length()));
+ out.printf(PER_LOG_FILE_INFO_PATTERN, "LogType", "LogLength");
+ out.println(StringUtils.repeat("=", containerString.length()));
while (true) {
try {
- LogReader.readContainerMetaDataAndSkipData(valueStream, out);
+ Pair<String, String> logMeta =
+ LogReader.readContainerMetaDataAndSkipData(
+ valueStream, out);
+ out.printf(PER_LOG_FILE_INFO_PATTERN,
+ logMeta.getFirst(), logMeta.getSecond());
} catch (EOFException eof) {
break;
}
@@ -473,7 +486,9 @@ public class LogCLIHelpers implements Configurable {
err.println("Can not find log metadata for container: "
+ containerIdStr);
}
+ return -1;
}
+ return 0;
}
@Private
@@ -501,6 +516,60 @@ public class LogCLIHelpers implements Configurable {
}
}
+ @Private
+ public void printContainersList(ContainerLogsRequest options,
+ PrintStream out, PrintStream err) throws IOException {
+ ApplicationId appId = options.getAppId();
+ String appOwner = options.getAppOwner();
+ String nodeId = options.getNodeId();
+ String nodeIdStr = (nodeId == null) ? null
+ : LogAggregationUtils.getNodeString(nodeId);
+ RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
+ appId, appOwner);
+ if (nodeFiles == null) {
+ return;
+ }
+ boolean foundAnyLogs = false;
+ while (nodeFiles.hasNext()) {
+ FileStatus thisNodeFile = nodeFiles.next();
+ if (nodeIdStr != null) {
+ if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) {
+ continue;
+ }
+ }
+ if (!thisNodeFile.getPath().getName()
+ .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
+ AggregatedLogFormat.LogReader reader =
+ new AggregatedLogFormat.LogReader(getConf(),
+ thisNodeFile.getPath());
+ try {
+ DataInputStream valueStream;
+ LogKey key = new LogKey();
+ valueStream = reader.next(key);
+ while (valueStream != null) {
+ out.println(String.format(CONTAINER_ON_NODE_PATTERN, key,
+ thisNodeFile.getPath().getName()));
+ foundAnyLogs = true;
+ // Next container
+ key = new LogKey();
+ valueStream = reader.next(key);
+ }
+ } finally {
+ reader.close();
+ }
+ }
+ }
+ if (!foundAnyLogs) {
+ if (nodeId != null) {
+ err.println("Can not find information for any containers on "
+ + nodeId);
+ } else {
+ err.println("Can not find any container information for "
+ + "the application: " + appId);
+ }
+ }
+ }
+
private RemoteIterator<FileStatus> getRemoteNodeFileDir(ApplicationId appId,
String appOwner) throws IOException {
Path remoteAppLogDir = getRemoteAppLogDir(appId, appOwner);
@@ -621,7 +690,7 @@ public class LogCLIHelpers implements Configurable {
while (true) {
try {
String logFile = LogReader.readContainerMetaDataAndSkipData(
- valueStream, null);
+ valueStream, null).getFirst();
logTypes.add(logFile);
} catch (EOFException eof) {
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] hadoop git commit: MAPREDUCE-6733. MapReduce JerseyTest tests
failing with java.net.BindException: Address already in use. Contributed by
Vinod Kumar Vavilapalli.
Posted by su...@apache.org.
MAPREDUCE-6733. MapReduce JerseyTest tests failing with java.net.BindException: Address already in use. Contributed by Vinod Kumar Vavilapalli.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ac42f24
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ac42f24
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ac42f24
Branch: refs/heads/YARN-2915
Commit: 4ac42f243f79ac2807396da67db943ad2a501147
Parents: 672a52f
Author: Junping Du <ju...@apache.org>
Authored: Fri Jul 15 11:23:32 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:31 2016 -0700
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java | 4 ++--
.../mapreduce/v2/app/webapp/TestAMWebServicesAttempt.java | 4 ++--
.../mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java | 7 +++----
.../mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java | 4 ++--
.../hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java | 4 ++--
.../mapreduce/v2/app/webapp/TestAMWebServicesTasks.java | 4 ++--
.../hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java | 4 ++--
.../mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java | 5 ++---
.../mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java | 4 ++--
.../hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java | 5 ++---
.../mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java | 4 ++--
.../hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java | 4 ++--
12 files changed, 25 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac42f24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java
index 1fb6a08..7138196 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.MockAppContext;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
@@ -54,7 +55,6 @@ import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
@@ -64,7 +64,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
* /ws/v1/mapreduce
* /ws/v1/mapreduce/info
*/
-public class TestAMWebServices extends JerseyTest {
+public class TestAMWebServices extends JerseyTestBase {
private static Configuration conf = new Configuration();
private static MockAppContext appContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac42f24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempt.java
index 68bb964..f2e6d63 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempt.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Before;
@@ -60,7 +61,6 @@ import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
@@ -69,7 +69,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
*
* /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}/state
*/
-public class TestAMWebServicesAttempt extends JerseyTest {
+public class TestAMWebServicesAttempt extends JerseyTestBase {
private static Configuration conf = new Configuration();
private static AppContext appContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac42f24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
index 9f2e1b1..cbf8a55 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
@@ -43,9 +43,9 @@ import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
@@ -60,11 +60,10 @@ import org.xml.sax.InputSource;
import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
@@ -75,7 +74,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
* /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}
* /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}/counters
*/
-public class TestAMWebServicesAttempts extends JerseyTest {
+public class TestAMWebServicesAttempts extends JerseyTestBase {
private static Configuration conf = new Configuration();
private static AppContext appContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac42f24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java
index a7958a9..cf71b34 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
@@ -62,7 +63,6 @@ import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
@@ -71,7 +71,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
*
* /ws/v1/mapreduce/job/{jobid}/conf
*/
-public class TestAMWebServicesJobConf extends JerseyTest {
+public class TestAMWebServicesJobConf extends JerseyTestBase {
private static Configuration conf = new Configuration();
private static AppContext appContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac42f24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
index 128bb5a..a38d630 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
@@ -65,7 +66,6 @@ import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
@@ -77,7 +77,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
* /ws/v1/mapreduce/jobs/{jobid}/counters
* /ws/v1/mapreduce/jobs/{jobid}/jobattempts
*/
-public class TestAMWebServicesJobs extends JerseyTest {
+public class TestAMWebServicesJobs extends JerseyTestBase {
private static Configuration conf = new Configuration();
private static AppContext appContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac42f24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java
index b9bcc01..242c243 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
@@ -61,7 +62,6 @@ import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
@@ -72,7 +72,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
* /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}
* /ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/counters
*/
-public class TestAMWebServicesTasks extends JerseyTest {
+public class TestAMWebServicesTasks extends JerseyTestBase {
private static Configuration conf = new Configuration();
private static AppContext appContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac42f24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java
index b9b45af..ab812bb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONException;
@@ -55,7 +56,6 @@ import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
@@ -64,7 +64,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
* /ws/v1/history
* /ws/v1/history/info
*/
-public class TestHsWebServices extends JerseyTest {
+public class TestHsWebServices extends JerseyTestBase {
private static Configuration conf = new Configuration();
private static HistoryContext appContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac42f24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java
index f7b137f..38058b6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java
@@ -45,9 +45,9 @@ import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.hs.HistoryContext;
import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONArray;
@@ -67,7 +67,6 @@ import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
@@ -79,7 +78,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
* /ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}/
* counters
*/
-public class TestHsWebServicesAttempts extends JerseyTest {
+public class TestHsWebServicesAttempts extends JerseyTestBase {
private static Configuration conf = new Configuration();
private static HistoryContext appContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac42f24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java
index a222302..53c5adc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONArray;
@@ -66,7 +67,6 @@ import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
@@ -75,7 +75,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
*
* /ws/v1/history/mapreduce/jobs/{jobid}/conf
*/
-public class TestHsWebServicesJobConf extends JerseyTest {
+public class TestHsWebServicesJobConf extends JerseyTestBase {
private static Configuration conf = new Configuration();
private static HistoryContext appContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac42f24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
index b2df3c6..665815b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
@@ -43,9 +43,9 @@ import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.hs.HistoryContext;
import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
-import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONArray;
@@ -65,7 +65,6 @@ import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
@@ -76,7 +75,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
* /ws/v1/history/mapreduce/jobs/{jobid}/counters
* /ws/v1/history/mapreduce/jobs/{jobid}/jobattempts
*/
-public class TestHsWebServicesJobs extends JerseyTest {
+public class TestHsWebServicesJobs extends JerseyTestBase {
private static Configuration conf = new Configuration();
private static MockHistoryContext appContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac42f24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java
index 343fa6d..c452fd9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONArray;
@@ -55,7 +56,6 @@ import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
@@ -64,7 +64,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
*
* /ws/v1/history/mapreduce/jobs?{query=value}
*/
-public class TestHsWebServicesJobsQuery extends JerseyTest {
+public class TestHsWebServicesJobsQuery extends JerseyTestBase {
private static Configuration conf = new Configuration();
private static MockHistoryContext appContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac42f24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java
index c05c151..0dbf29e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONArray;
@@ -65,7 +66,6 @@ import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
/**
@@ -76,7 +76,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
* /ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}
* /ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/counters
*/
-public class TestHsWebServicesTasks extends JerseyTest {
+public class TestHsWebServicesTasks extends JerseyTestBase {
private static Configuration conf = new Configuration();
private static MockHistoryContext appContext;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] hadoop git commit: Revert "YARN=5181. ClusterNodeTracker: add
method to get list of nodes matching a specific resourceName. (kasha via
asuresh)"
Posted by su...@apache.org.
Revert "YARN=5181. ClusterNodeTracker: add method to get list of nodes matching a specific resourceName. (kasha via asuresh)"
This reverts commit e905a42a2c6c88210b5c2120d7daf0bb3739cd1d.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a71a1a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a71a1a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a71a1a7
Branch: refs/heads/YARN-2915
Commit: 8a71a1a7d750fec26b92e65a27c13740e4048a85
Parents: 6e0b385
Author: Arun Suresh <as...@apache.org>
Authored: Tue Jul 19 10:43:19 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:33 2016 -0700
----------------------------------------------------------------------
.../scheduler/ClusterNodeTracker.java | 56 ++++------------
.../scheduler/TestClusterNodeTracker.java | 69 --------------------
2 files changed, 11 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a71a1a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index e487f69..413bbdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -18,13 +18,11 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
-import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -52,8 +50,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
private Lock writeLock = readWriteLock.writeLock();
private HashMap<NodeId, N> nodes = new HashMap<>();
- private Map<String, N> nodeNameToNodeMap = new HashMap<>();
- private Map<String, List<N>> nodesPerRack = new HashMap<>();
+ private Map<String, Integer> nodesPerRack = new HashMap<>();
private Resource clusterCapacity = Resources.clone(Resources.none());
private Resource staleClusterCapacity = null;
@@ -69,16 +66,14 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
writeLock.lock();
try {
nodes.put(node.getNodeID(), node);
- nodeNameToNodeMap.put(node.getNodeName(), node);
// Update nodes per rack as well
String rackName = node.getRackName();
- List<N> nodesList = nodesPerRack.get(rackName);
- if (nodesList == null) {
- nodesList = new ArrayList<>();
- nodesPerRack.put(rackName, nodesList);
+ Integer numNodes = nodesPerRack.get(rackName);
+ if (numNodes == null) {
+ numNodes = 0;
}
- nodesList.add(node);
+ nodesPerRack.put(rackName, ++numNodes);
// Update cluster capacity
Resources.addTo(clusterCapacity, node.getTotalResource());
@@ -131,8 +126,8 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
readLock.lock();
String rName = rackName == null ? "NULL" : rackName;
try {
- List<N> nodesList = nodesPerRack.get(rName);
- return nodesList == null ? 0 : nodesList.size();
+ Integer nodeCount = nodesPerRack.get(rName);
+ return nodeCount == null ? 0 : nodeCount;
} finally {
readLock.unlock();
}
@@ -159,18 +154,14 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
LOG.warn("Attempting to remove a non-existent node " + nodeId);
return null;
}
- nodeNameToNodeMap.remove(node.getNodeName());
// Update nodes per rack as well
String rackName = node.getRackName();
- List<N> nodesList = nodesPerRack.get(rackName);
- if (nodesList == null) {
- LOG.error("Attempting to remove node from an empty rack " + rackName);
+ Integer numNodes = nodesPerRack.get(rackName);
+ if (numNodes > 0) {
+ nodesPerRack.put(rackName, --numNodes);
} else {
- nodesList.remove(node);
- if (nodesList.isEmpty()) {
- nodesPerRack.remove(rackName);
- }
+ LOG.error("Attempting to remove node from an empty rack " + rackName);
}
// Update cluster capacity
@@ -306,29 +297,4 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
Collections.sort(sortedList, comparator);
return sortedList;
}
-
- /**
- * Convenience method to return list of nodes corresponding to resourceName
- * passed in the {@link ResourceRequest}.
- *
- * @param resourceName Host/rack name of the resource, or
- * {@link ResourceRequest#ANY}
- * @return list of nodes that match the resourceName
- */
- public List<N> getNodesByResourceName(final String resourceName) {
- Preconditions.checkArgument(
- resourceName != null && !resourceName.isEmpty());
- List<N> retNodes = new ArrayList<>();
- if (ResourceRequest.ANY.equals(resourceName)) {
- return getAllNodes();
- } else if (nodeNameToNodeMap.containsKey(resourceName)) {
- retNodes.add(nodeNameToNodeMap.get(resourceName));
- } else if (nodesPerRack.containsKey(resourceName)) {
- return nodesPerRack.get(resourceName);
- } else {
- LOG.info(
- "Could not find a node matching given resourceName " + resourceName);
- }
- return retNodes;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a71a1a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
deleted file mode 100644
index 7f527f1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestClusterNodeTracker.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSSchedulerNode;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test class to verify ClusterNodeTracker. Using FSSchedulerNode without
- * loss of generality.
- */
-public class TestClusterNodeTracker {
- private ClusterNodeTracker<FSSchedulerNode> nodeTracker =
- new ClusterNodeTracker<>();
-
- @Before
- public void setup() {
- List<RMNode> rmNodes =
- MockNodes.newNodes(2, 4, Resource.newInstance(4096, 4));
- for (RMNode rmNode : rmNodes) {
- nodeTracker.addNode(new FSSchedulerNode(rmNode, false));
- }
- }
-
- @Test
- public void testGetNodeCount() {
- assertEquals("Incorrect number of nodes in the cluster",
- 8, nodeTracker.nodeCount());
-
- assertEquals("Incorrect number of nodes in each rack",
- 4, nodeTracker.nodeCount("rack0"));
- }
-
- @Test
- public void testGetNodesForResourceName() throws Exception {
- assertEquals("Incorrect number of nodes matching ANY",
- 8, nodeTracker.getNodesByResourceName(ResourceRequest.ANY).size());
-
- assertEquals("Incorrect number of nodes matching rack",
- 4, nodeTracker.getNodesByResourceName("rack0").size());
-
- assertEquals("Incorrect number of nodes matching node",
- 1, nodeTracker.getNodesByResourceName("host0").size());
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] hadoop git commit: HDFS-10544. Balancer doesn't work with
IPFailoverProxyProvider.
Posted by su...@apache.org.
HDFS-10544. Balancer doesn't work with IPFailoverProxyProvider.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e1ef53d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e1ef53d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e1ef53d2
Branch: refs/heads/YARN-2915
Commit: e1ef53d28f987604bad5711a5459bc92e1060241
Parents: a0d8398
Author: Zhe Zhang <zh...@apache.org>
Authored: Tue Jul 12 23:18:37 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:29 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 42 ++++++++++++------
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 45 +++++++++++++++++++-
2 files changed, 73 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1ef53d2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index 5236ee5..f1a6de7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -718,10 +718,11 @@ public class DFSUtil {
/**
* Get a URI for each internal nameservice. If a nameservice is
- * HA-enabled, then the logical URI of the nameservice is returned. If the
- * nameservice is not HA-enabled, then a URI corresponding to an RPC address
- * of the single NN for that nameservice is returned, preferring the service
- * RPC address over the client RPC address.
+ * HA-enabled, and the configured failover proxy provider supports logical
+ * URIs, then the logical URI of the nameservice is returned.
+ * Otherwise, a URI corresponding to an RPC address of the single NN for that
+ * nameservice is returned, preferring the service RPC address over the
+ * client RPC address.
*
* @param conf configuration
* @return a collection of all configured NN URIs, preferring service
@@ -735,9 +736,10 @@ public class DFSUtil {
/**
* Get a URI for each configured nameservice. If a nameservice is
- * HA-enabled, then the logical URI of the nameservice is returned. If the
- * nameservice is not HA-enabled, then a URI corresponding to the address of
- * the single NN for that nameservice is returned.
+ * HA-enabled, and the configured failover proxy provider supports logical
+ * URIs, then the logical URI of the nameservice is returned.
+ * Otherwise, a URI corresponding to the address of the single NN for that
+ * nameservice is returned.
*
* @param conf configuration
* @param keys configuration keys to try in order to get the URI for non-HA
@@ -756,13 +758,27 @@ public class DFSUtil {
Set<URI> nonPreferredUris = new HashSet<URI>();
for (String nsId : nameServices) {
- if (HAUtil.isHAEnabled(conf, nsId)) {
+ URI nsUri;
+ try {
+ nsUri = new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + nsId);
+ } catch (URISyntaxException ue) {
+ throw new IllegalArgumentException(ue);
+ }
+ /**
+ * Determine whether the logical URI of the name service can be resolved
+ * by the configured failover proxy provider. If not, we should try to
+ * resolve the URI here
+ */
+ boolean useLogicalUri = false;
+ try {
+ useLogicalUri = HAUtil.useLogicalUri(conf, nsUri);
+ } catch (IOException e){
+ LOG.warn("Getting exception while trying to determine if nameservice "
+ + nsId + " can use logical URI: " + e);
+ }
+ if (HAUtil.isHAEnabled(conf, nsId) && useLogicalUri) {
// Add the logical URI of the nameservice.
- try {
- ret.add(new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + nsId));
- } catch (URISyntaxException ue) {
- throw new IllegalArgumentException(ue);
- }
+ ret.add(nsUri);
} else {
// Add the URI corresponding to the address of the NN.
boolean uriFound = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1ef53d2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index 448ef6a..b139845 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
@@ -533,7 +534,11 @@ public class TestDFSUtil {
// Ditto for nameservice IDs, if multiple are defined
assertEquals(null, DFSUtil.getNamenodeNameServiceId(conf));
assertEquals(null, DFSUtil.getSecondaryNameServiceId(conf));
-
+
+ String proxyProviderKey = HdfsClientConfigKeys.Failover.
+ PROXY_PROVIDER_KEY_PREFIX + ".ns2";
+ conf.set(proxyProviderKey, "org.apache.hadoop.hdfs.server.namenode.ha."
+ + "ConfiguredFailoverProxyProvider");
Collection<URI> uris = getInternalNameServiceUris(conf, DFS_NAMENODE_RPC_ADDRESS_KEY);
assertEquals(2, uris.size());
assertTrue(uris.contains(new URI("hdfs://ns1")));
@@ -633,6 +638,7 @@ public class TestDFSUtil {
public void testGetNNUris() throws Exception {
HdfsConfiguration conf = new HdfsConfiguration();
+ final String NS1_NN_ADDR = "ns1-nn.example.com:9820";
final String NS1_NN1_ADDR = "ns1-nn1.example.com:9820";
final String NS1_NN2_ADDR = "ns1-nn2.example.com:9820";
final String NS2_NN_ADDR = "ns2-nn.example.com:9820";
@@ -665,12 +671,49 @@ public class TestDFSUtil {
DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_ADDR);
conf.set(DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1"), NS1_NN_ADDR);
+
+ conf.set(DFSUtil.addKeySuffixes(
DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns2"), NS2_NN_ADDR);
conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, "hdfs://" + NN1_ADDR);
conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN2_ADDR);
+ /**
+ * {@link DFSUtil#getInternalNsRpcUris} decides whether to resolve a logical
+ * URI based on whether the failover proxy provider supports logical URIs.
+ * We will test both cases.
+ *
+ * First configure ns1 to use {@link IPFailoverProxyProvider} which doesn't
+ * support logical Uris. So {@link DFSUtil#getInternalNsRpcUris} will
+ * resolve the logical URI of ns1 based on the configured value at
+ * dfs.namenode.servicerpc-address.ns1, which is {@link NS1_NN_ADDR}
+ */
+ String proxyProviderKey = HdfsClientConfigKeys.Failover.
+ PROXY_PROVIDER_KEY_PREFIX + ".ns1";
+ conf.set(proxyProviderKey, "org.apache.hadoop.hdfs.server.namenode.ha."
+ + "IPFailoverProxyProvider");
+
+ uris = DFSUtil.getInternalNsRpcUris(conf);
+ assertEquals("Incorrect number of URIs returned", 3, uris.size());
+ assertTrue("Missing URI for RPC address",
+ uris.contains(new URI("hdfs://" + NN1_ADDR)));
+ assertTrue("Missing URI for name service ns2",
+ uris.contains(new URI(HdfsConstants.HDFS_URI_SCHEME + "://" +
+ NS1_NN_ADDR)));
+ assertTrue("Missing URI for name service ns2",
+ uris.contains(new URI(HdfsConstants.HDFS_URI_SCHEME + "://" +
+ NS2_NN_ADDR)));
+
+ /**
+ * Second, test ns1 with {@link ConfiguredFailoverProxyProvider} which does
+ * support logical URIs. So instead of {@link NS1_NN_ADDR}, the logical URI
+ * of ns1, hdfs://ns1, will be returned.
+ */
+ conf.set(proxyProviderKey, "org.apache.hadoop.hdfs.server.namenode.ha."
+ + "ConfiguredFailoverProxyProvider");
+
uris = DFSUtil.getInternalNsRpcUris(conf);
assertEquals("Incorrect number of URIs returned", 3, uris.size());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] hadoop git commit: HADOOP-13301. Millisecond timestamp for
FsShell console log and MapReduce jobsummary log. Contributed by John Zhuge.
Posted by su...@apache.org.
HADOOP-13301. Millisecond timestamp for FsShell console log and MapReduce jobsummary log. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5cab56c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5cab56c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5cab56c3
Branch: refs/heads/YARN-2915
Commit: 5cab56c3ff6c471389f07ae8f0dd2e3ef5fb8419
Parents: aa55067
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Jul 18 15:45:44 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
.../hadoop-common/src/main/conf/log4j.properties | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cab56c3/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index 95afc61..b8c84e7 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -73,7 +73,7 @@ log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
#
# TaskLog Appender
@@ -227,7 +227,7 @@ log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file
log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize}
log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex}
log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
-log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+log4j.appender.JSA.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger}
log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] hadoop git commit: HDFS-10617.
PendingReconstructionBlocks.size() should be synchronized. Contributed by
Eric Badger.
Posted by su...@apache.org.
HDFS-10617. PendingReconstructionBlocks.size() should be synchronized. Contributed by Eric Badger.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/288eade0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/288eade0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/288eade0
Branch: refs/heads/YARN-2915
Commit: 288eade0867ac373c065cc3ee7a937fdb8362b48
Parents: 60ac09c
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Jul 13 21:59:35 2016 -0500
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../hdfs/server/blockmanagement/PendingReconstructionBlocks.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288eade0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
index 956e94f..6628c43 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
@@ -134,7 +134,9 @@ class PendingReconstructionBlocks {
* The total number of blocks that are undergoing reconstruction.
*/
int size() {
- return pendingReconstructions.size();
+ synchronized (pendingReconstructions) {
+ return pendingReconstructions.size();
+ }
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] hadoop git commit: HADOOP-13202. Avoid possible overflow in
org.apache.hadoop.util.bloom.BloomFilter#getNBytes. Contributed by Kai
Sasaki.
Posted by su...@apache.org.
HADOOP-13202. Avoid possible overflow in org.apache.hadoop.util.bloom.BloomFilter#getNBytes. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa55067d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa55067d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa55067d
Branch: refs/heads/YARN-2915
Commit: aa55067de662b286d0add3421fb3dd6cd5ed4de6
Parents: 825310a
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Jul 18 11:42:56 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/util/bloom/BloomFilter.java | 2 +-
.../org/apache/hadoop/util/bloom/TestBloomFilters.java | 12 ++++++++++++
2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa55067d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/BloomFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/BloomFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/BloomFilter.java
index f8b9519..82d18b8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/BloomFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/BloomFilter.java
@@ -234,6 +234,6 @@ public class BloomFilter extends Filter {
/* @return number of bytes needed to hold bit vector */
private int getNBytes() {
- return (vectorSize + 7) / 8;
+ return (int)(((long)vectorSize + 7) / 8);
}
}//end class
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa55067d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java
index 6ff854d..8552565 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java
@@ -240,6 +240,18 @@ public class TestBloomFilters {
}
@Test
+ public void testFiltersWithLargeVectorSize() {
+ int hashId = Hash.MURMUR_HASH;
+ Filter filter
+ = new BloomFilter(Integer.MAX_VALUE, hashFunctionNumber, hashId);
+ BloomFilterCommonTester.of(hashId, numInsertions)
+ .withFilterInstance(filter)
+ .withTestCases(ImmutableSet.of(
+ BloomFilterTestStrategy.WRITE_READ_STRATEGY
+ )).test();
+ }
+
+ @Test
public void testNot() {
BloomFilter bf = new BloomFilter(8, 1, Hash.JENKINS_HASH);
bf.bits = BitSet.valueOf(new byte[] { (byte) 0x95 });
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] hadoop git commit: YARN-5339. Passing file to -out for YARN
log CLI doesnt give warning or error code. Contributed by Xuan Gong.
Posted by su...@apache.org.
YARN-5339. Passing file to -out for YARN log CLI doesnt give warning or error code. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eee5643d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eee5643d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eee5643d
Branch: refs/heads/YARN-2915
Commit: eee5643d6e10732e650edb78f7cd25047b1c7e86
Parents: 339c76b
Author: Junping Du <ju...@apache.org>
Authored: Fri Jul 15 09:23:11 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:31 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 30 ++++++++++++++++----
.../logaggregation/ContainerLogsRequest.java | 14 ++++++++-
2 files changed, 38 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee5643d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 2153e1e..e2c4552 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -26,7 +26,9 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import java.util.regex.Pattern;
import javax.ws.rs.core.MediaType;
@@ -51,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -249,7 +252,7 @@ public class LogsCLI extends Configured implements Tool {
ContainerLogsRequest request = new ContainerLogsRequest(appId,
isApplicationFinished(appState), appOwner, nodeAddress, null,
- containerIdStr, localDir, logs, bytes);
+ containerIdStr, localDir, logs, bytes, null);
if (showContainerLogInfo) {
return showContainerLogInfo(request, logCliHelper);
@@ -470,9 +473,14 @@ public class LogsCLI extends Configured implements Tool {
.queryParam("size", Long.toString(request.getBytes()))
.accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
out.println(response.getEntity(String.class));
- out.println("End of LogType:" + logFile + ". This log file belongs"
- + " to a running container (" + containerIdStr + ") and so may"
- + " not be complete.");
+ StringBuilder sb = new StringBuilder();
+ sb.append("End of LogType:" + logFile + ".");
+ if (request.getContainerState() == ContainerState.RUNNING) {
+ sb.append(" This log file belongs"
+ + " to a running container (" + containerIdStr + ") and so may"
+ + " not be complete.");
+ }
+ out.println(sb.toString());
out.flush();
foundAnyLogs = true;
} catch (ClientHandlerException | UniformInterfaceException ex) {
@@ -645,6 +653,9 @@ public class LogsCLI extends Configured implements Tool {
} else {
if (nodeHttpAddress != null && containerId != null
&& !nodeHttpAddress.isEmpty() && !containerId.isEmpty()) {
+ ContainerState containerState = getContainerReport(containerId)
+ .getContainerState();
+ request.setContainerState(containerState);
printContainerLogsFromRunningApplication(conf,
request, logCliHelper);
}
@@ -880,6 +891,7 @@ public class LogsCLI extends Configured implements Tool {
}
nodeId = report.getAssignedNode().toString();
request.setNodeId(nodeId);
+ request.setContainerState(report.getContainerState());
} catch (IOException | YarnException ex) {
if (isAppFinished) {
return printContainerLogsForFinishedApplicationWithoutNodeId(
@@ -1020,6 +1032,7 @@ public class LogsCLI extends Configured implements Tool {
newOptions.setNodeHttpAddress(httpAddress
.replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
}
+ newOptions.setContainerState(container.getContainerState());
newOptionsList.add(newOptions);
}
return newOptionsList;
@@ -1030,11 +1043,18 @@ public class LogsCLI extends Configured implements Tool {
List<ContainerReport> reports = new ArrayList<ContainerReport>();
List<ApplicationAttemptReport> attempts =
yarnClient.getApplicationAttempts(options.getAppId());
+ Map<ContainerId, ContainerReport> containerMap = new TreeMap<
+ ContainerId, ContainerReport>();
for (ApplicationAttemptReport attempt : attempts) {
List<ContainerReport> containers = yarnClient.getContainers(
attempt.getApplicationAttemptId());
- reports.addAll(containers);
+ for (ContainerReport container : containers) {
+ if (!containerMap.containsKey(container.getContainerId())) {
+ containerMap.put(container.getContainerId(), container);
+ }
+ }
}
+ reports.addAll(containerMap.values());
return reports;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eee5643d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java
index f32285c..af65fef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.logaggregation;
import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
public class ContainerLogsRequest {
private ApplicationId appId;
@@ -31,6 +32,7 @@ public class ContainerLogsRequest {
private String outputLocalDir;
private List<String> logTypes;
private long bytes;
+ private ContainerState containerState;
public ContainerLogsRequest() {}
@@ -44,12 +46,13 @@ public class ContainerLogsRequest {
this.setOutputLocalDir(request.getOutputLocalDir());
this.setLogTypes(request.getLogTypes());
this.setBytes(request.getBytes());
+ this.setContainerState(request.getContainerState());
}
public ContainerLogsRequest(ApplicationId applicationId,
boolean isAppFinished, String owner,
String address, String httpAddress, String container, String localDir,
- List<String> logs, long bytes) {
+ List<String> logs, long bytes, ContainerState containerState) {
this.setAppId(applicationId);
this.setAppFinished(isAppFinished);
this.setAppOwner(owner);
@@ -59,6 +62,7 @@ public class ContainerLogsRequest {
this.setOutputLocalDir(localDir);
this.setLogTypes(logs);
this.setBytes(bytes);
+ this.setContainerState(containerState);
}
public ApplicationId getAppId() {
@@ -132,4 +136,12 @@ public class ContainerLogsRequest {
public void setBytes(long bytes) {
this.bytes = bytes;
}
+
+ public ContainerState getContainerState() {
+ return containerState;
+ }
+
+ public void setContainerState(ContainerState containerState) {
+ this.containerState = containerState;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] hadoop git commit: YARN-5213. Fix a bug in LogCLIHelpers
which cause TestLogsCLI#testFetchApplictionLogs fails intermittently.
Contributed by Xuan Gong.
Posted by su...@apache.org.
YARN-5213. Fix a bug in LogCLIHelpers which cause TestLogsCLI#testFetchApplictionLogs fails intermittently. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e0b3858
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e0b3858
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e0b3858
Branch: refs/heads/YARN-2915
Commit: 6e0b38589dc401027cb6aae93bd21d3db74b3116
Parents: 82c4d7d
Author: Junping Du <ju...@apache.org>
Authored: Tue Jul 19 07:17:58 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/client/cli/TestLogsCLI.java | 66 +++++++++++---------
.../yarn/logaggregation/LogCLIHelpers.java | 11 ++++
2 files changed, 46 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e0b3858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index bda489f..329d856 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -329,26 +329,26 @@ public class TestLogsCLI {
int exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000001 in syslog!"));
+ logMessage(containerId1, "syslog")));
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000002 in syslog!"));
+ logMessage(containerId2, "syslog")));
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in syslog!"));
+ logMessage(containerId3, "syslog")));
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in stdout!"));
+ logMessage(containerId3, "stdout")));
sysOutStream.reset();
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
"-log_files", ".*"});
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000001 in syslog!"));
+ logMessage(containerId1, "syslog")));
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000002 in syslog!"));
+ logMessage(containerId2, "syslog")));
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in syslog!"));
+ logMessage(containerId3, "syslog")));
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in stdout!"));
+ logMessage(containerId3, "stdout")));
int fullSize = sysOutStream.toByteArray().length;
sysOutStream.reset();
@@ -356,13 +356,13 @@ public class TestLogsCLI {
"-log_files", "std*"});
assertTrue(exitCode == 0);
assertFalse(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000001 in syslog!"));
+ logMessage(containerId1, "syslog")));
assertFalse(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000002 in syslog!"));
+ logMessage(containerId2, "syslog")));
assertFalse(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in syslog!"));
+ logMessage(containerId3, "syslog")));
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in stdout!"));
+ logMessage(containerId3, "stdout")));
sysOutStream.reset();
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
@@ -390,7 +390,7 @@ public class TestLogsCLI {
containerId1.toString() });
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000001 in syslog!"));
+ logMessage(containerId1, "syslog")));
assertTrue(sysOutStream.toString().contains("Log Upload Time"));
assertTrue(!sysOutStream.toString().contains(
"Logs for container " + containerId1.toString()
@@ -413,9 +413,9 @@ public class TestLogsCLI {
String fullContext = sysOutStream.toString();
sysOutStream.reset();
- String logMessage = "Hello container_0_0001_01_000003 in stdout!";
+ String logMessage = logMessage(containerId3, "stdout");
int fileContentSize = logMessage.getBytes().length;
- int tailContentSize = "\nEnd of LogType:syslog\n\n".getBytes().length;
+ int tailContentSize = "\nEnd of LogType:stdout\n\n".getBytes().length;
// specify how many bytes we should get from logs
// specify a position number, it would get the first n bytes from
@@ -470,9 +470,9 @@ public class TestLogsCLI {
containerId3.toString() });
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in syslog!"));
+ logMessage(containerId3, "syslog")));
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in stdout!"));
+ logMessage(containerId3, "stdout")));
sysOutStream.reset();
// set -log_files option as stdout
@@ -483,9 +483,9 @@ public class TestLogsCLI {
containerId3.toString() , "-log_files", "stdout"});
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in stdout!"));
+ logMessage(containerId3, "stdout")));
assertTrue(!sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in syslog!"));
+ logMessage(containerId3, "syslog")));
sysOutStream.reset();
YarnClient mockYarnClientWithException =
@@ -498,9 +498,9 @@ public class TestLogsCLI {
"-containerId", containerId3.toString() });
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in syslog!"));
+ logMessage(containerId3, "syslog")));
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in stdout!"));
+ logMessage(containerId3, "stdout")));
assertTrue(sysOutStream.toString().contains(
containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId)));
sysOutStream.reset();
@@ -510,9 +510,9 @@ public class TestLogsCLI {
cli.run(new String[] { "-containerId", containerId3.toString() });
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in syslog!"));
+ logMessage(containerId3, "syslog")));
assertTrue(sysOutStream.toString().contains(
- "Hello container_0_0001_01_000003 in stdout!"));
+ logMessage(containerId3, "stdout")));
assertTrue(sysOutStream.toString().contains(
containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId)));
sysOutStream.reset();
@@ -662,7 +662,7 @@ public class TestLogsCLI {
"-appOwner", testUser});
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
- "Hello " + containerId + " in syslog!"));
+ logMessage(containerId, "syslog")));
sysOutStream.reset();
// Verify that we can not get the application logs
@@ -684,8 +684,8 @@ public class TestLogsCLI {
exitCode = cli.run(new String[] {
"-applicationId", appId.toString()});
assertTrue(exitCode == 0);
- assertTrue(sysOutStream.toString().contains("Hello "
- + containerId + " in syslog!"));
+ assertTrue(sysOutStream.toString().contains(
+ logMessage(containerId, "syslog")));
sysOutStream.reset();
// Verify that we could get the err message "Can not find the appOwner"
@@ -847,16 +847,14 @@ public class TestLogsCLI {
assertTrue(container1Dir[0].getPath().getName().equals(
containerId1.toString()));
String container1= readContainerContent(container1Dir[0].getPath(), fs);
- assertTrue(container1.contains("Hello " + containerId1
- + " in syslog!"));
+ assertTrue(container1.contains(logMessage(containerId1, "syslog")));
FileStatus[] container2Dir = fs.listStatus(nodeDir[1].getPath());
assertTrue(container2Dir.length == 1);
assertTrue(container2Dir[0].getPath().getName().equals(
containerId2.toString()));
String container2= readContainerContent(container2Dir[0].getPath(), fs);
- assertTrue(container2.contains("Hello " + containerId2
- + " in syslog!"));
+ assertTrue(container2.contains(logMessage(containerId2, "syslog")));
} finally {
fs.delete(new Path(remoteLogRootDir), true);
fs.delete(new Path(rootLogDir), true);
@@ -1139,11 +1137,17 @@ public class TestLogsCLI {
for (String logType : logTypes) {
Writer writer =
new FileWriter(new File(containerLogsDir.toString(), logType));
- writer.write("Hello " + containerId + " in " + logType + "!");
+ writer.write(logMessage(containerId, logType));
writer.close();
}
}
+ private static String logMessage(ContainerId containerId, String logType) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Hello " + containerId + " in " + logType + "!");
+ return sb.toString();
+ }
+
private static void uploadContainerLogIntoRemoteDir(UserGroupInformation ugi,
Configuration configuration, List<String> rootLogDirs, NodeId nodeId,
ContainerId containerId, Path appDir, FileSystem fs) throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e0b3858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index ef56504..13934d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -157,10 +157,18 @@ public class LogCLIHelpers implements Configurable {
AggregatedLogFormat.LogReader reader = null;
PrintStream out = createPrintStream(localDir, fileName, containerId);
try {
+ reader = new AggregatedLogFormat.LogReader(getConf(),
+ thisNodeFile.getPath());
+ if (getContainerLogsStream(containerId, reader) == null) {
+ continue;
+ }
String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
containerId, thisNodeFile.getPath().getName());
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
+ // We have to re-create reader object to reset the stream index
+ // after calling getContainerLogsStream which would move the stream
+ // index to the end of the log file.
reader =
new AggregatedLogFormat.LogReader(getConf(),
thisNodeFile.getPath());
@@ -220,6 +228,9 @@ public class LogCLIHelpers implements Configurable {
if (getContainerLogsStream(containerId, reader) == null) {
continue;
}
+ // We have to re-create reader object to reset the stream index
+ // after calling getContainerLogsStream which would move the stream
+ // index to the end of the log file.
reader =
new AggregatedLogFormat.LogReader(getConf(),
thisNodeFile.getPath());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] hadoop git commit: YARN-5380. NMTimelinePublisher should use
getMemorySize instead of getMemory. Contributed by Vrushali C
Posted by su...@apache.org.
YARN-5380. NMTimelinePublisher should use getMemorySize instead of getMemory. Contributed by Vrushali C
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/339c76b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/339c76b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/339c76b5
Branch: refs/heads/YARN-2915
Commit: 339c76b53f9d3c7f0f242cbb39f0fe4eefae97df
Parents: 16872b3
Author: Naganarasimha <na...@apache.org>
Authored: Fri Jul 15 14:34:48 2016 +0530
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:31 2016 -0700
----------------------------------------------------------------------
.../server/nodemanager/timelineservice/NMTimelinePublisher.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/339c76b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index c4d45a9..0371863 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -168,7 +168,7 @@ public class NMTimelinePublisher extends CompositeService {
Map<String, Object> entityInfo = new HashMap<String, Object>();
entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO,
- resource.getMemory());
+ resource.getMemorySize());
entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO,
resource.getVirtualCores());
entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] hadoop git commit: HDFS-10600.
PlanCommand#getThrsholdPercentage should not use throughput value. (Yiqun Lin
via lei)
Posted by su...@apache.org.
HDFS-10600. PlanCommand#getThrsholdPercentage should not use throughput value. (Yiqun Lin via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/678d0d4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/678d0d4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/678d0d4f
Branch: refs/heads/YARN-2915
Commit: 678d0d4f56d1c5f0eb490000cb474799429c3cc8
Parents: 7bd489c
Author: Lei Xu <le...@apache.org>
Authored: Thu Jul 14 10:40:34 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:31 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 ++++++--
.../apache/hadoop/hdfs/server/datanode/DiskBalancer.java | 10 +++++-----
.../hdfs/server/diskbalancer/command/PlanCommand.java | 4 ++--
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 9 +++++++++
4 files changed, 22 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/678d0d4f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index e734055..40711a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -947,9 +947,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.disk.balancer.enabled";
public static final boolean DFS_DISK_BALANCER_ENABLED_DEFAULT = false;
- public static final String DFS_DISK_BALANCER_MAX_DISK_THRUPUT =
+ public static final String DFS_DISK_BALANCER_MAX_DISK_THROUGHPUT =
"dfs.disk.balancer.max.disk.throughputInMBperSec";
- public static final int DFS_DISK_BALANCER_MAX_DISK_THRUPUT_DEFAULT =
+ public static final int DFS_DISK_BALANCER_MAX_DISK_THROUGHPUT_DEFAULT =
10;
public static final String DFS_DISK_BALANCER_MAX_DISK_ERRORS =
@@ -961,6 +961,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.disk.balancer.block.tolerance.percent";
public static final int DFS_DISK_BALANCER_BLOCK_TOLERANCE_DEFAULT = 5;
+ public static final String DFS_DISK_BALANCER_PLAN_THRESHOLD =
+ "dfs.disk.balancer.plan.threshold.percent";
+ public static final int DFS_DISK_BALANCER_PLAN_THRESHOLD_DEFAULT = 10;
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
http://git-wip-us.apache.org/repos/asf/hadoop/blob/678d0d4f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
index 5a1fb9e..c6948f88 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
@@ -109,8 +109,8 @@ public class DiskBalancer {
DFSConfigKeys.DFS_DISK_BALANCER_ENABLED,
DFSConfigKeys.DFS_DISK_BALANCER_ENABLED_DEFAULT);
this.bandwidth = conf.getInt(
- DFSConfigKeys.DFS_DISK_BALANCER_MAX_DISK_THRUPUT,
- DFSConfigKeys.DFS_DISK_BALANCER_MAX_DISK_THRUPUT_DEFAULT);
+ DFSConfigKeys.DFS_DISK_BALANCER_MAX_DISK_THROUGHPUT,
+ DFSConfigKeys.DFS_DISK_BALANCER_MAX_DISK_THROUGHPUT_DEFAULT);
}
/**
@@ -656,8 +656,8 @@ public class DiskBalancer {
shouldRun = new AtomicBoolean(false);
this.diskBandwidth = conf.getLong(
- DFSConfigKeys.DFS_DISK_BALANCER_MAX_DISK_THRUPUT,
- DFSConfigKeys.DFS_DISK_BALANCER_MAX_DISK_THRUPUT_DEFAULT);
+ DFSConfigKeys.DFS_DISK_BALANCER_MAX_DISK_THROUGHPUT,
+ DFSConfigKeys.DFS_DISK_BALANCER_MAX_DISK_THROUGHPUT_DEFAULT);
this.blockTolerance = conf.getLong(
DFSConfigKeys.DFS_DISK_BALANCER_BLOCK_TOLERANCE,
@@ -673,7 +673,7 @@ public class DiskBalancer {
LOG.debug("Found 0 or less as max disk throughput, ignoring config " +
"value. value : " + diskBandwidth);
diskBandwidth =
- DFSConfigKeys.DFS_DISK_BALANCER_MAX_DISK_THRUPUT_DEFAULT;
+ DFSConfigKeys.DFS_DISK_BALANCER_MAX_DISK_THROUGHPUT_DEFAULT;
}
if (this.blockTolerance <= 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/678d0d4f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
index 54a63ec..3159312 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
@@ -214,8 +214,8 @@ public class PlanCommand extends Command {
if ((value <= 0.0) || (value > 100.0)) {
value = getConf().getDouble(
- DFSConfigKeys.DFS_DISK_BALANCER_MAX_DISK_THRUPUT,
- DFSConfigKeys.DFS_DISK_BALANCER_MAX_DISK_THRUPUT_DEFAULT);
+ DFSConfigKeys.DFS_DISK_BALANCER_PLAN_THRESHOLD,
+ DFSConfigKeys.DFS_DISK_BALANCER_PLAN_THRESHOLD_DEFAULT);
}
return value;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/678d0d4f/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index c8dc66b..698a0cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4158,4 +4158,13 @@
balancer is disabled.
</description>
</property>
+
+ <property>
+ <name>dfs.disk.balancer.plan.threshold.percent</name>
+ <value>10</value>
+ <description>
+ The percentage that disk tolerance that we are ok with in
+ a plan.
+ </description>
+ </property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] hadoop git commit: YARN-5353. ResourceManager can leak
delegation tokens when they are shared across apps. (Jason Lowe via Varun
Saxena).
Posted by su...@apache.org.
YARN-5353. ResourceManager can leak delegation tokens when they are shared across apps. (Jason Lowe via Varun Saxena).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0d8398e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0d8398e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0d8398e
Branch: refs/heads/YARN-2915
Commit: a0d8398e234b0778c3009f06ce5ebee44ec36039
Parents: bc7bfe0
Author: Varun Saxena <va...@apache.org>
Authored: Wed Jul 13 07:55:34 2016 +0530
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:29 2016 -0700
----------------------------------------------------------------------
.../resourcemanager/security/DelegationTokenRenewer.java | 7 +------
.../resourcemanager/security/TestDelegationTokenRenewer.java | 3 +++
2 files changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0d8398e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
index 4177ee2..dfbf333 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
@@ -773,7 +773,7 @@ public class DelegationTokenRenewer extends AbstractService {
private void removeApplicationFromRenewal(ApplicationId applicationId) {
rmContext.getSystemCredentialsForApps().remove(applicationId);
- Set<DelegationTokenToRenew> tokens = appTokens.get(applicationId);
+ Set<DelegationTokenToRenew> tokens = appTokens.remove(applicationId);
if (tokens != null && !tokens.isEmpty()) {
synchronized (tokens) {
@@ -798,15 +798,10 @@ public class DelegationTokenRenewer extends AbstractService {
// cancel the token
cancelToken(dttr);
- it.remove();
allTokens.remove(dttr.token);
}
}
}
-
- if(tokens != null && tokens.isEmpty()) {
- appTokens.remove(applicationId);
- }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0d8398e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
index 4aee3c1..5dfee89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
@@ -1254,6 +1254,9 @@ public class TestDelegationTokenRenewer {
Assert.assertTrue(dttr.referringAppIds.isEmpty());
Assert.assertTrue(dttr.isTimerCancelled());
Assert.assertTrue(Renewer.cancelled);
+
+ // make sure the token also has been removed from appTokens
+ Assert.assertFalse(renewer.getDelegationTokens().contains(token1));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] hadoop git commit: MAPREDUCE-6625. TestCLI#testGetJob fails
occasionally (haibochen via rkanter)
Posted by su...@apache.org.
MAPREDUCE-6625. TestCLI#testGetJob fails occasionally (haibochen via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef14fa6a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef14fa6a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef14fa6a
Branch: refs/heads/YARN-2915
Commit: ef14fa6a1a02ef3cf1291f2cb5aa5666505d5281
Parents: 4972edc
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Jul 12 15:51:18 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:29 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/mapreduce/tools/TestCLI.java | 47 ++++++++++++--------
1 file changed, 28 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef14fa6a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/tools/TestCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/tools/TestCLI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/tools/TestCLI.java
index 112f585..985eea3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/tools/TestCLI.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/tools/TestCLI.java
@@ -31,9 +31,10 @@ import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.JobPriority;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.JobStatus.State;
-import org.apache.hadoop.util.Time;
import org.junit.Assert;
import org.junit.Test;
+
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.times;
@@ -162,25 +163,33 @@ public class TestCLI {
}
@Test
- public void testGetJob() throws Exception {
+ public void testGetJobWithoutRetry() throws Exception {
Configuration conf = new Configuration();
- long sleepTime = 100;
- conf.setLong(MRJobConfig.MR_CLIENT_JOB_RETRY_INTERVAL, sleepTime);
- Cluster mockCluster = mock(Cluster.class);
- JobID jobId1 = JobID.forName("job_1234654654_001");
- when(mockCluster.getJob(jobId1)).thenReturn(null);
-
- for (int i = 0; i < 2; ++i) {
- conf.setInt(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, i);
- CLI cli = spy(new CLI(conf));
- cli.cluster = mockCluster;
- doReturn(mockCluster).when(cli).createCluster();
- long start = Time.monotonicNow();
- cli.getJob(jobId1);
- long end = Time.monotonicNow();
- Assert.assertTrue(end - start > (i * sleepTime));
- Assert.assertTrue(end - start < ((i + 1) * sleepTime));
- }
+ conf.setInt(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, 0);
+
+ final Cluster mockCluster = mock(Cluster.class);
+ when(mockCluster.getJob(any(JobID.class))).thenReturn(null);
+ CLI cli = new CLI(conf);
+ cli.cluster = mockCluster;
+
+ Job job = cli.getJob(JobID.forName("job_1234654654_001"));
+ Assert.assertTrue("job is not null", job == null);
+ }
+
+ @Test
+ public void testGetJobWithRetry() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, 1);
+
+ final Cluster mockCluster = mock(Cluster.class);
+ final Job mockJob = Job.getInstance(conf);
+ when(mockCluster.getJob(any(JobID.class))).thenReturn(
+ null).thenReturn(mockJob);
+ CLI cli = new CLI(conf);
+ cli.cluster = mockCluster;
+
+ Job job = cli.getJob(JobID.forName("job_1234654654_001"));
+ Assert.assertTrue("job is null", job != null);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] hadoop git commit: HADOOP-13073 RawLocalFileSystem does not
react on changing umask. Contributed by Andras Bokor
Posted by su...@apache.org.
HADOOP-13073 RawLocalFileSystem does not react on changing umask. Contributed by Andras Bokor
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d8b75ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d8b75ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d8b75ca
Branch: refs/heads/YARN-2915
Commit: 3d8b75caf12295652a1e33e6acc692c0a725db70
Parents: 4a97b0f
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jul 14 10:29:29 2016 +0100
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/RawLocalFileSystem.java | 7 +--
.../fs/TestLocalFileSystemPermission.java | 56 +++++++++++++++++++-
2 files changed, 56 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d8b75ca/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index cc41f4a..0fcddcf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -64,8 +64,6 @@ public class RawLocalFileSystem extends FileSystem {
// Temporary workaround for HADOOP-9652.
private static boolean useDeprecatedFileStatus = true;
- private FsPermission umask;
-
@VisibleForTesting
public static void useStatIfAvailable() {
useDeprecatedFileStatus = !Stat.isAvailable();
@@ -99,7 +97,6 @@ public class RawLocalFileSystem extends FileSystem {
public void initialize(URI uri, Configuration conf) throws IOException {
super.initialize(uri, conf);
setConf(conf);
- umask = FsPermission.getUMask(conf);
}
/*******************************************************
@@ -233,7 +230,7 @@ public class RawLocalFileSystem extends FileSystem {
if (permission == null) {
this.fos = new FileOutputStream(file, append);
} else {
- permission = permission.applyUMask(umask);
+ permission = permission.applyUMask(FsPermission.getUMask(getConf()));
if (Shell.WINDOWS && NativeIO.isAvailable()) {
this.fos = NativeIO.Windows.createFileOutputStreamWithMode(file,
append, permission.toShort());
@@ -510,7 +507,7 @@ public class RawLocalFileSystem extends FileSystem {
if (permission == null) {
permission = FsPermission.getDirDefault();
}
- permission = permission.applyUMask(umask);
+ permission = permission.applyUMask(FsPermission.getUMask(getConf()));
if (Shell.WINDOWS && NativeIO.isAvailable()) {
try {
NativeIO.Windows.createDirectoryWithMode(p2f, permission.toShort());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d8b75ca/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
index e37de19..7252ab9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
@@ -29,6 +29,10 @@ import java.util.*;
import junit.framework.*;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+
/**
* This class tests the local file system via the FileSystem abstraction.
*/
@@ -67,9 +71,9 @@ public class TestLocalFileSystemPermission extends TestCase {
System.out.println("Cannot run test for Windows");
return;
}
- Configuration conf = new Configuration();
+ LocalFileSystem localfs = FileSystem.getLocal(new Configuration());
+ Configuration conf = localfs.getConf();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "044");
- LocalFileSystem localfs = FileSystem.getLocal(conf);
Path dir = new Path(TEST_PATH_PREFIX + "dir");
localfs.mkdirs(dir);
try {
@@ -237,6 +241,54 @@ public class TestLocalFileSystemPermission extends TestCase {
finally {cleanup(localfs, f);}
}
+ /**
+ * Steps:
+ * 1. Create a directory with default permissions: 777 with umask 022
+ * 2. Check the directory has good permissions: 755
+ * 3. Set the umask to 062.
+ * 4. Create a new directory with default permissions.
+ * 5. For this directory we expect 715 as permission not 755
+ * @throws Exception we can throw away all the exception.
+ */
+ public void testSetUmaskInRealTime() throws Exception {
+ if (Path.WINDOWS) {
+ System.out.println("Cannot run test for Windows");
+ return;
+ }
+
+ LocalFileSystem localfs = FileSystem.getLocal(new Configuration());
+ Configuration conf = localfs.getConf();
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "022");
+ System.out.println(
+ conf.get(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY));
+ Path dir = new Path(TEST_PATH_PREFIX + "dir");
+ Path dir2 = new Path(TEST_PATH_PREFIX + "dir2");
+ try {
+ assertTrue(localfs.mkdirs(dir));
+ FsPermission initialPermission = getPermission(localfs, dir);
+ assertEquals(
+ "With umask 022 permission should be 755 since the default " +
+ "permission is 777", new FsPermission("755"), initialPermission);
+
+ // Modify umask and create a new directory
+ // and check if new umask is applied
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "062");
+ assertTrue(localfs.mkdirs(dir2));
+ FsPermission finalPermission = localfs.getFileStatus(dir2)
+ .getPermission();
+ assertThat("With umask 062 permission should not be 755 since the " +
+ "default permission is 777", new FsPermission("755"),
+ is(not(finalPermission)));
+ assertEquals(
+ "With umask 062 we expect 715 since the default permission is 777",
+ new FsPermission("715"), finalPermission);
+ } finally {
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "022");
+ cleanup(localfs, dir);
+ cleanup(localfs, dir2);
+ }
+ }
+
static List<String> getGroups() throws IOException {
List<String> a = new ArrayList<String>();
String s = Shell.execCommand(Shell.getGroupsCommand());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] hadoop git commit: YARN-5363. For AM containers,
or for containers of running-apps,
"yarn logs" incorrectly only (tries to) shows syslog file-type by default.
(Contributed by Vinod Kumar Vavilapalli)
Posted by su...@apache.org.
YARN-5363. For AM containers, or for containers of running-apps, "yarn logs" incorrectly only (tries to) shows syslog file-type by default. (Contributed by Vinod Kumar Vavilapalli)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bd489c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bd489c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bd489c4
Branch: refs/heads/YARN-2915
Commit: 7bd489c41a1c0e7a8618fa8398ec922a57808d4a
Parents: 1ba99e6
Author: Xuan <xg...@apache.org>
Authored: Thu Jul 14 10:20:08 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 63 ++++++++------------
.../hadoop/yarn/client/cli/TestLogsCLI.java | 60 ++++++++++++-------
2 files changed, 64 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bd489c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 1560557..2153e1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -28,7 +28,9 @@ import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
+
import javax.ws.rs.core.MediaType;
+
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -53,9 +55,8 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
-import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.codehaus.jettison.json.JSONArray;
@@ -78,7 +79,7 @@ public class LogsCLI extends Configured implements Tool {
private static final String NODE_ADDRESS_OPTION = "nodeAddress";
private static final String APP_OWNER_OPTION = "appOwner";
private static final String AM_CONTAINER_OPTION = "am";
- private static final String CONTAINER_LOG_FILES = "logFiles";
+ private static final String PER_CONTAINER_LOG_FILES_OPTION = "log_files";
private static final String LIST_NODES_OPTION = "list_nodes";
private static final String SHOW_APPLICATION_LOG_INFO
= "show_application_log_info";
@@ -146,8 +147,8 @@ public class LogsCLI extends Configured implements Tool {
return -1;
}
}
- if (commandLine.hasOption(CONTAINER_LOG_FILES)) {
- logFiles = commandLine.getOptionValues(CONTAINER_LOG_FILES);
+ if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_OPTION)) {
+ logFiles = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_OPTION);
}
if (commandLine.hasOption(SIZE_OPTION)) {
bytes = Long.parseLong(commandLine.getOptionValue(SIZE_OPTION));
@@ -359,12 +360,18 @@ public class LogsCLI extends Configured implements Tool {
}
private boolean fetchAllLogFiles(String[] logFiles) {
- if(logFiles != null) {
- List<String> logs = Arrays.asList(logFiles);
- if(logs.contains("ALL") || logs.contains(".*")) {
- return true;
- }
+
+ // If no value is specified for the PER_CONTAINER_LOG_FILES_OPTION option,
+ // we will assume all logs.
+ if (logFiles == null || logFiles.length == 0) {
+ return true;
}
+
+ List<String> logs = Arrays.asList(logFiles);
+ if (logs.contains("ALL") || logs.contains(".*")) {
+ return true;
+ }
+
return false;
}
@@ -424,7 +431,7 @@ public class LogsCLI extends Configured implements Tool {
containerIdStr);
try {
// fetch all the log files for the container
- // filter the log files based on the given --logFiles pattern
+ // filter the log files based on the given -log_files pattern
List<PerLogFileInfo> allLogFileInfos=
getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress);
List<String> fileNames = new ArrayList<String>();
@@ -700,8 +707,8 @@ public class LogsCLI extends Configured implements Tool {
new Option(APPLICATION_ID_OPTION, true, "ApplicationId (required)");
opts.addOption(appIdOpt);
opts.addOption(CONTAINER_ID_OPTION, true, "ContainerId. "
- + "By default, it will only print syslog if the application is running."
- + " Work with -logFiles to get other logs. If specified, the"
+ + "By default, it will print all available logs."
+ + " Work with -log_files to get only specific logs. If specified, the"
+ " applicationId can be omitted");
opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress in the format "
+ "nodename:port");
@@ -714,13 +721,13 @@ public class LogsCLI extends Configured implements Tool {
+ "the logs for the first AM Container as well as the second "
+ "AM Container. To get logs for all AM Containers, use -am ALL. "
+ "To get logs for the latest AM Container, use -am -1. "
- + "By default, it will only print out syslog. Work with -logFiles "
- + "to get other logs");
+ + "By default, it will print all available logs. Work with -log_files "
+ + "to get only specific logs.");
amOption.setValueSeparator(',');
amOption.setArgs(Option.UNLIMITED_VALUES);
amOption.setArgName("AM Containers");
opts.addOption(amOption);
- Option logFileOpt = new Option(CONTAINER_LOG_FILES, true,
+ Option logFileOpt = new Option(PER_CONTAINER_LOG_FILES_OPTION, true,
"Specify comma-separated value "
+ "to get specified container log files. Use \"ALL\" to fetch all the "
+ "log files for the container. It also supports Java Regex.");
@@ -764,7 +771,7 @@ public class LogsCLI extends Configured implements Tool {
printOpts.addOption(commandOpts.getOption(NODE_ADDRESS_OPTION));
printOpts.addOption(commandOpts.getOption(APP_OWNER_OPTION));
printOpts.addOption(commandOpts.getOption(AM_CONTAINER_OPTION));
- printOpts.addOption(commandOpts.getOption(CONTAINER_LOG_FILES));
+ printOpts.addOption(commandOpts.getOption(PER_CONTAINER_LOG_FILES_OPTION));
printOpts.addOption(commandOpts.getOption(LIST_NODES_OPTION));
printOpts.addOption(commandOpts.getOption(SHOW_APPLICATION_LOG_INFO));
printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO));
@@ -807,13 +814,7 @@ public class LogsCLI extends Configured implements Tool {
private int fetchAMContainerLogs(ContainerLogsRequest request,
List<String> amContainersList, LogCLIHelpers logCliHelper)
throws Exception {
- List<String> logFiles = request.getLogTypes();
- // if we do not specify the value for CONTAINER_LOG_FILES option,
- // we will only output syslog
- if (logFiles == null || logFiles.isEmpty()) {
- logFiles = Arrays.asList("syslog");
- }
- request.setLogTypes(logFiles);
+
// If the application is running, we will call the RM WebService
// to get the AppAttempts which includes the nodeHttpAddress
// and containerId for all the AM Containers.
@@ -856,7 +857,6 @@ public class LogsCLI extends Configured implements Tool {
String nodeAddress = request.getNodeId();
String appOwner = request.getAppOwner();
boolean isAppFinished = request.isAppFinished();
- List<String> logFiles = request.getLogTypes();
// if we provide the node address and the application is in the final
// state, we could directly get logs from HDFS.
if (nodeAddress != null && isAppFinished) {
@@ -899,12 +899,6 @@ public class LogsCLI extends Configured implements Tool {
// we will provide the NodeHttpAddress and get the container logs
// by calling NodeManager webservice.
if (!isAppFinished) {
- // if we do not specify the value for CONTAINER_LOG_FILES option,
- // we will only output syslog
- if (logFiles == null || logFiles.isEmpty()) {
- logFiles = Arrays.asList("syslog");
- }
- request.setLogTypes(logFiles);
resultCode = printContainerLogsFromRunningApplication(getConf(), request,
logCliHelper);
} else {
@@ -1026,13 +1020,6 @@ public class LogsCLI extends Configured implements Tool {
newOptions.setNodeHttpAddress(httpAddress
.replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
}
- // if we do not specify the value for CONTAINER_LOG_FILES option,
- // we will only output syslog
- List<String> logFiles = newOptions.getLogTypes();
- if (logFiles == null || logFiles.isEmpty()) {
- logFiles = Arrays.asList("syslog");
- newOptions.setLogTypes(logFiles);
- }
newOptionsList.add(newOptions);
}
return newOptionsList;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bd489c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 3fe53d9..bda489f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -46,6 +46,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -74,6 +75,7 @@ import org.codehaus.jettison.json.JSONObject;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentCaptor;
public class TestLogsCLI {
@@ -195,20 +197,20 @@ public class TestLogsCLI {
pw.println(" Container. To get logs for all AM");
pw.println(" Containers, use -am ALL. To get logs for");
pw.println(" the latest AM Container, use -am -1. By");
- pw.println(" default, it will only print out syslog.");
- pw.println(" Work with -logFiles to get other logs");
+ pw.println(" default, it will print all available");
+ pw.println(" logs. Work with -log_files to get only");
+ pw.println(" specific logs.");
pw.println(" -appOwner <Application Owner> AppOwner (assumed to be current user if");
pw.println(" not specified)");
- pw.println(" -containerId <Container ID> ContainerId. By default, it will only");
- pw.println(" print syslog if the application is");
- pw.println(" running. Work with -logFiles to get other");
- pw.println(" logs. If specified, the applicationId can");
- pw.println(" be omitted");
+ pw.println(" -containerId <Container ID> ContainerId. By default, it will print");
+ pw.println(" all available logs. Work with -log_files");
+ pw.println(" to get only specific logs. If specified,");
+ pw.println(" the applicationId can be omitted");
pw.println(" -help Displays help for all commands.");
pw.println(" -list_nodes Show the list of nodes that successfully");
pw.println(" aggregated logs. This option can only be");
pw.println(" used with finished applications.");
- pw.println(" -logFiles <Log File Name> Specify comma-separated value to get");
+ pw.println(" -log_files <Log File Name> Specify comma-separated value to get");
pw.println(" specified container log files. Use \"ALL\"");
pw.println(" to fetch all the log files for the");
pw.println(" container. It also supports Java Regex.");
@@ -240,7 +242,7 @@ public class TestLogsCLI {
}
@Test (timeout = 15000)
- public void testFetchApplictionLogs() throws Exception {
+ public void testFetchFinishedApplictionLogs() throws Exception {
String remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
@@ -337,7 +339,7 @@ public class TestLogsCLI {
sysOutStream.reset();
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
- "-logFiles", ".*"});
+ "-log_files", ".*"});
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000001 in syslog!"));
@@ -351,7 +353,7 @@ public class TestLogsCLI {
sysOutStream.reset();
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
- "-logFiles", "std*"});
+ "-log_files", "std*"});
assertTrue(exitCode == 0);
assertFalse(sysOutStream.toString().contains(
"Hello container_0_0001_01_000001 in syslog!"));
@@ -364,7 +366,7 @@ public class TestLogsCLI {
sysOutStream.reset();
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
- "-logFiles", "123"});
+ "-log_files", "123"});
assertTrue(exitCode == -1);
assertTrue(sysErrStream.toString().contains(
"Can not find any log file matching the pattern: [123] "
@@ -374,7 +376,7 @@ public class TestLogsCLI {
// specify the bytes which is larger than the actual file size,
// we would get the full logs
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
- "-logFiles", ".*", "-size", "10000" });
+ "-log_files", ".*", "-size", "10000" });
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toByteArray().length == fullSize);
sysOutStream.reset();
@@ -396,7 +398,7 @@ public class TestLogsCLI {
sysOutStream.reset();
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
- "-containerId", containerId3.toString(), "-logFiles", "123" });
+ "-containerId", containerId3.toString(), "-log_files", "123" });
assertTrue(exitCode == -1);
assertTrue(sysErrStream.toString().contains(
"Can not find any log file matching the pattern: [123] "
@@ -405,7 +407,7 @@ public class TestLogsCLI {
sysErrStream.reset();
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
- "-containerId", containerId3.toString(), "-logFiles", "stdout" });
+ "-containerId", containerId3.toString(), "-log_files", "stdout" });
assertTrue(exitCode == 0);
int fullContextSize = sysOutStream.toByteArray().length;
String fullContext = sysOutStream.toString();
@@ -419,7 +421,7 @@ public class TestLogsCLI {
// specify a position number, it would get the first n bytes from
// container log
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
- "-containerId", containerId3.toString(), "-logFiles", "stdout",
+ "-containerId", containerId3.toString(), "-log_files", "stdout",
"-size", "5"});
assertTrue(exitCode == 0);
Assert.assertEquals(new String(logMessage.getBytes(), 0, 5),
@@ -430,7 +432,7 @@ public class TestLogsCLI {
// specify a negative number, it would get the last n bytes from
// container log
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
- "-containerId", containerId3.toString(), "-logFiles", "stdout",
+ "-containerId", containerId3.toString(), "-log_files", "stdout",
"-size", "-5"});
assertTrue(exitCode == 0);
Assert.assertEquals(new String(logMessage.getBytes(),
@@ -441,7 +443,7 @@ public class TestLogsCLI {
long negative = (fullContextSize + 1000) * (-1);
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
- "-containerId", containerId3.toString(), "-logFiles", "stdout",
+ "-containerId", containerId3.toString(), "-log_files", "stdout",
"-size", Long.toString(negative)});
assertTrue(exitCode == 0);
Assert.assertEquals(fullContext, sysOutStream.toString());
@@ -473,12 +475,12 @@ public class TestLogsCLI {
"Hello container_0_0001_01_000003 in stdout!"));
sysOutStream.reset();
- // set -logFiles option as stdout
+ // set -log_files option as stdout
// should only print log with the name as stdout
exitCode =
cli.run(new String[] { "-applicationId", appId.toString(),
"-nodeAddress", nodeId.toString(), "-containerId",
- containerId3.toString() , "-logFiles", "stdout"});
+ containerId3.toString() , "-log_files", "stdout"});
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000003 in stdout!"));
@@ -527,11 +529,14 @@ public class TestLogsCLI {
@Test (timeout = 5000)
public void testFetchRunningApplicationLogs() throws Exception {
+
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+
NodeId nodeId = NodeId.newInstance("localhost", 1234);
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId
.newInstance(appId, 1);
+
// Create a mock ApplicationAttempt Report
ApplicationAttemptReport mockAttemptReport = mock(
ApplicationAttemptReport.class);
@@ -568,11 +573,24 @@ public class TestLogsCLI {
cli.setConf(new YarnConfiguration());
int exitCode = cli.run(new String[] {"-applicationId", appId.toString()});
assertTrue(exitCode == 0);
+
+ ArgumentCaptor<ContainerLogsRequest> logsRequestCaptor =
+ ArgumentCaptor.forClass(ContainerLogsRequest.class);
+
// we have two container reports, so make sure we have called
// printContainerLogsFromRunningApplication twice
verify(cli, times(2)).printContainerLogsFromRunningApplication(
- any(Configuration.class), any(ContainerLogsRequest.class),
+ any(Configuration.class), logsRequestCaptor.capture(),
any(LogCLIHelpers.class));
+
+ // Verify that the log-type is *
+ List<ContainerLogsRequest> capturedRequests =
+ logsRequestCaptor.getAllValues();
+ Assert.assertEquals(2, capturedRequests.size());
+ List<String> logTypes0 = capturedRequests.get(0).getLogTypes();
+ List<String> logTypes1 = capturedRequests.get(1).getLogTypes();
+ Assert.assertEquals(".*", logTypes0.get(0));
+ Assert.assertEquals(".*", logTypes1.get(0));
}
@Test (timeout = 15000)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] hadoop git commit: HADOOP-13290. Appropriate use of generics
in FairCallQueue. Contributed by Jonathan Hung.
Posted by su...@apache.org.
HADOOP-13290. Appropriate use of generics in FairCallQueue. Contributed by Jonathan Hung.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60ac09c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60ac09c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60ac09c7
Branch: refs/heads/YARN-2915
Commit: 60ac09c724176503ee0fdb69f191f21f56f7d02b
Parents: 9b697ba
Author: Zhe Zhang <zh...@apache.org>
Authored: Wed Jul 13 16:37:40 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:30 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/ipc/FairCallQueue.java | 15 +++++++-------
.../apache/hadoop/ipc/TestFairCallQueue.java | 21 +++++++++++++++++++-
2 files changed, 28 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60ac09c7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index 38b196d..c2d3cd8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -300,7 +300,7 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
@Override
public int size() {
int size = 0;
- for (BlockingQueue q : this.queues) {
+ for (BlockingQueue<E> q : this.queues) {
size += q.size();
}
return size;
@@ -346,7 +346,7 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
@Override
public int remainingCapacity() {
int sum = 0;
- for (BlockingQueue q : this.queues) {
+ for (BlockingQueue<E> q : this.queues) {
sum += q.remainingCapacity();
}
return sum;
@@ -362,7 +362,7 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
new HashMap<String, MetricsProxy>();
// Weakref for delegate, so we don't retain it forever if it can be GC'd
- private WeakReference<FairCallQueue> delegate;
+ private WeakReference<FairCallQueue<? extends Schedulable>> delegate;
// Keep track of how many objects we registered
private int revisionNumber = 0;
@@ -381,14 +381,15 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
return mp;
}
- public void setDelegate(FairCallQueue obj) {
- this.delegate = new WeakReference<FairCallQueue>(obj);
+ public void setDelegate(FairCallQueue<? extends Schedulable> obj) {
+ this.delegate
+ = new WeakReference<FairCallQueue<? extends Schedulable>>(obj);
this.revisionNumber++;
}
@Override
public int[] getQueueSizes() {
- FairCallQueue obj = this.delegate.get();
+ FairCallQueue<? extends Schedulable> obj = this.delegate.get();
if (obj == null) {
return new int[]{};
}
@@ -398,7 +399,7 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
@Override
public long[] getOverflowedCalls() {
- FairCallQueue obj = this.delegate.get();
+ FairCallQueue<? extends Schedulable> obj = this.delegate.get();
if (obj == null) {
return new long[]{};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60ac09c7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
index 8c96c2e..d5e5f90 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
@@ -23,6 +23,9 @@ import static org.mockito.Mockito.when;
import junit.framework.TestCase;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import java.lang.management.ManagementFactory;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.BlockingQueue;
@@ -392,4 +395,20 @@ public class TestFairCallQueue extends TestCase {
assertEquals(call, fcq.take());
assertEquals(0, fcq.size());
}
-}
\ No newline at end of file
+
+ public void testFairCallQueueMXBean() throws Exception {
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ ObjectName mxbeanName = new ObjectName(
+ "Hadoop:service=ns,name=FairCallQueue");
+
+ Schedulable call = mockCall("c");
+ fcq.put(call);
+ int[] queueSizes = (int[]) mbs.getAttribute(mxbeanName, "QueueSizes");
+ assertEquals(1, queueSizes[0]);
+ assertEquals(0, queueSizes[1]);
+ fcq.take();
+ queueSizes = (int[]) mbs.getAttribute(mxbeanName, "QueueSizes");
+ assertEquals(0, queueSizes[0]);
+ assertEquals(0, queueSizes[1]);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] hadoop git commit: YARN-5159. Wrong Javadoc tag in
MiniYarnCluster. Contributed by Andras Bokor.
Posted by su...@apache.org.
YARN-5159. Wrong Javadoc tag in MiniYarnCluster. Contributed by Andras Bokor.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35a57c97
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35a57c97
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35a57c97
Branch: refs/heads/YARN-2915
Commit: 35a57c972a9ad0b430f3c9a46cf586f9883d72fd
Parents: 678d0d4
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Jul 14 10:43:43 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:31 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/server/MiniYARNCluster.java | 21 ++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35a57c97/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index af9a43e..67b652b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -97,16 +97,20 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
/**
+ * <p>
* Embedded Yarn minicluster for testcases that need to interact with a cluster.
- * <p/>
+ * </p>
+ * <p>
* In a real cluster, resource request matching is done using the hostname, and
* by default Yarn minicluster works in the exact same way as a real cluster.
- * <p/>
- * If a testcase needs to use multiple nodes and exercise resource request
+ * </p>
+ * <p>
+ * If a testcase needs to use multiple nodes and exercise resource request
* matching to a specific node, then the property
- * {@YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME} should be set
- * <code>true</code> in the configuration used to initialize the minicluster.
- * <p/>
+ * {@value org.apache.hadoop.yarn.conf.YarnConfiguration#RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME}
+ * should be set <code>true</code> in the configuration used to initialize
+ * the minicluster.
+ * </p>
* With this property set to <code>true</code>, the matching will be done using
* the <code>hostname:port</code> of the namenodes. In such case, the AM must
* do resource request using <code>hostname:port</code> as the location.
@@ -736,8 +740,9 @@ public class MiniYARNCluster extends CompositeService {
* @param timeout Time to wait (sleeps in 10 ms intervals) in milliseconds.
* @return true if all NodeManagers connect to the (Active)
* ResourceManager, false otherwise.
- * @throws YarnException
- * @throws InterruptedException
+ * @throws YarnException if there is no active RM
+ * @throws InterruptedException if any thread has interrupted
+ * the current thread
*/
public boolean waitForNodeManagersToConnect(long timeout)
throws YarnException, InterruptedException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] hadoop git commit: HDFS-10639. Fix typos in
HDFSDiskbalancer.md. Contributed by Yiqun Lin.
Posted by su...@apache.org.
HDFS-10639. Fix typos in HDFSDiskbalancer.md. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ff64617
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ff64617
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ff64617
Branch: refs/heads/YARN-2915
Commit: 5ff646177a3a20af285d7e3d751260bf8a919646
Parents: e677d72
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Jul 15 21:41:57 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ff64617/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
index c5f73cd..aea440a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
@@ -28,7 +28,7 @@ Diskbalancer is a command line tool that distributes data evenly on all disks of
This tool is different from [Balancer](./HdfsUserGuide.html#Balancer) which
takes care of cluster-wide data balancing. Data can have uneven spread between
disks on a node due to several reasons. This can happen due to large amount of
-writes and deletes or due to a disk replacement.This tool operates against a given datanode and moves blocks from one disk to another.
+writes and deletes or due to a disk replacement. This tool operates against a given datanode and moves blocks from one disk to another.
@@ -38,7 +38,7 @@ Architecture
Disk Balancer operates by creating a plan and goes on to execute that plan on the datanode.
A plan is a set of statements that describe how much data should move between two disks.
A plan is composed of multiple move steps. A move step has source disk, destination
-disk and number of bytes to move.A plan can be executed against an operational data node. Disk balancer should not
+disk and number of bytes to move. A plan can be executed against an operational data node. Disk balancer should not
interfere with other processes since it throttles how much data is copied
every second. Please note that disk balancer is not enabled by default on a cluster.
To enable diskbalancer `dfs.disk.balancer.enabled` must be set to `true` in hdfs-site.xml.
@@ -63,8 +63,8 @@ The following sections discusses what commands are supported by disk balancer
| COMMAND\_OPTION | Description |
|:---- |:---- |
| `-out`| Allows user to control the output location of the plan file.|
-| `-bandwidth`| Since datanode is operational and might be running other jobs, diskbalancer limits the amount of data moved per second. This parameter allows user to set the maximum bandwidth to be used. This is not required to be set since diskBalancer will use the deafult bandwidth if this is not specified.|
-| `-thresholdPercentage`| Since we operate against a snap-shot of datanode, themove operations have a tolerance percentage to declare success. If user specifies 10% and move operation is say 20GB in size, if we can move 18GB that operation is considered successful. This is to accomodate the changes in datanode in real time. This parameter is not needed and a default is used if not specified.|
+| `-bandwidth`| Since datanode is operational and might be running other jobs, diskbalancer limits the amount of data moved per second. This parameter allows user to set the maximum bandwidth to be used. This is not required to be set since diskBalancer will use the default bandwidth if this is not specified.|
+| `-thresholdPercentage`| Since we operate against a snap-shot of datanode, the move operations have a tolerance percentage to declare success. If user specifies 10% and move operation is say 20GB in size, if we can move 18GB that operation is considered successful. This is to accommodate the changes in datanode in real time. This parameter is not needed and a default is used if not specified.|
| `-maxerror` | Max error allows users to specify how many block copy operations must fail before we abort a move step. Once again, this is not a needed parameter and a system-default is used if not specified.|
| `-v`| Verbose mode, specifying this parameter forces the plan command to print out a summary of the plan on stdout.|
@@ -117,4 +117,4 @@ There is a set of diskbalancer settings that can be controlled via hdfs-site.xml
|`dfs.disk.balancer.enabled`| This parameter controls if diskbalancer is enabled for a cluster. if this is not enabled, any execute command will be rejected by the datanode.The default value is false.|
|`dfs.disk.balancer.max.disk.throughputInMBperSec` | This controls the maximum disk bandwidth consumed by diskbalancer while copying data. If a value like 10MB is specified then diskbalancer on the average will only copy 10MB/S. The default value is 10MB/S.|
|`dfs.disk.balancer.max.disk.errors`| sets the value of maximum number of errors we can ignore for a specific move between two disks before it is abandoned. For example, if a plan has 3 pair of disks to copy between , and the first disk set encounters more than 5 errors, then we abandon the first copy and start the second copy in the plan. The default value of max errors is set to 5.|
-|`dfs.disk.balancer.block.tolerance.percent`| The tolerance percent sepcifies when we have reached a good enough value for any copy step. For example, if you specify 10% then getting close to 10% of the target value is good enough.|
+|`dfs.disk.balancer.block.tolerance.percent`| The tolerance percent specifies when we have reached a good enough value for any copy step. For example, if you specify 10% then getting close to 10% of the target value is good enough.|
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org