You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by cd...@apache.org on 2009/12/23 04:50:31 UTC
svn commit: r893398 - in /hadoop/mapreduce/branches/branch-0.21: ./
src/java/ src/java/org/apache/hadoop/mapred/
src/test/mapred/org/apache/hadoop/mapred/
Author: cdouglas
Date: Wed Dec 23 03:50:30 2009
New Revision: 893398
URL: http://svn.apache.org/viewvc?rev=893398&view=rev
Log:
MAPREDUCE-1241. Use a default queue configuration in JobTracker when
mapred-queues.xml is unavailable. Contributed by Todd Lipcon
Added:
hadoop/mapreduce/branches/branch-0.21/src/java/mapred-queues-default.xml
Modified:
hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
hadoop/mapreduce/branches/branch-0.21/build.xml
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/QueueConfigurationParser.java
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/QueueManager.java
hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java
Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=893398&r1=893397&r2=893398&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Wed Dec 23 03:50:30 2009
@@ -894,3 +894,5 @@
when speculative attempts are running for a TIP.
(Rahul Kumar Singh via yhemanth)
+ MAPREDUCE-1241. Use a default queue configuration in JobTracker when
+ mapred-queues.xml is unavailable. (Todd Lipcon via cdouglas)
Modified: hadoop/mapreduce/branches/branch-0.21/build.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/build.xml?rev=893398&r1=893397&r2=893398&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/build.xml (original)
+++ hadoop/mapreduce/branches/branch-0.21/build.xml Wed Dec 23 03:50:30 2009
@@ -366,6 +366,7 @@
<copy todir="${build.classes}">
<fileset dir="${mapred.src.dir}" includes="**/*.properties"/>
<fileset dir="${mapred.src.dir}" includes="mapred-default.xml"/>
+ <fileset dir="${mapred.src.dir}" includes="mapred-queues-default.xml"/>
</copy>
</target>
Added: hadoop/mapreduce/branches/branch-0.21/src/java/mapred-queues-default.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/mapred-queues-default.xml?rev=893398&view=auto
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/mapred-queues-default.xml (added)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/mapred-queues-default.xml Wed Dec 23 03:50:30 2009
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- This is the default mapred-queues.xml file that is loaded in the case
+ that the user does not have such a file on their classpath. -->
+<queues aclsEnabled="false">
+ <queue>
+ <name>default</name>
+ <properties>
+ </properties>
+ <state>running</state>
+ <acl-submit-job>*</acl-submit-job>
+ <acl-administer-jobs>*</acl-administer-jobs>
+ </queue>
+</queues>
\ No newline at end of file
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/QueueConfigurationParser.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/QueueConfigurationParser.java?rev=893398&r1=893397&r2=893398&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/QueueConfigurationParser.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/QueueConfigurationParser.java Wed Dec 23 03:50:30 2009
@@ -19,6 +19,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.Queue.QueueOperation;
import org.apache.hadoop.mapreduce.QueueState;
import org.apache.hadoop.security.SecurityUtil.AccessControlList;
@@ -87,9 +88,30 @@
}
- QueueConfigurationParser(String file) {
+ QueueConfigurationParser(String confFile) {
+ File file = new File(confFile).getAbsoluteFile();
+ if (!file.exists()) {
+ throw new RuntimeException("Configuration file not found at " +
+ confFile);
+ }
+ InputStream in = null;
+ try {
+ in = new BufferedInputStream(new FileInputStream(file));
+ loadFrom(in);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ } finally {
+ IOUtils.closeStream(in);
+ }
+ }
+
+ QueueConfigurationParser(InputStream xmlInput) {
+ loadFrom(xmlInput);
+ }
+
+ private void loadFrom(InputStream xmlInput) {
try {
- this.root = loadResource(file);
+ this.root = loadResource(xmlInput);
} catch (ParserConfigurationException e) {
throw new RuntimeException(e);
} catch (SAXException e) {
@@ -120,13 +142,13 @@
* Method to load the resource file.
* generates the root.
*
- * @param confFile
+ * @param resourceInput InputStream that provides the XML to parse
* @return
* @throws ParserConfigurationException
* @throws SAXException
* @throws IOException
*/
- protected Queue loadResource(String confFile)
+ protected Queue loadResource(InputStream resourceInput)
throws ParserConfigurationException, SAXException, IOException {
DocumentBuilderFactory docBuilderFactory
= DocumentBuilderFactory.newInstance();
@@ -146,19 +168,8 @@
DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
Document doc = null;
Element queuesNode = null;
- File file = new File(confFile).getAbsoluteFile();
- if (file.exists()) {
- InputStream in = new BufferedInputStream(new FileInputStream(file));
- try {
- doc = builder.parse(in);
- } finally {
- in.close();
- }
- }
- if (doc == null) {
- throw new RuntimeException(file.getAbsolutePath() + " not found");
- }
+ doc = builder.parse(resourceInput);
queuesNode = doc.getDocumentElement();
return this.parseResource(queuesNode);
}
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/QueueManager.java?rev=893398&r1=893397&r2=893398&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/QueueManager.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/QueueManager.java Wed Dec 23 03:50:30 2009
@@ -22,6 +22,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.TaskScheduler.QueueRefresher;
import org.apache.hadoop.mapreduce.QueueState;
import org.apache.hadoop.security.SecurityUtil.AccessControlList;
@@ -31,6 +32,8 @@
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.JsonGenerator;
+import java.io.BufferedInputStream;
+import java.io.InputStream;
import java.io.IOException;
import java.io.Writer;
import java.util.ArrayList;
@@ -87,6 +90,7 @@
private Map<String, Queue> leafQueues = new HashMap<String,Queue>();
private Map<String, Queue> allQueues = new HashMap<String, Queue>();
static final String QUEUE_CONF_FILE_NAME = "mapred-queues.xml";
+ static final String QUEUE_CONF_DEFAULT_FILE_NAME = "mapred-queues-default.xml";
// Prefix in configuration for queue related keys
static final String QUEUE_CONF_PROPERTY_NAME_PREFIX
@@ -120,10 +124,24 @@
}
return new DeprecatedQueueConfigurationParser(conf);
} else {
- URL filePath =
+ URL xmlInUrl =
Thread.currentThread().getContextClassLoader()
.getResource(QUEUE_CONF_FILE_NAME);
- return new QueueConfigurationParser(filePath.getPath());
+ if (xmlInUrl == null) {
+ xmlInUrl = Thread.currentThread().getContextClassLoader()
+ .getResource(QUEUE_CONF_DEFAULT_FILE_NAME);
+ assert xmlInUrl != null; // this should be in our jar
+ }
+ InputStream stream = null;
+ try {
+ stream = xmlInUrl.openStream();
+ return new QueueConfigurationParser(new BufferedInputStream(stream));
+ } catch (IOException ioe) {
+ throw new RuntimeException("Couldn't open queue configuration at " +
+ xmlInUrl, ioe);
+ } finally {
+ IOUtils.closeStream(stream);
+ }
}
}
Modified: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java?rev=893398&r1=893397&r2=893398&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java Wed Dec 23 03:50:30 2009
@@ -250,6 +250,28 @@
}
@Test
+ public void testMissingConfigFile() throws Exception {
+ checkForConfigFile(); // deletes file
+
+ try {
+ new QueueManager(CONFIG);
+ fail("Should throw an exception for missing file when " +
+ "explicitly passed.");
+ } catch (RuntimeException re) {
+ }
+
+ // If we just want to pick up the queues from the class loader
+ // it should fall through to the default. The class loader is set to
+ // load CONFIG for the "mapred-queues.xml" resource, but it's missing
+ // so should fall through to mapred-queues-default.xml
+ QueueManager qm = new QueueManager();
+ List<JobQueueInfo> rootQueues =
+ qm.getRoot().getJobQueueInfo().getChildren();
+ assertEquals(1, rootQueues.size());
+ assertEquals("default", rootQueues.get(0).getQueueName());
+ }
+
+ @Test
public void testEmptyProperties() throws Exception {
checkForConfigFile();
Document doc = createDocument();