You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2012/03/23 22:03:30 UTC
svn commit: r1304597 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/net/
src/main/resources/ src/test/java/org/apache/hadoop/net/
Author: tomwhite
Date: Fri Mar 23 21:03:30 2012
New Revision: 1304597
URL: http://svn.apache.org/viewvc?rev=1304597&view=rev
Log:
HADOOP-7030. Add TableMapping topology implementation to read host to rack mapping from a file. Contributed by Patrick Angeles and tomwhite.
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java (with props)
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java (with props)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1304597&r1=1304596&r2=1304597&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Mar 23 21:03:30 2012
@@ -135,6 +135,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8121. Active Directory Group Mapping Service. (Jonathan Natkins via
atm)
+ HADOOP-7030. Add TableMapping topology implementation to read host to rack
+ mapping from a file. (Patrick Angeles and tomwhite via tomwhite)
+
IMPROVEMENTS
HADOOP-7524. Change RPC to allow multiple protocols including multuple
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1304597&r1=1304596&r2=1304597&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java Fri Mar 23 21:03:30 2012
@@ -63,6 +63,10 @@ public class CommonConfigurationKeysPubl
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY =
"net.topology.node.switch.mapping.impl";
+
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY =
+ "net.topology.table.file.name";
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String FS_TRASH_CHECKPOINT_INTERVAL_KEY =
Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java?rev=1304597&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java (added)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java Fri Mar 23 21:03:30 2012
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.net;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+
+/**
+ * <p>
+ * Simple {@link DNSToSwitchMapping} implementation that reads a 2 column text
+ * file. The columns are separated by whitespace. The first column is a DNS or
+ * IP address and the second column specifies the rack where the address maps.
+ * </p>
+ * <p>
+ * This class uses the configuration parameter {@code
+ * net.topology.table.file.name} to locate the mapping file.
+ * </p>
+ * <p>
+ * Calls to {@link #resolve(List)} will look up the address as defined in the
+ * mapping file. If no entry corresponding to the address is found, the value
+ * {@code /default-rack} is returned.
+ * </p>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class TableMapping extends CachedDNSToSwitchMapping {
+
+ private static final Log LOG = LogFactory.getLog(TableMapping.class);
+
+ public TableMapping() {
+ super(new RawTableMapping());
+ }
+
+ private RawTableMapping getRawMapping() {
+ return (RawTableMapping) rawMapping;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return getRawMapping().getConf();
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ getRawMapping().setConf(conf);
+ }
+
+ private static final class RawTableMapping extends Configured
+ implements DNSToSwitchMapping {
+
+ private final Map<String, String> map = new HashMap<String, String>();
+ private boolean initialized = false;
+
+ private synchronized void load() {
+ map.clear();
+
+ String filename = getConf().get(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, null);
+ if (StringUtils.isBlank(filename)) {
+ LOG.warn(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY + " not configured. "
+ + NetworkTopology.DEFAULT_RACK + " will be returned.");
+ return;
+ }
+
+ BufferedReader reader = null;
+ try {
+ reader = new BufferedReader(new FileReader(filename));
+ String line = reader.readLine();
+ while (line != null) {
+ line = line.trim();
+ if (line.length() != 0 && line.charAt(0) != '#') {
+ String[] columns = line.split("\\s+");
+ if (columns.length == 2) {
+ map.put(columns[0], columns[1]);
+ } else {
+ LOG.warn("Line does not have two columns. Ignoring. " + line);
+ }
+ }
+ line = reader.readLine();
+ }
+ } catch (Exception e) {
+ LOG.warn(filename + " cannot be read. " + NetworkTopology.DEFAULT_RACK
+ + " will be returned.", e);
+ map.clear();
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ LOG.warn(filename + " cannot be read. "
+ + NetworkTopology.DEFAULT_RACK + " will be returned.", e);
+ map.clear();
+ }
+ }
+ }
+ }
+
+ public synchronized List<String> resolve(List<String> names) {
+ if (!initialized) {
+ initialized = true;
+ load();
+ }
+
+ List<String> results = new ArrayList<String>(names.size());
+ for (String name : names) {
+ String result = map.get(name);
+ if (result != null) {
+ results.add(result);
+ } else {
+ results.add(NetworkTopology.DEFAULT_RACK);
+ }
+ }
+ return results;
+ }
+
+ }
+}
Propchange: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1304597&r1=1304596&r2=1304597&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Fri Mar 23 21:03:30 2012
@@ -688,6 +688,19 @@
</description>
</property>
+<property>
+ <name>net.topology.table.file.name</name>
+ <value></value>
+ <description> The file name for a topology file, which is used when the
+ net.topology.script.file.name property is set to
+ org.apache.hadoop.net.TableMapping. The file format is a two column text
+ file, with columns separated by whitespace. The first column is a DNS or
+ IP address and the second column specifies the rack where the address maps.
+ If no entry corresponding to a host in the cluster is found, then
+ /default-rack is assumed.
+ </description>
+</property>
+
<!-- Local file system -->
<property>
<name>file.stream-buffer-size</name>
Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java?rev=1304597&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java (added)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java Fri Mar 23 21:03:30 2012
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.net;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestTableMapping {
+
+ private File mappingFile;
+
+ @Before
+ public void setUp() throws IOException {
+ mappingFile = File.createTempFile(getClass().getSimpleName(), ".txt");
+ Files.write("a.b.c /rack1\n" +
+ "1.2.3\t/rack2\n", mappingFile, Charsets.UTF_8);
+ mappingFile.deleteOnExit();
+ }
+
+ @Test
+ public void testResolve() throws IOException {
+ TableMapping mapping = new TableMapping();
+
+ Configuration conf = new Configuration();
+ conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, mappingFile.getCanonicalPath());
+ mapping.setConf(conf);
+
+ List<String> names = new ArrayList<String>();
+ names.add("a.b.c");
+ names.add("1.2.3");
+
+ List<String> result = mapping.resolve(names);
+ assertEquals(names.size(), result.size());
+ assertEquals("/rack1", result.get(0));
+ assertEquals("/rack2", result.get(1));
+ }
+
+ @Test
+ public void testTableCaching() throws IOException {
+ TableMapping mapping = new TableMapping();
+
+ Configuration conf = new Configuration();
+ conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, mappingFile.getCanonicalPath());
+ mapping.setConf(conf);
+
+ List<String> names = new ArrayList<String>();
+ names.add("a.b.c");
+ names.add("1.2.3");
+
+ List<String> result1 = mapping.resolve(names);
+ assertEquals(names.size(), result1.size());
+ assertEquals("/rack1", result1.get(0));
+ assertEquals("/rack2", result1.get(1));
+
+ // unset the file, see if it gets read again
+ conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, "some bad value for a file");
+
+ List<String> result2 = mapping.resolve(names);
+ assertEquals(result1, result2);
+ }
+
+ @Test
+ public void testNoFile() {
+ TableMapping mapping = new TableMapping();
+
+ Configuration conf = new Configuration();
+ mapping.setConf(conf);
+
+ List<String> names = new ArrayList<String>();
+ names.add("a.b.c");
+ names.add("1.2.3");
+
+ List<String> result = mapping.resolve(names);
+ assertEquals(names.size(), result.size());
+ assertEquals(NetworkTopology.DEFAULT_RACK, result.get(0));
+ assertEquals(NetworkTopology.DEFAULT_RACK, result.get(1));
+ }
+
+ @Test
+ public void testFileDoesNotExist() {
+ TableMapping mapping = new TableMapping();
+
+ Configuration conf = new Configuration();
+ conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, "/this/file/does/not/exist");
+ mapping.setConf(conf);
+
+ List<String> names = new ArrayList<String>();
+ names.add("a.b.c");
+ names.add("1.2.3");
+
+ List<String> result = mapping.resolve(names);
+ assertEquals(names.size(), result.size());
+ assertEquals(result.get(0), NetworkTopology.DEFAULT_RACK);
+ assertEquals(result.get(1), NetworkTopology.DEFAULT_RACK);
+ }
+
+ @Test
+ public void testBadFile() throws IOException {
+ Files.write("bad contents", mappingFile, Charsets.UTF_8);
+
+ TableMapping mapping = new TableMapping();
+
+ Configuration conf = new Configuration();
+ conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, mappingFile.getCanonicalPath());
+ mapping.setConf(conf);
+
+ List<String> names = new ArrayList<String>();
+ names.add("a.b.c");
+ names.add("1.2.3");
+
+ List<String> result = mapping.resolve(names);
+ assertEquals(names.size(), result.size());
+ assertEquals(result.get(0), NetworkTopology.DEFAULT_RACK);
+ assertEquals(result.get(1), NetworkTopology.DEFAULT_RACK);
+ }
+
+}
Propchange: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java
------------------------------------------------------------------------------
svn:eol-style = native