You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2017/09/21 01:07:38 UTC
[35/50] [abbrv] hadoop git commit: HDFS-12473. Change hosts JSON file
format.
HDFS-12473. Change hosts JSON file format.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/230b85d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/230b85d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/230b85d5
Branch: refs/heads/YARN-5734
Commit: 230b85d5865b7e08fb7aaeab45295b5b966011ef
Parents: 7e58b24
Author: Ming Ma <mi...@twitter.com>
Authored: Wed Sep 20 09:03:59 2017 -0700
Committer: Ming Ma <mi...@twitter.com>
Committed: Wed Sep 20 09:03:59 2017 -0700
----------------------------------------------------------------------
.../hdfs/util/CombinedHostsFileReader.java | 67 ++++++++++++++------
.../hdfs/util/CombinedHostsFileWriter.java | 23 ++++---
.../CombinedHostFileManager.java | 3 +-
.../hdfs/util/TestCombinedHostsFileReader.java | 44 ++++++++-----
.../src/test/resources/dfs.hosts.json | 16 +++--
.../src/test/resources/legacy.dfs.hosts.json | 7 ++
6 files changed, 102 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/230b85d5/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileReader.java
index 8da5655..aa8e4c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileReader.java
@@ -19,58 +19,85 @@
package org.apache.hadoop.hdfs.util;
import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
+
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.Reader;
+import java.util.ArrayList;
import java.util.Iterator;
-import java.util.Set;
-import java.util.HashSet;
+import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.DatanodeAdminProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
- * Reader support for JSON based datanode configuration, an alternative
+ * Reader support for JSON-based datanode configuration, an alternative format
* to the exclude/include files configuration.
- * The JSON file format is the array of elements where each element
+ * The JSON file format defines the array of elements where each element
* in the array describes the properties of a datanode. The properties of
- * a datanode is defined in {@link DatanodeAdminProperties}. For example,
+ * a datanode is defined by {@link DatanodeAdminProperties}. For example,
*
- * {"hostName": "host1"}
- * {"hostName": "host2", "port": 50, "upgradeDomain": "ud0"}
- * {"hostName": "host3", "port": 0, "adminState": "DECOMMISSIONED"}
+ * [
+ * {"hostName": "host1"},
+ * {"hostName": "host2", "port": 50, "upgradeDomain": "ud0"},
+ * {"hostName": "host3", "port": 0, "adminState": "DECOMMISSIONED"}
+ * ]
*/
@InterfaceAudience.LimitedPrivate({"HDFS"})
@InterfaceStability.Unstable
public final class CombinedHostsFileReader {
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(DatanodeAdminProperties.class);
- private static final JsonFactory JSON_FACTORY = new JsonFactory();
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(CombinedHostsFileReader.class);
private CombinedHostsFileReader() {
}
/**
* Deserialize a set of DatanodeAdminProperties from a json file.
- * @param hostsFile the input json file to read from.
+ * @param hostsFile the input json file to read from
* @return the set of DatanodeAdminProperties
* @throws IOException
*/
- public static Set<DatanodeAdminProperties>
+ public static DatanodeAdminProperties[]
readFile(final String hostsFile) throws IOException {
- HashSet<DatanodeAdminProperties> allDNs = new HashSet<>();
+ DatanodeAdminProperties[] allDNs = new DatanodeAdminProperties[0];
+ ObjectMapper objectMapper = new ObjectMapper();
+ boolean tryOldFormat = false;
try (Reader input =
- new InputStreamReader(new FileInputStream(hostsFile), "UTF-8")) {
- Iterator<DatanodeAdminProperties> iterator =
- READER.readValues(JSON_FACTORY.createParser(input));
- while (iterator.hasNext()) {
- DatanodeAdminProperties properties = iterator.next();
- allDNs.add(properties);
+ new InputStreamReader(new FileInputStream(hostsFile), "UTF-8")) {
+ allDNs = objectMapper.readValue(input, DatanodeAdminProperties[].class);
+ } catch (JsonMappingException jme) {
+ // The old format doesn't have json top-level token to enclose the array.
+ // For backward compatibility, try parsing the old format.
+ tryOldFormat = true;
+ LOG.warn("{} has invalid JSON format." +
+ "Try the old format without top-level token defined.", hostsFile);
+ }
+
+ if (tryOldFormat) {
+ ObjectReader objectReader =
+ objectMapper.readerFor(DatanodeAdminProperties.class);
+ JsonFactory jsonFactory = new JsonFactory();
+ List<DatanodeAdminProperties> all = new ArrayList<>();
+ try (Reader input =
+ new InputStreamReader(new FileInputStream(hostsFile), "UTF-8")) {
+ Iterator<DatanodeAdminProperties> iterator =
+ objectReader.readValues(jsonFactory.createParser(input));
+ while (iterator.hasNext()) {
+ DatanodeAdminProperties properties = iterator.next();
+ all.add(properties);
+ }
}
+ allDNs = all.toArray(new DatanodeAdminProperties[all.size()]);
}
return allDNs;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/230b85d5/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileWriter.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileWriter.java
index d771191..2d9bbf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileWriter.java
@@ -32,20 +32,21 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.DatanodeAdminProperties;
/**
- * Writer support for JSON based datanode configuration, an alternative
+ * Writer support for JSON-based datanode configuration, an alternative format
* to the exclude/include files configuration.
- * The JSON file format is the array of elements where each element
+ * The JSON file format defines the array of elements where each element
* in the array describes the properties of a datanode. The properties of
- * a datanode is defined in {@link DatanodeAdminProperties}. For example,
+ * a datanode is defined by {@link DatanodeAdminProperties}. For example,
*
- * {"hostName": "host1"}
- * {"hostName": "host2", "port": 50, "upgradeDomain": "ud0"}
- * {"hostName": "host3", "port": 0, "adminState": "DECOMMISSIONED"}
+ * [
+ * {"hostName": "host1"},
+ * {"hostName": "host2", "port": 50, "upgradeDomain": "ud0"},
+ * {"hostName": "host3", "port": 0, "adminState": "DECOMMISSIONED"}
+ * ]
*/
@InterfaceAudience.LimitedPrivate({"HDFS"})
@InterfaceStability.Unstable
public final class CombinedHostsFileWriter {
- private static final ObjectMapper MAPPER = new ObjectMapper();
private CombinedHostsFileWriter() {
}
@@ -57,13 +58,11 @@ public final class CombinedHostsFileWriter {
*/
public static void writeFile(final String hostsFile,
final Set<DatanodeAdminProperties> allDNs) throws IOException {
- StringBuilder configs = new StringBuilder();
+ final ObjectMapper objectMapper = new ObjectMapper();
+
try (Writer output =
new OutputStreamWriter(new FileOutputStream(hostsFile), "UTF-8")) {
- for (DatanodeAdminProperties datanodeAdminProperties: allDNs) {
- configs.append(MAPPER.writeValueAsString(datanodeAdminProperties));
- }
- output.write(configs.toString());
+ objectMapper.writeValue(output, allDNs);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/230b85d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java
index 6f9c35e..d6a0972 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CombinedHostFileManager.java
@@ -39,7 +39,6 @@ import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
-import java.util.Set;
import com.google.common.base.Predicate;
@@ -212,7 +211,7 @@ public class CombinedHostFileManager extends HostConfigManager {
}
private void refresh(final String hostsFile) throws IOException {
HostProperties hostProps = new HostProperties();
- Set<DatanodeAdminProperties> all =
+ DatanodeAdminProperties[] all =
CombinedHostsFileReader.readFile(hostsFile);
for(DatanodeAdminProperties properties : all) {
InetSocketAddress addr = parseEntry(hostsFile,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/230b85d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestCombinedHostsFileReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestCombinedHostsFileReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestCombinedHostsFileReader.java
index b48784f..cf02180 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestCombinedHostsFileReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestCombinedHostsFileReader.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.hdfs.util;
import java.io.File;
import java.io.FileWriter;
-import java.util.Set;
-
import org.apache.hadoop.hdfs.protocol.DatanodeAdminProperties;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Before;
@@ -30,19 +28,21 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
-/*
- * Test for JSON based HostsFileReader
+/**
+ * Test for JSON based HostsFileReader.
*/
public class TestCombinedHostsFileReader {
// Using /test/build/data/tmp directory to store temporary files
- static final String HOSTS_TEST_DIR = GenericTestUtils.getTestDir()
+ static final String HOSTSTESTDIR = GenericTestUtils.getTestDir()
.getAbsolutePath();
- File NEW_FILE = new File(HOSTS_TEST_DIR, "dfs.hosts.new.json");
+ private final File newFile = new File(HOSTSTESTDIR, "dfs.hosts.new.json");
- static final String TEST_CACHE_DATA_DIR =
+ static final String TESTCACHEDATADIR =
System.getProperty("test.cache.data", "build/test/cache");
- File EXISTING_FILE = new File(TEST_CACHE_DATA_DIR, "dfs.hosts.json");
+ private final File jsonFile = new File(TESTCACHEDATADIR, "dfs.hosts.json");
+ private final File legacyFile =
+ new File(TESTCACHEDATADIR, "legacy.dfs.hosts.json");
@Before
public void setUp() throws Exception {
@@ -51,18 +51,28 @@ public class TestCombinedHostsFileReader {
@After
public void tearDown() throws Exception {
// Delete test file after running tests
- NEW_FILE.delete();
+ newFile.delete();
+
+ }
+ /*
+ * Load the legacy test json file
+ */
+ @Test
+ public void testLoadLegacyJsonFile() throws Exception {
+ DatanodeAdminProperties[] all =
+ CombinedHostsFileReader.readFile(legacyFile.getAbsolutePath());
+ assertEquals(7, all.length);
}
/*
- * Load the existing test json file
+ * Load the test json file
*/
@Test
public void testLoadExistingJsonFile() throws Exception {
- Set<DatanodeAdminProperties> all =
- CombinedHostsFileReader.readFile(EXISTING_FILE.getAbsolutePath());
- assertEquals(7, all.size());
+ DatanodeAdminProperties[] all =
+ CombinedHostsFileReader.readFile(jsonFile.getAbsolutePath());
+ assertEquals(7, all.length);
}
/*
@@ -70,11 +80,11 @@ public class TestCombinedHostsFileReader {
*/
@Test
public void testEmptyCombinedHostsFileReader() throws Exception {
- FileWriter hosts = new FileWriter(NEW_FILE);
+ FileWriter hosts = new FileWriter(newFile);
hosts.write("");
hosts.close();
- Set<DatanodeAdminProperties> all =
- CombinedHostsFileReader.readFile(NEW_FILE.getAbsolutePath());
- assertEquals(0, all.size());
+ DatanodeAdminProperties[] all =
+ CombinedHostsFileReader.readFile(newFile.getAbsolutePath());
+ assertEquals(0, all.length);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/230b85d5/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/dfs.hosts.json
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/dfs.hosts.json b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/dfs.hosts.json
index 9c852e0..615b17f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/dfs.hosts.json
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/dfs.hosts.json
@@ -1,7 +1,9 @@
-{"hostName": "host1"}
-{"hostName": "host2", "upgradeDomain": "ud0"}
-{"hostName": "host3", "adminState": "DECOMMISSIONED"}
-{"hostName": "host4", "upgradeDomain": "ud2", "adminState": "DECOMMISSIONED"}
-{"hostName": "host5", "port": 8090}
-{"hostName": "host6", "adminState": "IN_MAINTENANCE"}
-{"hostName": "host7", "adminState": "IN_MAINTENANCE", "maintenanceExpireTimeInMS": "112233"}
+[
+ {"hostName": "host1"},
+ {"hostName": "host2", "upgradeDomain": "ud0"},
+ {"hostName": "host3", "adminState": "DECOMMISSIONED"},
+ {"hostName": "host4", "upgradeDomain": "ud2", "adminState": "DECOMMISSIONED"},
+ {"hostName": "host5", "port": 8090},
+ {"hostName": "host6", "adminState": "IN_MAINTENANCE"},
+ {"hostName": "host7", "adminState": "IN_MAINTENANCE", "maintenanceExpireTimeInMS": "112233"}
+]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/230b85d5/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/legacy.dfs.hosts.json
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/legacy.dfs.hosts.json b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/legacy.dfs.hosts.json
new file mode 100644
index 0000000..9c852e0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/legacy.dfs.hosts.json
@@ -0,0 +1,7 @@
+{"hostName": "host1"}
+{"hostName": "host2", "upgradeDomain": "ud0"}
+{"hostName": "host3", "adminState": "DECOMMISSIONED"}
+{"hostName": "host4", "upgradeDomain": "ud2", "adminState": "DECOMMISSIONED"}
+{"hostName": "host5", "port": 8090}
+{"hostName": "host6", "adminState": "IN_MAINTENANCE"}
+{"hostName": "host7", "adminState": "IN_MAINTENANCE", "maintenanceExpireTimeInMS": "112233"}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org