You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vv...@apache.org on 2016/07/25 08:25:22 UTC
[34/50] [abbrv] hadoop git commit: HDFS-10653. Optimize conversion
from path string to components. Contributed by Daryn Sharp.
HDFS-10653. Optimize conversion from path string to components. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd3dcf46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd3dcf46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd3dcf46
Branch: refs/heads/YARN-4757
Commit: bd3dcf46e263b6e6aa3fca6a5d9936cc49e3280f
Parents: 557a245
Author: Jing Zhao <ji...@apache.org>
Authored: Thu Jul 21 11:14:39 2016 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Thu Jul 21 11:14:39 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 9 ++++++
.../hadoop/hdfs/server/namenode/INode.java | 20 +++++--------
.../server/namenode/TestSnapshotPathINodes.java | 30 +++++++-------------
3 files changed, 26 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd3dcf46/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index f1a6de7..0ba80d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -320,6 +320,15 @@ public class DFSUtil {
}
/**
+ * Convert a UTF8 string to an array of byte arrays.
+ */
+ public static byte[][] getPathComponents(String path) {
+ // avoid intermediate split to String[]
+ final byte[] bytes = string2Bytes(path);
+ return bytes2byteArray(bytes, bytes.length, (byte)Path.SEPARATOR_CHAR);
+ }
+
+ /**
* Splits the array of bytes into array of arrays of bytes
* on byte separator
* @param bytes the array of bytes to split
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd3dcf46/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index c8f36e1..eb910d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -727,18 +727,8 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
*/
@VisibleForTesting
public static byte[][] getPathComponents(String path) {
- return getPathComponents(getPathNames(path));
- }
-
- /** Convert strings to byte arrays for path components. */
- static byte[][] getPathComponents(String[] strings) {
- if (strings.length == 0) {
- return new byte[][]{null};
- }
- byte[][] bytes = new byte[strings.length][];
- for (int i = 0; i < strings.length; i++)
- bytes[i] = DFSUtil.string2Bytes(strings[i]);
- return bytes;
+ checkAbsolutePath(path);
+ return DFSUtil.getPathComponents(path);
}
/**
@@ -747,11 +737,15 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* @return array of path components.
*/
public static String[] getPathNames(String path) {
+ checkAbsolutePath(path);
+ return StringUtils.split(path, Path.SEPARATOR_CHAR);
+ }
+
+ private static void checkAbsolutePath(final String path) {
if (path == null || !path.startsWith(Path.SEPARATOR)) {
throw new AssertionError("Absolute path required, but got '"
+ path + "'");
}
- return StringUtils.split(path, Path.SEPARATOR_CHAR);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd3dcf46/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
index e416e00..45c65ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
@@ -138,8 +138,7 @@ public class TestSnapshotPathINodes {
@Test (timeout=15000)
public void testNonSnapshotPathINodes() throws Exception {
// Get the inodes by resolving the path of a normal file
- String[] names = INode.getPathNames(file1.toString());
- byte[][] components = INode.getPathComponents(names);
+ byte[][] components = INode.getPathComponents(file1.toString());
INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
components, false);
// The number of inodes should be equal to components.length
@@ -175,8 +174,7 @@ public class TestSnapshotPathINodes {
// The path when accessing the snapshot file of file1 is
// /TestSnapshot/sub1/.snapshot/s1/file1
String snapshotPath = sub1.toString() + "/.snapshot/s1/file1";
- String[] names = INode.getPathNames(snapshotPath);
- byte[][] components = INode.getPathComponents(names);
+ byte[][] components = INode.getPathComponents(snapshotPath);
INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
components, false);
// Length of inodes should be (components.length - 1), since we will ignore
@@ -199,8 +197,7 @@ public class TestSnapshotPathINodes {
// Resolve the path "/TestSnapshot/sub1/.snapshot"
String dotSnapshotPath = sub1.toString() + "/.snapshot";
- names = INode.getPathNames(dotSnapshotPath);
- components = INode.getPathComponents(names);
+ components = INode.getPathComponents(dotSnapshotPath);
nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false);
// The number of INodes returned should still be components.length
// since we put a null in the inode array for ".snapshot"
@@ -246,8 +243,7 @@ public class TestSnapshotPathINodes {
// Resolve the path for the snapshot file
// /TestSnapshot/sub1/.snapshot/s2/file1
String snapshotPath = sub1.toString() + "/.snapshot/s2/file1";
- String[] names = INode.getPathNames(snapshotPath);
- byte[][] components = INode.getPathComponents(names);
+ byte[][] components = INode.getPathComponents(snapshotPath);
INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
components, false);
// Length of inodes should be (components.length - 1), since we will ignore
@@ -264,8 +260,7 @@ public class TestSnapshotPathINodes {
}
// Check the INodes for path /TestSnapshot/sub1/file1
- String[] names = INode.getPathNames(file1.toString());
- byte[][] components = INode.getPathComponents(names);
+ byte[][] components = INode.getPathComponents(file1.toString());
INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
components, false);
// The length of inodes should be equal to components.length
@@ -314,8 +309,7 @@ public class TestSnapshotPathINodes {
{
// Check the inodes for /TestSnapshot/sub1/.snapshot/s4/file3
String snapshotPath = sub1.toString() + "/.snapshot/s4/file3";
- String[] names = INode.getPathNames(snapshotPath);
- byte[][] components = INode.getPathComponents(names);
+ byte[][] components = INode.getPathComponents(snapshotPath);
INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
components, false);
// Length of inodes should be (components.length - 1), since we will ignore
@@ -334,8 +328,7 @@ public class TestSnapshotPathINodes {
}
// Check the inodes for /TestSnapshot/sub1/file3
- String[] names = INode.getPathNames(file3.toString());
- byte[][] components = INode.getPathComponents(names);
+ byte[][] components = INode.getPathComponents(file3.toString());
INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
components, false);
// The number of inodes should be equal to components.length
@@ -361,8 +354,7 @@ public class TestSnapshotPathINodes {
@Test (timeout=15000)
public void testSnapshotPathINodesAfterModification() throws Exception {
// First check the INode for /TestSnapshot/sub1/file1
- String[] names = INode.getPathNames(file1.toString());
- byte[][] components = INode.getPathComponents(names);
+ byte[][] components = INode.getPathComponents(file1.toString());
INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
components, false);
// The number of inodes should be equal to components.length
@@ -385,8 +377,7 @@ public class TestSnapshotPathINodes {
// Check the INodes for snapshot of file1
String snapshotPath = sub1.toString() + "/.snapshot/s3/file1";
- names = INode.getPathNames(snapshotPath);
- components = INode.getPathComponents(names);
+ components = INode.getPathComponents(snapshotPath);
INodesInPath ssNodesInPath = INodesInPath.resolve(fsdir.rootDir,
components, false);
// Length of ssInodes should be (components.length - 1), since we will
@@ -404,8 +395,7 @@ public class TestSnapshotPathINodes {
snapshotFileNode.getModificationTime(ssNodesInPath.getPathSnapshotId()));
// Check the INode for /TestSnapshot/sub1/file1 again
- names = INode.getPathNames(file1.toString());
- components = INode.getPathComponents(names);
+ components = INode.getPathComponents(file1.toString());
INodesInPath newNodesInPath = INodesInPath.resolve(fsdir.rootDir,
components, false);
assertSnapshot(newNodesInPath, false, s3, -1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org