You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2016/02/17 05:46:49 UTC
[1/4] hadoop git commit: HADOOP-12810. FileSystem#listLocatedStatus
causes unnecessary RPC calls (Contributed by Ryan Blue)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 6874a142d -> 2bf5a005f
refs/heads/branch-2.7 05c782153 -> a9c1bb424
refs/heads/branch-2.8 c7be3deff -> e88c01fa5
refs/heads/trunk 77ba5add0 -> 96ea30943
HADOOP-12810. FileSystem#listLocatedStatus causes unnecessary RPC calls (Contributed by Ryan Blue)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96ea3094
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96ea3094
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96ea3094
Branch: refs/heads/trunk
Commit: 96ea3094315bb1e1a5e268e3817c7fdedc3e9462
Parents: 77ba5ad
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Feb 17 10:13:41 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Feb 17 10:13:41 2016 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/fs/FileSystem.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96ea3094/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1a2b32f..0aa39d9 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1720,6 +1720,9 @@ Release 2.7.3 - UNRELEASED
OPTIMIZATIONS
+ HADOOP-12810. FileSystem#listLocatedStatus causes unnecessary RPC calls
+ (Ryan Blue via vinayakumarb)
+
BUG FIXES
HADOOP-12296. when setnetgrent returns 0 in linux, exception should be
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96ea3094/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 920c5a6..f4a2e7d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1749,8 +1749,10 @@ public abstract class FileSystem extends Configured implements Closeable {
throw new NoSuchElementException("No more entries in " + f);
}
FileStatus result = stats[i++];
+ // for files, use getBlockLocations(FileStatus, int, int) to avoid
+ // calling getFileStatus(Path) to load the FileStatus again
BlockLocation[] locs = result.isFile() ?
- getFileBlockLocations(result.getPath(), 0, result.getLen()) :
+ getFileBlockLocations(result, 0, result.getLen()) :
null;
return new LocatedFileStatus(result, locs);
}
[3/4] hadoop git commit: HADOOP-12810. FileSystem#listLocatedStatus
causes unnecessary RPC calls (Contributed by Ryan Blue)
Posted by vi...@apache.org.
HADOOP-12810. FileSystem#listLocatedStatus causes unnecessary RPC calls (Contributed by Ryan Blue)
(cherry picked from commit 96ea3094315bb1e1a5e268e3817c7fdedc3e9462)
(cherry picked from commit 2bf5a005fab287689a148c1928f3b04c0a8dcedf)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e88c01fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e88c01fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e88c01fa
Branch: refs/heads/branch-2.8
Commit: e88c01fa5808afc4176c35cf5cd49eecdbd13d8c
Parents: c7be3de
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Feb 17 10:13:41 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Feb 17 10:15:43 2016 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/fs/FileSystem.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e88c01fa/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index bd6b08a..5f49c95 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1019,6 +1019,9 @@ Release 2.7.3 - UNRELEASED
OPTIMIZATIONS
+ HADOOP-12810. FileSystem#listLocatedStatus causes unnecessary RPC calls
+ (Ryan Blue via vinayakumarb)
+
BUG FIXES
HADOOP-12296. when setnetgrent returns 0 in linux, exception should be
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e88c01fa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index e0320c0..4eb3ed5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1732,8 +1732,10 @@ public abstract class FileSystem extends Configured implements Closeable {
throw new NoSuchElementException("No more entry in " + f);
}
FileStatus result = stats[i++];
+ // for files, use getBlockLocations(FileStatus, int, int) to avoid
+ // calling getFileStatus(Path) to load the FileStatus again
BlockLocation[] locs = result.isFile() ?
- getFileBlockLocations(result.getPath(), 0, result.getLen()) :
+ getFileBlockLocations(result, 0, result.getLen()) :
null;
return new LocatedFileStatus(result, locs);
}
[4/4] hadoop git commit: HADOOP-12810. FileSystem#listLocatedStatus
causes unnecessary RPC calls (Contributed by Ryan Blue)
Posted by vi...@apache.org.
HADOOP-12810. FileSystem#listLocatedStatus causes unnecessary RPC calls (Contributed by Ryan Blue)
(cherry picked from commit 96ea3094315bb1e1a5e268e3817c7fdedc3e9462)
(cherry picked from commit 2bf5a005fab287689a148c1928f3b04c0a8dcedf)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9c1bb42
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9c1bb42
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9c1bb42
Branch: refs/heads/branch-2.7
Commit: a9c1bb42418dbde8958dbbc115f6045e696ea2ea
Parents: 05c7821
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Feb 17 10:13:41 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Feb 17 10:16:13 2016 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/fs/FileSystem.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c1bb42/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 446818f..cb10059 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -13,6 +13,9 @@ Release 2.7.3 - UNRELEASED
OPTIMIZATIONS
+ HADOOP-12810. FileSystem#listLocatedStatus causes unnecessary RPC calls
+ (Ryan Blue via vinayakumarb)
+
BUG FIXES
HADOOP-12296. when setnetgrent returns 0 in linux, exception should be
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c1bb42/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 623dd4a..35c28b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1723,8 +1723,10 @@ public abstract class FileSystem extends Configured implements Closeable {
throw new NoSuchElementException("No more entry in " + f);
}
FileStatus result = stats[i++];
+ // for files, use getBlockLocations(FileStatus, int, int) to avoid
+ // calling getFileStatus(Path) to load the FileStatus again
BlockLocation[] locs = result.isFile() ?
- getFileBlockLocations(result.getPath(), 0, result.getLen()) :
+ getFileBlockLocations(result, 0, result.getLen()) :
null;
return new LocatedFileStatus(result, locs);
}
[2/4] hadoop git commit: HADOOP-12810. FileSystem#listLocatedStatus
causes unnecessary RPC calls (Contributed by Ryan Blue)
Posted by vi...@apache.org.
HADOOP-12810. FileSystem#listLocatedStatus causes unnecessary RPC calls (Contributed by Ryan Blue)
(cherry picked from commit 96ea3094315bb1e1a5e268e3817c7fdedc3e9462)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2bf5a005
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2bf5a005
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2bf5a005
Branch: refs/heads/branch-2
Commit: 2bf5a005fab287689a148c1928f3b04c0a8dcedf
Parents: 6874a14
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Feb 17 10:13:41 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Feb 17 10:15:08 2016 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/fs/FileSystem.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bf5a005/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 7cb918f..900b6de 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1081,6 +1081,9 @@ Release 2.7.3 - UNRELEASED
OPTIMIZATIONS
+ HADOOP-12810. FileSystem#listLocatedStatus causes unnecessary RPC calls
+ (Ryan Blue via vinayakumarb)
+
BUG FIXES
HADOOP-12296. when setnetgrent returns 0 in linux, exception should be
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bf5a005/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 4b389dc..d9cb9e4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1739,8 +1739,10 @@ public abstract class FileSystem extends Configured implements Closeable {
throw new NoSuchElementException("No more entry in " + f);
}
FileStatus result = stats[i++];
+ // for files, use getBlockLocations(FileStatus, int, int) to avoid
+ // calling getFileStatus(Path) to load the FileStatus again
BlockLocation[] locs = result.isFile() ?
- getFileBlockLocations(result.getPath(), 0, result.getLen()) :
+ getFileBlockLocations(result, 0, result.getLen()) :
null;
return new LocatedFileStatus(result, locs);
}