You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by le...@apache.org on 2016/07/08 21:30:24 UTC
[17/21] hadoop git commit: get filestatus
get filestatus
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2817992
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2817992
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2817992
Branch: refs/heads/s3_create
Commit: a2817992526473643050b54576f8b6ff9fd99f26
Parents: 1e48a8a
Author: Lei Xu <le...@cloudera.com>
Authored: Wed Jul 6 15:52:52 2016 -0700
Committer: Lei Xu <le...@cloudera.com>
Committed: Wed Jul 6 15:52:52 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/s3a/CachedFileStatus.java | 6 +++---
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 21 ++++++++++++++++++--
.../org/apache/hadoop/fs/s3a/Statistic.java | 2 ++
3 files changed, 24 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2817992/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java
index 3cc7a39..4b0156e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java
@@ -12,13 +12,13 @@ import org.apache.hadoop.fs.FileStatus;
public class CachedFileStatus {
// TODO this may change to inheritance, not sure yet.
- protected FileStatus fileStatus;
+ protected S3AFileStatus fileStatus;
- public CachedFileStatus(FileStatus fileStatus) {
+ public CachedFileStatus(S3AFileStatus fileStatus) {
this.fileStatus = fileStatus;
}
- public FileStatus getFileStatus() {
+ public S3AFileStatus getFileStatus() {
return fileStatus;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2817992/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index d9f01b0..3299f2b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -126,6 +126,7 @@ public class S3AFileSystem extends FileSystem {
private S3AInstrumentation instrumentation;
private S3AStorageStatistics storageStatistics;
private long readAhead;
+ private MetadataStore metadataStore;
// The maximum number of entries that can be deleted in any call to s3
private static final int MAX_ENTRIES_TO_DELETE = 1000;
@@ -227,6 +228,8 @@ public class S3AFileSystem extends FileSystem {
serverSideEncryptionAlgorithm =
conf.getTrimmed(SERVER_SIDE_ENCRYPTION_ALGORITHM);
+
+ metadataStore = new LocalMetadataStore();
} catch (AmazonClientException e) {
throw translateException("initializing ", new Path(name), e);
}
@@ -1330,10 +1333,24 @@ public class S3AFileSystem extends FileSystem {
* @throws IOException on other problems.
*/
public S3AFileStatus getFileStatus(Path f) throws IOException {
+ if (metadataStore != null) {
+ incrementStatistic(INVOCATION_GET_FILE_STATUS_METASTORE);
+ S3AFileStatus status = metadataStore.get(f).getFileStatus();
+ if (status != null) {
+ return status;
+ }
+ }
+ S3AFileStatus status = innerGetFileStatus(f);
+ if (metadataStore != null) {
+ metadataStore.put(new CachedFileStatus(status));
+ }
+ return status;
+ }
+
+ private S3AFileStatus innerGetFileStatus(Path f) throws IOException {
String key = pathToKey(f);
incrementStatistic(INVOCATION_GET_FILE_STATUS);
- LOG.debug("Getting path status for {} ({})", f , key);
-
+ LOG.debug("Getting path status for {} ({})", f, key);
if (!key.isEmpty()) {
try {
ObjectMetadata meta = getObjectMetadata(key);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2817992/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index d29cb2f..b6ab450 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -44,6 +44,8 @@ public enum Statistic {
"Calls of exists()"),
INVOCATION_GET_FILE_STATUS("invocations_getfilestatus",
"Calls of getFileStatus()"),
+ INVOCATION_GET_FILE_STATUS_METASTORE("invocations_getfilestatus_metastore",
+ "Calls of getFileStatus() from metadata store"),
INVOCATION_GLOB_STATUS("invocations_globstatus",
"Calls of globStatus()"),
INVOCATION_IS_DIRECTORY("invocations_is_directory",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org