You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by le...@apache.org on 2016/07/08 21:30:24 UTC

[17/21] hadoop git commit: get filestatus

get filestatus


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2817992
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2817992
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2817992

Branch: refs/heads/s3_create
Commit: a2817992526473643050b54576f8b6ff9fd99f26
Parents: 1e48a8a
Author: Lei Xu <le...@cloudera.com>
Authored: Wed Jul 6 15:52:52 2016 -0700
Committer: Lei Xu <le...@cloudera.com>
Committed: Wed Jul 6 15:52:52 2016 -0700

----------------------------------------------------------------------
 .../apache/hadoop/fs/s3a/CachedFileStatus.java  |  6 +++---
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 21 ++++++++++++++++++--
 .../org/apache/hadoop/fs/s3a/Statistic.java     |  2 ++
 3 files changed, 24 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2817992/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java
index 3cc7a39..4b0156e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CachedFileStatus.java
@@ -12,13 +12,13 @@ import org.apache.hadoop.fs.FileStatus;
 public class CachedFileStatus {
 
   // TODO this may change to inheritance, not sure yet.
-  protected FileStatus fileStatus;
+  protected S3AFileStatus fileStatus;
 
-  public CachedFileStatus(FileStatus fileStatus) {
+  public CachedFileStatus(S3AFileStatus fileStatus) {
     this.fileStatus = fileStatus;
   }
 
-  public FileStatus getFileStatus() {
+  public S3AFileStatus getFileStatus() {
     return fileStatus;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2817992/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index d9f01b0..3299f2b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -126,6 +126,7 @@ public class S3AFileSystem extends FileSystem {
   private S3AInstrumentation instrumentation;
   private S3AStorageStatistics storageStatistics;
   private long readAhead;
+  private MetadataStore metadataStore;
 
   // The maximum number of entries that can be deleted in any call to s3
   private static final int MAX_ENTRIES_TO_DELETE = 1000;
@@ -227,6 +228,8 @@ public class S3AFileSystem extends FileSystem {
 
       serverSideEncryptionAlgorithm =
           conf.getTrimmed(SERVER_SIDE_ENCRYPTION_ALGORITHM);
+
+      metadataStore = new LocalMetadataStore();
     } catch (AmazonClientException e) {
       throw translateException("initializing ", new Path(name), e);
     }
@@ -1330,10 +1333,24 @@ public class S3AFileSystem extends FileSystem {
    * @throws IOException on other problems.
    */
   public S3AFileStatus getFileStatus(Path f) throws IOException {
+    if (metadataStore != null) {
+      incrementStatistic(INVOCATION_GET_FILE_STATUS_METASTORE);
+      S3AFileStatus status = metadataStore.get(f).getFileStatus();
+      if (status != null) {
+        return status;
+      }
+    }
+    S3AFileStatus status = innerGetFileStatus(f);
+    if (metadataStore != null) {
+       metadataStore.put(new CachedFileStatus(status));
+    }
+    return status;
+  }
+
+  private S3AFileStatus innerGetFileStatus(Path f) throws IOException {
     String key = pathToKey(f);
     incrementStatistic(INVOCATION_GET_FILE_STATUS);
-    LOG.debug("Getting path status for {}  ({})", f , key);
-
+    LOG.debug("Getting path status for {}  ({})", f, key);
     if (!key.isEmpty()) {
       try {
         ObjectMetadata meta = getObjectMetadata(key);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2817992/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index d29cb2f..b6ab450 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -44,6 +44,8 @@ public enum Statistic {
       "Calls of exists()"),
   INVOCATION_GET_FILE_STATUS("invocations_getfilestatus",
       "Calls of getFileStatus()"),
+  INVOCATION_GET_FILE_STATUS_METASTORE("invocations_getfilestatus_metastore",
+      "Calls of getFileStatus() from metadata store"),
   INVOCATION_GLOB_STATUS("invocations_globstatus",
       "Calls of globStatus()"),
   INVOCATION_IS_DIRECTORY("invocations_is_directory",


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org