[hbase] branch master updated: HBASE-23095 Reuse FileStatus in StoreFileInfo (#674)

[zh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
     new ff52994  HBASE-23095 Reuse FileStatus in StoreFileInfo (#674)
ff52994 is described below

commit ff529940766f6ead5aea6cafd5797c3f3a91a245
Author: Karthik Palanisamy <kp...@hortonworks.com>
AuthorDate: Tue Oct 8 00:47:12 2019 -0700

    HBASE-23095 Reuse FileStatus in StoreFileInfo (#674)
    
    Signed-off-by: Anoop Sam John <an...@gmail.com>
    Signed-off-by: Duo Zhang <zh...@apache.org>
---
 .../hadoop/hbase/regionserver/StoreFileInfo.java   |  26 ++++-
 .../hadoop/hbase/snapshot/SnapshotManifestV2.java  |   6 +-
 .../hbase/snapshot/TestSnapshotStoreFileSize.java  | 124 +++++++++++++++++++++
 3 files changed, 153 insertions(+), 3 deletions(-)

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
index 54e2780..f0437d5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
@@ -101,6 +101,8 @@ public class StoreFileInfo {
   // timestamp on when the file was created, is 0 and ignored for reference or link files
   private long createdTimestamp;
 
+  private long size;
+
   /**
    * Create a Store File Info
    * @param conf the {@link Configuration} to use
@@ -109,6 +111,11 @@ public class StoreFileInfo {
    */
   public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path initialPath)
       throws IOException {
+    this(conf, fs, null, initialPath);
+  }
+
+  private StoreFileInfo(final Configuration conf, final FileSystem fs,
+      final FileStatus fileStatus, final Path initialPath) throws IOException {
     assert fs != null;
     assert initialPath != null;
     assert conf != null;
@@ -136,7 +143,14 @@ public class StoreFileInfo {
               " reference to " + referencePath);
     } else if (isHFile(p)) {
       // HFile
-      this.createdTimestamp = fs.getFileStatus(initialPath).getModificationTime();
+      if (fileStatus != null) {
+        this.createdTimestamp = fileStatus.getModificationTime();
+        this.size = fileStatus.getLen();
+      } else {
+        FileStatus fStatus = fs.getFileStatus(initialPath);
+        this.createdTimestamp = fStatus.getModificationTime();
+        this.size = fStatus.getLen();
+      }
       this.reference = null;
       this.link = null;
     } else {
@@ -152,7 +166,7 @@ public class StoreFileInfo {
    */
   public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
       throws IOException {
-    this(conf, fs, fileStatus.getPath());
+    this(conf, fs, fileStatus, fileStatus.getPath());
   }
 
   /**
@@ -208,6 +222,14 @@ public class StoreFileInfo {
   }
 
   /**
+   * Size of the Hfile
+   * @return size
+   */
+  public long getSize() {
+    return size;
+  }
+
+  /**
    * Sets the region coprocessor env.
    * @param coprocessorHost
    */
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
index f5b97a0..f85dcfd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
@@ -129,7 +129,11 @@ public final class SnapshotManifestV2 {
       if (storeFile.isReference()) {
         sfManifest.setReference(storeFile.getReference().convert());
       }
-      sfManifest.setFileSize(storeFile.getReferencedFileStatus(rootFs).getLen());
+      if (!storeFile.isReference() && !storeFile.isLink()) {
+        sfManifest.setFileSize(storeFile.getSize());
+      } else {
+        sfManifest.setFileSize(storeFile.getReferencedFileStatus(rootFs).getLen());
+      }
       family.addStoreFiles(sfManifest.build());
     }
   }
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotStoreFileSize.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotStoreFileSize.java
new file mode 100644
index 0000000..9baa4e2
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotStoreFileSize.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.snapshot;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
+
+/**
+ * Validate if storefile length match
+ * both snapshop manifest and filesystem.
+ */
+@Category({ MasterTests.class, MediumTests.class })
+public class TestSnapshotStoreFileSize {
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+      HBaseClassTestRule.forClass(TestSnapshotStoreFileSize.class);
+
+  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static final TableName TABLE_NAME = TableName.valueOf("t1");
+  private static final String SNAPSHOT_NAME = "s1";
+  private static final String FAMILY_NAME = "cf";
+  private static Configuration conf;
+  private Admin admin;
+  private FileSystem fs;
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    conf = UTIL.getConfiguration();
+    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
+    UTIL.startMiniCluster(1);
+  }
+
+  @AfterClass
+  public static void teardown() throws Exception {
+    UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testIsStoreFileSizeMatchFilesystemAndManifest() throws IOException {
+    admin = UTIL.getAdmin();
+    fs = UTIL.getTestFileSystem();
+    UTIL.createTable(TABLE_NAME, FAMILY_NAME.getBytes());
+    Table table = admin.getConnection().getTable(TABLE_NAME);
+    UTIL.loadRandomRows(table, FAMILY_NAME.getBytes(), 3, 1000);
+    admin.snapshot(SNAPSHOT_NAME, TABLE_NAME);
+
+    Map<String, Long> storeFileInfoFromManifest = new HashMap<String, Long>();
+    Map<String, Long> storeFileInfoFromFS = new HashMap<String, Long>();
+    String storeFileName = "";
+    long storeFilesize = 0L;
+    Path snapshotDir = SnapshotDescriptionUtils
+        .getCompletedSnapshotDir(SNAPSHOT_NAME, UTIL.getDefaultRootDirPath());
+    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+    SnapshotManifest snaphotManifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
+    List<SnapshotRegionManifest> regionManifest = snaphotManifest.getRegionManifests();
+    for (int i = 0; i < regionManifest.size(); i++) {
+      SnapshotRegionManifest.FamilyFiles family = regionManifest.get(i).getFamilyFiles(0);
+      List<SnapshotRegionManifest.StoreFile> storeFiles = family.getStoreFilesList();
+      for (int j = 0; j < storeFiles.size(); j++) {
+        storeFileName = storeFiles.get(j).getName();
+        storeFilesize = storeFiles.get(j).getFileSize();
+        storeFileInfoFromManifest.put(storeFileName, storeFilesize);
+      }
+    }
+    List<RegionInfo> regionsInfo = admin.getRegions(TABLE_NAME);
+    Path path = FSUtils.getTableDir(UTIL.getDefaultRootDirPath(), TABLE_NAME);
+    for (RegionInfo regionInfo : regionsInfo) {
+      HRegionFileSystem hRegionFileSystem =
+          HRegionFileSystem.openRegionFromFileSystem(conf, fs, path, regionInfo, true);
+      Collection<StoreFileInfo> storeFilesFS = hRegionFileSystem.getStoreFiles(FAMILY_NAME);
+      Iterator<StoreFileInfo> sfIterator = storeFilesFS.iterator();
+      while (sfIterator.hasNext()) {
+        StoreFileInfo sfi = sfIterator.next();
+        FileStatus[] fileStatus = FSUtils.listStatus(fs, sfi.getPath());
+        storeFileName = fileStatus[0].getPath().getName();
+        storeFilesize = fileStatus[0].getLen();
+        storeFileInfoFromFS.put(storeFileName, storeFilesize);
+      }
+    }
+    Assert.assertEquals(storeFileInfoFromManifest, storeFileInfoFromFS);
+  }
+}