You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2016/09/29 16:08:22 UTC

[01/18] hbase git commit: HBASE-16723 RMI registry is not destroyed after stopping JMX Connector Server (Pankaj Kumar) [Forced Update!]

Repository: hbase
Updated Branches:
  refs/heads/hbase-14439 ab9e15127 -> 9f7596269 (forced update)


HBASE-16723 RMI registry is not destroyed after stopping JMX Connector Server (Pankaj Kumar)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f3270d44
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f3270d44
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f3270d44

Branch: refs/heads/hbase-14439
Commit: f3270d44c379c162dc3659fc892f52988b3a653b
Parents: 09a31bd
Author: tedyu <yu...@gmail.com>
Authored: Thu Sep 29 01:57:54 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu Sep 29 01:57:54 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hbase/JMXListener.java  | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/f3270d44/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java
index 2872cfa..9265fb8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java
@@ -27,8 +27,10 @@ import org.apache.hadoop.hbase.coprocessor.*;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.rmi.registry.LocateRegistry;
+import java.rmi.registry.Registry;
 import java.rmi.server.RMIClientSocketFactory;
 import java.rmi.server.RMIServerSocketFactory;
+import java.rmi.server.UnicastRemoteObject;
 import java.util.HashMap;
 
 import javax.management.MBeanServer;
@@ -36,8 +38,6 @@ import javax.management.remote.JMXConnectorServer;
 import javax.management.remote.JMXConnectorServerFactory;
 import javax.management.remote.JMXServiceURL;
 import javax.management.remote.rmi.RMIConnectorServer;
-import javax.rmi.ssl.SslRMIClientSocketFactory;
-import javax.rmi.ssl.SslRMIServerSocketFactory;
 
 /**
  * Pluggable JMX Agent for HBase(to fix the 2 random TCP ports issue
@@ -61,6 +61,7 @@ public class JMXListener implements Coprocessor {
    * we only load regionserver coprocessor on master
    */
   private static JMXConnectorServer JMX_CS = null;
+  private Registry rmiRegistry = null;
 
   public static JMXServiceURL buildJMXServiceURL(int rmiRegistryPort,
       int rmiConnectorPort) throws IOException {
@@ -128,7 +129,7 @@ public class JMXListener implements Coprocessor {
     }
 
     // Create the RMI registry
-    LocateRegistry.createRegistry(rmiRegistryPort);
+    rmiRegistry = LocateRegistry.createRegistry(rmiRegistryPort);
     // Retrieve the PlatformMBeanServer.
     MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
 
@@ -147,17 +148,25 @@ public class JMXListener implements Coprocessor {
       LOG.info("ConnectorServer started!");
     } catch (IOException e) {
       LOG.error("fail to start connector server!", e);
+      // deregister the RMI registry
+      if (rmiRegistry != null) {
+        UnicastRemoteObject.unexportObject(rmiRegistry, true);
+      }
     }
 
   }
 
   public void stopConnectorServer() throws IOException {
-    synchronized(JMXListener.class) {
+    synchronized (JMXListener.class) {
       if (JMX_CS != null) {
         JMX_CS.stop();
         LOG.info("ConnectorServer stopped!");
         JMX_CS = null;
       }
+      // deregister the RMI registry
+      if (rmiRegistry != null) {
+        UnicastRemoteObject.unexportObject(rmiRegistry, true);
+      }
     }
   }
 


[09/18] hbase git commit: HBASE-14439 Moved from MasterFileSystem to MasterStorage abstract class for storage abstraction layer.

Posted by bu...@apache.org.
HBASE-14439 Moved from MasterFileSystem to MasterStorage abstract class for storage abstraction layer.

Code compiles with 'mvn -DskipTests clean compile'.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9f759626
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9f759626
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9f759626

Branch: refs/heads/hbase-14439
Commit: 9f7596269c40e09d113e807e91b1a6235c4f0410
Parents: 9fddf3c
Author: Umesh Agashe <ua...@cloudera.com>
Authored: Fri Sep 23 16:40:39 2016 -0700
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Sep 29 11:07:29 2016 -0500

----------------------------------------------------------------------
 .../SplitLogManagerCoordination.java            |   4 +-
 .../org/apache/hadoop/hbase/fs/FsContext.java   |  28 --
 .../hadoop/hbase/fs/MasterFileSystem.java       | 330 -------------------
 .../apache/hadoop/hbase/fs/MasterStorage.java   | 271 +++++++++++++++
 .../apache/hadoop/hbase/fs/RegionStorage.java   |  63 +---
 .../apache/hadoop/hbase/fs/StorageContext.java  |  28 ++
 .../hbase/fs/legacy/HFileArchiveUtil.java       |   4 +-
 .../hbase/fs/legacy/LegacyMasterFileSystem.java | 164 ++++++---
 .../hadoop/hbase/master/CatalogJanitor.java     |  10 +-
 .../hbase/master/ClusterStatusPublisher.java    |   2 +-
 .../org/apache/hadoop/hbase/master/HMaster.java |  29 +-
 .../hadoop/hbase/master/MasterServices.java     |   6 +-
 .../hadoop/hbase/master/MasterWalManager.java   |  22 +-
 .../hadoop/hbase/master/RegionStates.java       |   6 +-
 .../procedure/CloneSnapshotProcedure.java       |  31 +-
 .../procedure/CreateNamespaceProcedure.java     |   3 +-
 .../master/procedure/CreateTableProcedure.java  |  20 +-
 .../procedure/DeleteNamespaceProcedure.java     |   8 +-
 .../master/procedure/DeleteTableProcedure.java  |  25 +-
 .../procedure/MasterDDLOperationHelper.java     |   8 +-
 .../procedure/RestoreSnapshotProcedure.java     |  15 +-
 .../master/snapshot/MasterSnapshotVerifier.java |   2 +-
 .../hbase/master/snapshot/SnapshotManager.java  |  43 ++-
 .../master/snapshot/TakeSnapshotHandler.java    |   5 +-
 .../hadoop/hbase/util/FSTableDescriptors.java   |  28 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java   |  27 +-
 .../org/apache/hadoop/hbase/util/HBaseFsck.java |  12 +-
 .../TestHColumnDescriptorDefaultVersions.java   |   5 +-
 .../org/apache/hadoop/hbase/TestNamespace.java  |   2 +-
 .../client/TestRestoreSnapshotFromClient.java   |   6 +-
 .../client/TestSnapshotCloneIndependence.java   |   5 +-
 .../hbase/client/TestSnapshotFromClient.java    |  12 +-
 .../hbase/client/TestSnapshotMetadata.java      |   4 +-
 .../hbase/client/TestTableSnapshotScanner.java  |   2 +-
 .../hadoop/hbase/fs/TestBlockReorder.java       |   2 +-
 .../TableSnapshotInputFormatTestBase.java       |   5 +-
 .../hadoop/hbase/mapreduce/TestWALPlayer.java   |   2 +-
 .../hbase/master/MockNoopMasterServices.java    |   2 +-
 .../hadoop/hbase/master/TestCatalogJanitor.java |  22 +-
 .../master/TestDistributedLogSplitting.java     |  11 +-
 .../hbase/master/TestMasterFileSystem.java      |   5 +-
 .../master/cleaner/TestSnapshotFromMaster.java  |   2 +-
 .../MasterProcedureTestingUtility.java          |  10 +-
 .../TestMasterFailoverWithProcedures.java       |   4 +-
 .../procedure/TestMasterProcedureWalLease.java  |   6 +-
 ...stTableDescriptorModificationFromClient.java |   5 +-
 .../master/snapshot/TestSnapshotManager.java    |   6 +-
 .../hbase/namespace/TestNamespaceAuditor.java   |   5 +-
 .../regionserver/TestCompactSplitThread.java    |   2 +-
 .../TestCorruptedRegionStoreFile.java           |  13 +-
 .../TestRegionMergeTransactionOnCluster.java    |   4 +-
 .../TestScannerRetriableFailure.java            |   4 +-
 .../TestSplitTransactionOnCluster.java          |  15 +-
 .../hbase/snapshot/SnapshotTestingUtils.java    |   8 +-
 .../hbase/snapshot/TestExportSnapshot.java      |   4 +-
 .../snapshot/TestFlushSnapshotFromClient.java   |  20 +-
 .../TestRestoreFlushSnapshotFromClient.java     |   2 +-
 .../hadoop/hbase/util/TestHBaseFsckOneRS.java   |   2 +-
 58 files changed, 666 insertions(+), 725 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
index 3afce96..3561330 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
@@ -42,11 +42,11 @@ import com.google.common.annotations.VisibleForTesting;
  * <P>
  * Methods required for task life circle: <BR>
  * {@link #markRegionsRecovering(ServerName, Set)} mark regions for log replaying. Used by
- * {@link org.apache.hadoop.hbase.fs.MasterFileSystem} <BR>
+ * {@link org.apache.hadoop.hbase.fs.MasterStorage} <BR>
  * {@link #removeRecoveringRegions(Set, Boolean)} make regions cleanup that previous were marked as
  * recovering. Called after all tasks processed <BR>
  * {@link #removeStaleRecoveringRegions(Set)} remove stale recovering. called by
- * {@link org.apache.hadoop.hbase.fs.MasterFileSystem} after Active Master is initialized <BR>
+ * {@link org.apache.hadoop.hbase.fs.MasterStorage} after Active Master is initialized <BR>
  * {@link #getLastRecoveryTime()} required for garbage collector and should indicate when the last
  * recovery has been made<BR>
  * {@link #checkTaskStillAvailable(String)} Check that task is still there <BR>

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java
deleted file mode 100644
index ced095d..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.fs;
-
-public enum FsContext {
-  TEMP,
-  DATA,
-  ARCHIVE,
-  SNAPSHOT,
-  SIDELINE,
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
deleted file mode 100644
index 3bc89d4..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.fs;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.ClusterId;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.legacy.LegacyMasterFileSystem;
-import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-
-@InterfaceAudience.Private
-public abstract class MasterFileSystem {
-  private static Log LOG = LogFactory.getLog(MasterFileSystem.class);
-
-  // Persisted unique cluster ID
-  private ClusterId clusterId;
-
-
-  private Configuration conf;
-  private FileSystem fs;
-  private Path rootDir;
-
-  protected MasterFileSystem(Configuration conf, FileSystem fs, Path rootDir) {
-    this.rootDir = rootDir;
-    this.conf = conf;
-    this.fs = fs;
-  }
-
-  public Configuration getConfiguration() { return conf; }
-  public FileSystem getFileSystem() { return fs; }
-  public Path getRootDir() { return rootDir; }
-
-  // ==========================================================================
-  //  PUBLIC Interfaces - Visitors
-  // ==========================================================================
-  public interface NamespaceVisitor {
-    void visitNamespace(String namespace) throws IOException;
-  }
-
-  public interface TableVisitor {
-    void visitTable(TableName tableName) throws IOException;
-  }
-
-  public interface RegionVisitor {
-    void visitRegion(HRegionInfo regionInfo) throws IOException;
-  }
-
-  // ==========================================================================
-  //  PUBLIC Methods - Namespace related
-  // ==========================================================================
-  public abstract void createNamespace(NamespaceDescriptor nsDescriptor) throws IOException;
-  public abstract void deleteNamespace(String namespaceName) throws IOException;
-  public abstract Collection<String> getNamespaces(FsContext ctx) throws IOException;
-
-  public Collection<String> getNamespaces() throws IOException {
-    return getNamespaces(FsContext.DATA);
-  }
-  // should return or get a NamespaceDescriptor? how is that different from HTD?
-
-  // ==========================================================================
-  //  PUBLIC Methods - Table Descriptor related
-  // ==========================================================================
-  public HTableDescriptor getTableDescriptor(TableName tableName)
-      throws IOException {
-    return getTableDescriptor(FsContext.DATA, tableName);
-  }
-
-  public boolean createTableDescriptor(HTableDescriptor tableDesc, boolean force)
-      throws IOException {
-    return createTableDescriptor(FsContext.DATA, tableDesc, force);
-  }
-
-  public void updateTableDescriptor(HTableDescriptor tableDesc) throws IOException {
-    updateTableDescriptor(FsContext.DATA, tableDesc);
-  }
-
-  public abstract HTableDescriptor getTableDescriptor(FsContext ctx, TableName tableName)
-      throws IOException;
-  public abstract boolean createTableDescriptor(FsContext ctx, HTableDescriptor tableDesc,
-      boolean force) throws IOException;
-  public abstract void updateTableDescriptor(FsContext ctx, HTableDescriptor tableDesc)
-      throws IOException;
-
-  // ==========================================================================
-  //  PUBLIC Methods - Table related
-  // ==========================================================================
-  public void deleteTable(TableName tableName) throws IOException {
-    deleteTable(FsContext.DATA, tableName);
-  }
-
-  public Collection<TableName> getTables(String namespace) throws IOException {
-    return getTables(FsContext.DATA, namespace);
-  }
-
-  public abstract void deleteTable(FsContext ctx, TableName tableName) throws IOException;
-
-  public abstract Collection<TableName> getTables(FsContext ctx, String namespace)
-    throws IOException;
-
-  public Collection<TableName> getTables() throws IOException {
-    ArrayList<TableName> tables = new ArrayList<TableName>();
-    for (String ns: getNamespaces()) {
-      tables.addAll(getTables(ns));
-    }
-    return tables;
-  }
-
-  // ==========================================================================
-  //  PUBLIC Methods - Table Region related
-  // ==========================================================================
-  public void deleteRegion(HRegionInfo regionInfo) throws IOException {
-    RegionStorage.destroy(conf, regionInfo);
-  }
-
-  public Collection<HRegionInfo> getRegions(TableName tableName) throws IOException {
-    return getRegions(FsContext.DATA, tableName);
-  }
-
-  public abstract Collection<HRegionInfo> getRegions(FsContext ctx, TableName tableName)
-    throws IOException;
-
-  // TODO: Move in HRegionStorage
-  public void deleteFamilyFromFS(HRegionInfo regionInfo, byte[] familyName, boolean hasMob)
-      throws IOException {
-    getRegionStorage(regionInfo).deleteFamily(Bytes.toString(familyName), hasMob);
-  }
-
-  public RegionStorage getRegionStorage(HRegionInfo regionInfo) throws IOException {
-    return RegionStorage.open(conf, regionInfo, false);
-  }
-
-  // ==========================================================================
-  //  PUBLIC Methods - visitors
-  // ==========================================================================
-  public void visitStoreFiles(StoreFileVisitor visitor)
-      throws IOException {
-    visitStoreFiles(FsContext.DATA, visitor);
-  }
-
-  public void visitStoreFiles(String namespace, StoreFileVisitor visitor)
-      throws IOException {
-    visitStoreFiles(FsContext.DATA, namespace, visitor);
-  }
-
-  public void visitStoreFiles(TableName table, StoreFileVisitor visitor)
-      throws IOException {
-    visitStoreFiles(FsContext.DATA, table, visitor);
-  }
-
-  public void visitStoreFiles(FsContext ctx, StoreFileVisitor visitor)
-      throws IOException {
-    for (String namespace: getNamespaces()) {
-      visitStoreFiles(ctx, namespace, visitor);
-    }
-  }
-
-  public void visitStoreFiles(FsContext ctx, String namespace, StoreFileVisitor visitor)
-      throws IOException {
-    for (TableName tableName: getTables(namespace)) {
-      visitStoreFiles(ctx, tableName, visitor);
-    }
-  }
-
-  public void visitStoreFiles(FsContext ctx, TableName table, StoreFileVisitor visitor)
-      throws IOException {
-    for (HRegionInfo hri: getRegions(ctx, table)) {
-      RegionStorage.open(conf, hri, false).visitStoreFiles(visitor);
-    }
-  }
-
-  // ==========================================================================
-  //  PUBLIC Methods - bootstrap
-  // ==========================================================================
-  public abstract Path getTempDir();
-
-  public void logFileSystemState(Log log) throws IOException {
-    FSUtils.logFileSystemState(getFileSystem(), getRootDir(), LOG);
-  }
-
-  /**
-   * @return The unique identifier generated for this cluster
-   */
-  public ClusterId getClusterId() {
-    return clusterId;
-  }
-
-  protected void bootstrap() throws IOException {
-    // check if the root directory exists
-    createInitialLayout(getRootDir(), conf, this.fs);
-
-    // check if temp directory exists and clean it
-    startupCleanup();
-  }
-
-  protected abstract void bootstrapMeta() throws IOException;
-  protected abstract void startupCleanup() throws IOException;
-
-  /**
-   * Create initial layout in filesystem.
-   * <ol>
-   * <li>Check if the meta region exists and is readable, if not create it.
-   * Create hbase.version and the hbase:meta directory if not one.
-   * </li>
-   * <li>Create a log archive directory for RS to put archived logs</li>
-   * </ol>
-   * Idempotent.
-   */
-  private void createInitialLayout(final Path rd, final Configuration c, final FileSystem fs)
-      throws IOException {
-    // If FS is in safe mode wait till out of it.
-    FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
-
-    // Filesystem is good. Go ahead and check for hbase.rootdir.
-    try {
-      if (!fs.exists(rd)) {
-        fs.mkdirs(rd);
-        // DFS leaves safe mode with 0 DNs when there are 0 blocks.
-        // We used to handle this by checking the current DN count and waiting until
-        // it is nonzero. With security, the check for datanode count doesn't work --
-        // it is a privileged op. So instead we adopt the strategy of the jobtracker
-        // and simply retry file creation during bootstrap indefinitely. As soon as
-        // there is one datanode it will succeed. Permission problems should have
-        // already been caught by mkdirs above.
-        FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
-          10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
-            HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
-      } else {
-        if (!fs.isDirectory(rd)) {
-          throw new IllegalArgumentException(rd.toString() + " is not a directory");
-        }
-        // as above
-        FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
-          10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
-            HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
-      }
-    } catch (DeserializationException de) {
-      LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
-      IOException ioe = new IOException();
-      ioe.initCause(de);
-      throw ioe;
-    } catch (IllegalArgumentException iae) {
-      LOG.fatal("Please fix invalid configuration for "
-        + HConstants.HBASE_DIR + " " + rd.toString(), iae);
-      throw iae;
-    }
-    // Make sure cluster ID exists
-    if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
-        HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
-      FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
-    }
-    clusterId = FSUtils.getClusterId(fs, rd);
-
-    // Make sure the meta region exists!
-    bootstrapMeta();
-  }
-
-  // ==========================================================================
-  //  PUBLIC
-  // ==========================================================================
-  public static MasterFileSystem open(Configuration conf, boolean bootstrap)
-      throws IOException {
-    return open(conf, FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), bootstrap);
-  }
-
-  public static MasterFileSystem open(Configuration conf, FileSystem fs,
-      Path rootDir, boolean bootstrap) throws IOException {
-    // Cover both bases, the old way of setting default fs and the new.
-    // We're supposed to run on 0.20 and 0.21 anyways.
-    fs = rootDir.getFileSystem(conf);
-    FSUtils.setFsDefault(conf, new Path(fs.getUri()));
-    // make sure the fs has the same conf
-    fs.setConf(conf);
-
-    MasterFileSystem mfs = getInstance(conf, fs, rootDir);
-    if (bootstrap) {
-      mfs.bootstrap();
-    }
-    HFileSystem.addLocationsOrderInterceptor(conf);
-    return mfs;
-  }
-
-  private static MasterFileSystem getInstance(Configuration conf, final FileSystem fs,
-      Path rootDir) throws IOException {
-    String fsType = conf.get("hbase.fs.layout.type", "legacy").toLowerCase();
-    switch (fsType) {
-      case "legacy":
-        return new LegacyMasterFileSystem(conf, fs, rootDir);
-      default:
-        throw new IOException("Invalid filesystem type " + fsType);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java
new file mode 100644
index 0000000..7b62dea
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java
@@ -0,0 +1,271 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.ClusterId;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.legacy.LegacyMasterFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+@InterfaceAudience.Private
+public abstract class MasterStorage<IDENTIFIER extends StorageIdentifier> {
+  private static Log LOG = LogFactory.getLog(MasterStorage.class);
+
+  // Persisted unique cluster ID
+  private ClusterId clusterId;
+
+
+  private Configuration conf;
+  private FileSystem fs;  // TODO: definitely remove
+  private IDENTIFIER rootContainer;
+
+  protected MasterStorage(Configuration conf, FileSystem fs, IDENTIFIER rootContainer) {
+    this.rootContainer = rootContainer;
+    this.conf = conf;
+    this.fs = fs;
+  }
+
+  public Configuration getConfiguration() { return conf; }
+  public FileSystem getFileSystem() { return fs; }  // TODO: definitely remove
+  public IDENTIFIER getRootContainer() { return rootContainer; }
+
+  // ==========================================================================
+  //  PUBLIC Interfaces - Visitors
+  // ==========================================================================
+  public interface NamespaceVisitor {
+    void visitNamespace(String namespace) throws IOException;
+  }
+
+  public interface TableVisitor {
+    void visitTable(TableName tableName) throws IOException;
+  }
+
+  public interface RegionVisitor {
+    void visitRegion(HRegionInfo regionInfo) throws IOException;
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - Namespace related
+  // ==========================================================================
+  public abstract void createNamespace(NamespaceDescriptor nsDescriptor) throws IOException;
+  public abstract void deleteNamespace(String namespaceName) throws IOException;
+  public abstract Collection<String> getNamespaces(StorageContext ctx) throws IOException;
+
+  public Collection<String> getNamespaces() throws IOException {
+    return getNamespaces(StorageContext.DATA);
+  }
+  // should return or get a NamespaceDescriptor? how is that different from HTD?
+
+  // ==========================================================================
+  //  PUBLIC Methods - Table Descriptor related
+  // ==========================================================================
+  public HTableDescriptor getTableDescriptor(TableName tableName)
+      throws IOException {
+    return getTableDescriptor(StorageContext.DATA, tableName);
+  }
+
+  public boolean createTableDescriptor(HTableDescriptor tableDesc, boolean force)
+      throws IOException {
+    return createTableDescriptor(StorageContext.DATA, tableDesc, force);
+  }
+
+  public void updateTableDescriptor(HTableDescriptor tableDesc) throws IOException {
+    updateTableDescriptor(StorageContext.DATA, tableDesc);
+  }
+
+  public abstract HTableDescriptor getTableDescriptor(StorageContext ctx, TableName tableName)
+      throws IOException;
+  public abstract boolean createTableDescriptor(StorageContext ctx, HTableDescriptor tableDesc,
+                                                boolean force) throws IOException;
+  public abstract void updateTableDescriptor(StorageContext ctx, HTableDescriptor tableDesc)
+      throws IOException;
+
+  // ==========================================================================
+  //  PUBLIC Methods - Table related
+  // ==========================================================================
+  public void deleteTable(TableName tableName) throws IOException {
+    deleteTable(StorageContext.DATA, tableName);
+  }
+
+  public Collection<TableName> getTables(String namespace) throws IOException {
+    return getTables(StorageContext.DATA, namespace);
+  }
+
+  public abstract void deleteTable(StorageContext ctx, TableName tableName) throws IOException;
+
+  public abstract Collection<TableName> getTables(StorageContext ctx, String namespace)
+    throws IOException;
+
+  public Collection<TableName> getTables() throws IOException {
+    ArrayList<TableName> tables = new ArrayList<TableName>();
+    for (String ns: getNamespaces()) {
+      tables.addAll(getTables(ns));
+    }
+    return tables;
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - Table Region related
+  // ==========================================================================
+  public void deleteRegion(HRegionInfo regionInfo) throws IOException {
+    RegionStorage.destroy(conf, regionInfo);
+  }
+
+  public Collection<HRegionInfo> getRegions(TableName tableName) throws IOException {
+    return getRegions(StorageContext.DATA, tableName);
+  }
+
+  public abstract Collection<HRegionInfo> getRegions(StorageContext ctx, TableName tableName)
+    throws IOException;
+
+  // TODO: Move in HRegionStorage
+  public void deleteFamilyFromStorage(HRegionInfo regionInfo, byte[] familyName, boolean hasMob)
+      throws IOException {
+    getRegionStorage(regionInfo).deleteFamily(Bytes.toString(familyName), hasMob);
+  }
+
+  public RegionStorage getRegionStorage(HRegionInfo regionInfo) throws IOException {
+    return RegionStorage.open(conf, regionInfo, false);
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - visitors
+  // ==========================================================================
+  public void visitStoreFiles(StoreFileVisitor visitor)
+      throws IOException {
+    visitStoreFiles(StorageContext.DATA, visitor);
+  }
+
+  public void visitStoreFiles(String namespace, StoreFileVisitor visitor)
+      throws IOException {
+    visitStoreFiles(StorageContext.DATA, namespace, visitor);
+  }
+
+  public void visitStoreFiles(TableName table, StoreFileVisitor visitor)
+      throws IOException {
+    visitStoreFiles(StorageContext.DATA, table, visitor);
+  }
+
+  public void visitStoreFiles(StorageContext ctx, StoreFileVisitor visitor)
+      throws IOException {
+    for (String namespace: getNamespaces()) {
+      visitStoreFiles(ctx, namespace, visitor);
+    }
+  }
+
+  public void visitStoreFiles(StorageContext ctx, String namespace, StoreFileVisitor visitor)
+      throws IOException {
+    for (TableName tableName: getTables(namespace)) {
+      visitStoreFiles(ctx, tableName, visitor);
+    }
+  }
+
+  public void visitStoreFiles(StorageContext ctx, TableName table, StoreFileVisitor visitor)
+      throws IOException {
+    for (HRegionInfo hri: getRegions(ctx, table)) {
+      RegionStorage.open(conf, hri, false).visitStoreFiles(visitor);
+    }
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - bootstrap
+  // ==========================================================================
+  public abstract IDENTIFIER getTempContainer();
+
+  public abstract void logStorageState(Log log) throws IOException;
+
+  /**
+   * @return The unique identifier generated for this cluster
+   */
+  public ClusterId getClusterId() {
+    return clusterId;
+  }
+
+  /**
+   * Bootstrap MasterStorage
+   * @throws IOException
+   */
+  protected void bootstrap() throws IOException {
+    // Initialize
+    clusterId = startup();
+
+    // Make sure the meta region exists!
+    bootstrapMeta();
+
+    // check if temp directory exists and clean it
+    startupCleanup();
+  }
+
+  protected abstract ClusterId startup() throws IOException;
+  protected abstract void bootstrapMeta() throws IOException;
+  protected abstract void startupCleanup() throws IOException;
+
+  // ==========================================================================
+  //  PUBLIC
+  // ==========================================================================
+  public static MasterStorage open(Configuration conf, boolean bootstrap)
+      throws IOException {
+    return open(conf, FSUtils.getRootDir(conf), bootstrap);
+  }
+
+  public static MasterStorage open(Configuration conf, Path rootDir, boolean bootstrap)
+      throws IOException {
+    // Cover both bases, the old way of setting default fs and the new.
+    // We're supposed to run on 0.20 and 0.21 anyways.
+    FileSystem fs = rootDir.getFileSystem(conf);
+    FSUtils.setFsDefault(conf, new Path(fs.getUri()));
+    // make sure the fs has the same conf
+    fs.setConf(conf);
+
+    MasterStorage ms = getInstance(conf, fs, rootDir);
+    if (bootstrap) {
+      ms.bootstrap();
+    }
+    HFileSystem.addLocationsOrderInterceptor(conf);
+    return ms;
+  }
+
+  private static MasterStorage getInstance(Configuration conf, final FileSystem fs,
+                                           Path rootDir) throws IOException {
+    String storageType = conf.get("hbase.storage.type", "legacy").toLowerCase();
+    switch (storageType) {
+      case "legacy":
+        return new LegacyMasterFileSystem(conf, fs, new LegacyPathIdentifier(rootDir));
+      default:
+        throw new IOException("Invalid filesystem type " + storageType);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
index ba7d0bf..990b00b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
@@ -20,20 +20,15 @@
 package org.apache.hadoop.hbase.fs;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collection;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.ClusterId;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
@@ -43,62 +38,14 @@ import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
-import java.util.UUID;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.FSUtilsWithRetries;
-import org.apache.hadoop.hbase.fs.FsContext;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.MetaUtils;
-import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.fs.HFileSystem;
+
 import org.apache.hadoop.hbase.regionserver.*;
-import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSHDFSUtils;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
 
 @InterfaceAudience.Private
 public abstract class RegionStorage<IDENTIFIER extends StorageIdentifier> {
@@ -331,12 +278,12 @@ public abstract class RegionStorage<IDENTIFIER extends StorageIdentifier> {
 
   private static RegionStorage getInstance(Configuration conf, FileSystem fs,
       StorageIdentifier rootContainer, StorageIdentifier regionContainer) throws IOException {
-    String fsType = conf.get("hbase.fs.storage.type", "legacy").toLowerCase();
-    switch (fsType) {
+    String storageType = conf.get("hbase.storage.type", "legacy").toLowerCase();
+    switch (storageType) {
       case "legacy":
         return new LegacyRegionStorage(conf, fs, (LegacyPathIdentifier)rootContainer, (LegacyPathIdentifier)regionContainer);
       default:
-        throw new IOException("Invalid filesystem type " + fsType);
+        throw new IOException("Invalid filesystem type " + storageType);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/StorageContext.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/StorageContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/StorageContext.java
new file mode 100644
index 0000000..cc324a9
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/StorageContext.java
@@ -0,0 +1,28 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+public enum StorageContext {
+  TEMP,
+  DATA,
+  ARCHIVE,
+  SNAPSHOT,
+  SIDELINE,
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
index 2fbaa92..e422861 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
@@ -133,7 +133,7 @@ public class HFileArchiveUtil {
 
   /**
    * Get the full path to the archive directory on the configured 
-   * {@link org.apache.hadoop.hbase.fs.MasterFileSystem}
+   * {@link org.apache.hadoop.hbase.fs.MasterStorage}
    * @param conf to look for archive directory name and root directory. Cannot be null. Notes for
    *          testing: requires a FileSystem root directory to be specified.
    * @return the full {@link Path} to the archive directory, as defined by the configuration
@@ -145,7 +145,7 @@ public class HFileArchiveUtil {
 
   /**
    * Get the full path to the archive directory on the configured 
-   * {@link org.apache.hadoop.hbase.fs.MasterFileSystem}
+   * {@link org.apache.hadoop.hbase.fs.MasterStorage}
    * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
    *          the archive path)
    * @return the full {@link Path} to the archive directory, as defined by the configuration

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
index 5cf363b..caa7ee5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
@@ -34,15 +34,12 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.FsContext;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.StorageContext;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
 import org.apache.hadoop.hbase.mob.MobConstants;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -52,7 +49,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.backup.HFileArchiver;
 
 @InterfaceAudience.Private
-public class LegacyMasterFileSystem extends MasterFileSystem {
+public class LegacyMasterFileSystem extends MasterStorage<LegacyPathIdentifier> {
   private static final Log LOG = LogFactory.getLog(LegacyMasterFileSystem.class);
 
   private final Path sidelineDir;
@@ -82,18 +79,18 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
 
   private final boolean isSecurityEnabled;
 
-  public LegacyMasterFileSystem(Configuration conf, FileSystem fs, Path rootDir) {
+  public LegacyMasterFileSystem(Configuration conf, FileSystem fs, LegacyPathIdentifier rootDir) {
     super(conf, fs, rootDir);
 
     // base directories
-    this.sidelineDir = LegacyLayout.getSidelineDir(rootDir);
-    this.snapshotDir = LegacyLayout.getSnapshotDir(rootDir);
-    this.archiveDir = LegacyLayout.getArchiveDir(rootDir);
+    this.sidelineDir = LegacyLayout.getSidelineDir(rootDir.path);
+    this.snapshotDir = LegacyLayout.getSnapshotDir(rootDir.path);
+    this.archiveDir = LegacyLayout.getArchiveDir(rootDir.path);
     this.archiveDataDir = LegacyLayout.getDataDir(this.archiveDir);
-    this.dataDir = LegacyLayout.getDataDir(rootDir);
-    this.tmpDir = LegacyLayout.getTempDir(rootDir);
+    this.dataDir = LegacyLayout.getDataDir(rootDir.path);
+    this.tmpDir = LegacyLayout.getTempDir(rootDir.path);
     this.tmpDataDir = LegacyLayout.getDataDir(this.tmpDir);
-    this.bulkDir = LegacyLayout.getBulkDir(rootDir);
+    this.bulkDir = LegacyLayout.getBulkDir(rootDir.path);
 
     this.secureRootSubDirPerms = new FsPermission(conf.get("hbase.rootdir.perms", "700"));
     this.isSecurityEnabled = "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
@@ -103,12 +100,12 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
   //  PUBLIC Methods - Namespace related
   // ==========================================================================
   public void createNamespace(NamespaceDescriptor nsDescriptor) throws IOException {
-    getFileSystem().mkdirs(getNamespaceDir(FsContext.DATA, nsDescriptor.getName()));
+    getFileSystem().mkdirs(getNamespaceDir(StorageContext.DATA, nsDescriptor.getName()));
   }
 
   public void deleteNamespace(String namespaceName) throws IOException {
     FileSystem fs = getFileSystem();
-    Path nsDir = getNamespaceDir(FsContext.DATA, namespaceName);
+    Path nsDir = getNamespaceDir(StorageContext.DATA, namespaceName);
 
     try {
       for (FileStatus status : fs.listStatus(nsDir)) {
@@ -125,7 +122,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
     }
   }
 
-  public Collection<String> getNamespaces(FsContext ctx) throws IOException {
+  public Collection<String> getNamespaces(StorageContext ctx) throws IOException {
     FileStatus[] stats = FSUtils.listStatus(getFileSystem(), getNamespaceDir(ctx));
     if (stats == null) return Collections.emptyList();
 
@@ -142,20 +139,20 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
   //  PUBLIC Methods - Table Descriptor related
   // ==========================================================================s
   @Override
-  public boolean createTableDescriptor(FsContext ctx, HTableDescriptor tableDesc, boolean force)
+  public boolean createTableDescriptor(StorageContext ctx, HTableDescriptor tableDesc, boolean force)
       throws IOException {
     return LegacyTableDescriptor.createTableDescriptor(getFileSystem(),
       getTableDir(ctx, tableDesc.getTableName()), tableDesc, force);
   }
 
   @Override
-  public void updateTableDescriptor(FsContext ctx, HTableDescriptor tableDesc) throws IOException {
+  public void updateTableDescriptor(StorageContext ctx, HTableDescriptor tableDesc) throws IOException {
     LegacyTableDescriptor.updateTableDescriptor(getFileSystem(),
         getTableDir(ctx, tableDesc.getTableName()), tableDesc);
   }
 
   @Override
-  public HTableDescriptor getTableDescriptor(FsContext ctx, TableName tableName)
+  public HTableDescriptor getTableDescriptor(StorageContext ctx, TableName tableName)
       throws IOException {
     return LegacyTableDescriptor.getTableDescriptorFromFs(
         getFileSystem(), getTableDir(ctx, tableName));
@@ -165,7 +162,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
   //  PUBLIC Methods - Table related
   // ==========================================================================
   @Override
-  public void deleteTable(FsContext ctx, TableName tableName) throws IOException {
+  public void deleteTable(StorageContext ctx, TableName tableName) throws IOException {
     Path tableDir = getTableDir(ctx, tableName);
     if (!FSUtils.deleteDirectory(getFileSystem(), tableDir)) {
       throw new IOException("Failed delete of " + tableName);
@@ -173,7 +170,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
   }
 
   @Override
-  public Collection<TableName> getTables(FsContext ctx, String namespace)
+  public Collection<TableName> getTables(StorageContext ctx, String namespace)
       throws IOException {
     FileStatus[] stats = FSUtils.listStatus(getFileSystem(),
         getNamespaceDir(ctx, namespace), new FSUtils.UserTableDirFilter(getFileSystem()));
@@ -190,7 +187,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
   //  PUBLIC Methods - Table Regions related
   // ==========================================================================
   @Override
-  public Collection<HRegionInfo> getRegions(FsContext ctx, TableName tableName)
+  public Collection<HRegionInfo> getRegions(StorageContext ctx, TableName tableName)
       throws IOException {
     FileStatus[] stats = FSUtils.listStatus(getFileSystem(),
         getTableDir(ctx, tableName), new FSUtils.RegionDirFilter(getFileSystem()));
@@ -215,10 +212,96 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
   // ==========================================================================
   //  PROTECTED Methods - Bootstrap
   // ==========================================================================
+
+  /**
+   * Create initial layout in filesystem.
+   * <ol>
+   * <li>Check if the meta region exists and is readable, if not create it.
+   * Create hbase.version and the hbase:meta directory if not one.
+   * </li>
+   * <li>Create a log archive directory for RS to put archived logs</li>
+   * </ol>
+   * Idempotent.
+   * @throws IOException
+   */
+  @Override
+  protected ClusterId startup() throws IOException {
+    Configuration c = getConfiguration();
+    Path rc = ((LegacyPathIdentifier)getRootContainer()).path;
+    FileSystem fs = getFileSystem();
+
+    // If FS is in safe mode wait till out of it.
+    FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
+
+    boolean isSecurityEnabled = "kerberos".equalsIgnoreCase(c.get("hbase.security.authentication"));
+    FsPermission rootDirPerms = new FsPermission(c.get("hbase.rootdir.perms", "700"));
+
+    // Filesystem is good. Go ahead and check for hbase.rootdir.
+    try {
+      if (!fs.exists(rc)) {
+        if (isSecurityEnabled) {
+          fs.mkdirs(rc, rootDirPerms);
+        } else {
+          fs.mkdirs(rc);
+        }
+        // DFS leaves safe mode with 0 DNs when there are 0 blocks.
+        // We used to handle this by checking the current DN count and waiting until
+        // it is nonzero. With security, the check for datanode count doesn't work --
+        // it is a privileged op. So instead we adopt the strategy of the jobtracker
+        // and simply retry file creation during bootstrap indefinitely. As soon as
+        // there is one datanode it will succeed. Permission problems should have
+        // already been caught by mkdirs above.
+        FSUtils.setVersion(fs, rc, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+            10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
+            HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
+      } else {
+        if (!fs.isDirectory(rc)) {
+          throw new IllegalArgumentException(rc.toString() + " is not a directory");
+        }
+        if (isSecurityEnabled && !rootDirPerms.equals(fs.getFileStatus(rc).getPermission())) {
+          // check whether the permission match
+          LOG.warn("Found rootdir permissions NOT matching expected \"hbase.rootdir.perms\" for "
+              + "rootdir=" + rc.toString() + " permissions=" + fs.getFileStatus(rc).getPermission()
+              + " and  \"hbase.rootdir.perms\" configured as "
+              + c.get("hbase.rootdir.perms", "700") + ". Automatically setting the permissions. You"
+              + " can change the permissions by setting \"hbase.rootdir.perms\" in hbase-site.xml "
+              + "and restarting the master");
+          fs.setPermission(rc, rootDirPerms);
+        }
+        // as above
+        FSUtils.checkVersion(fs, rc, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+            10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
+            HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
+      }
+    } catch (DeserializationException de) {
+      LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
+      IOException ioe = new IOException();
+      ioe.initCause(de);
+      throw ioe;
+    } catch (IllegalArgumentException iae) {
+      LOG.fatal("Please fix invalid configuration for "
+          + HConstants.HBASE_DIR + " " + rc.toString(), iae);
+      throw iae;
+    }
+    // Make sure cluster ID exists
+    if (!FSUtils.checkClusterIdExists(fs, rc, c.getInt(
+        HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
+      FSUtils.setClusterId(fs, rc, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10
+          * 1000));
+    }
+    return FSUtils.getClusterId(fs, rc);
+  }
+
+  @Override
+  public void logStorageState(Log log) throws IOException {
+    FSUtils.logFileSystemState(getFileSystem(), ((LegacyPathIdentifier)getRootContainer()).path,
+        LOG);
+  }
+
   @Override
   protected void bootstrapMeta() throws IOException {
     // TODO ask RegionStorage
-    if (!FSUtils.metaRegionExists(getFileSystem(), getRootDir())) {
+    if (!FSUtils.metaRegionExists(getFileSystem(), getRootContainer().path)) {
       bootstrapMeta(getConfiguration());
     }
 
@@ -254,7 +337,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
   protected void startupCleanup() throws IOException {
     final FileSystem fs = getFileSystem();
     // Check the directories under rootdir.
-    checkTempDir(getTempDir(), getConfiguration(), fs);
+    checkTempDir(getTempContainer().path, getConfiguration(), getFileSystem());
     final String[] protectedSubDirs = new String[] {
         HConstants.BASE_NAMESPACE_DIR,
         HConstants.HFILE_ARCHIVE_DIRECTORY,
@@ -266,7 +349,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
         MobConstants.MOB_DIR_NAME
     };
     for (String subDir : protectedSubDirs) {
-      checkSubDir(new Path(getRootDir(), subDir));
+      checkSubDir(new Path(getRootContainer().path, subDir));
     }
 
     checkStagingDir();
@@ -274,17 +357,17 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
     // Handle the last few special files and set the final rootDir permissions
     // rootDir needs 'x' for all to support bulk load staging dir
     if (isSecurityEnabled) {
-      fs.setPermission(new Path(getRootDir(), HConstants.VERSION_FILE_NAME), secureRootFilePerms);
-      fs.setPermission(new Path(getRootDir(), HConstants.CLUSTER_ID_FILE_NAME), secureRootFilePerms);
+      fs.setPermission(new Path(getRootContainer().path, HConstants.VERSION_FILE_NAME), secureRootFilePerms);
+      fs.setPermission(new Path(getRootContainer().path, HConstants.CLUSTER_ID_FILE_NAME), secureRootFilePerms);
     }
-    FsPermission currentRootPerms = fs.getFileStatus(getRootDir()).getPermission();
+    FsPermission currentRootPerms = fs.getFileStatus(getRootContainer().path).getPermission();
     if (!currentRootPerms.getUserAction().implies(FsAction.EXECUTE)
         || !currentRootPerms.getGroupAction().implies(FsAction.EXECUTE)
         || !currentRootPerms.getOtherAction().implies(FsAction.EXECUTE)) {
       LOG.warn("rootdir permissions do not contain 'excute' for user, group or other. "
         + "Automatically adding 'excute' permission for all");
       fs.setPermission(
-        getRootDir(),
+        getRootContainer().path,
         new FsPermission(currentRootPerms.getUserAction().or(FsAction.EXECUTE), currentRootPerms
             .getGroupAction().or(FsAction.EXECUTE), currentRootPerms.getOtherAction().or(
           FsAction.EXECUTE)));
@@ -303,7 +386,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
       // if not the cleaner will take care of them.
       for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
         for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
-          HFileArchiver.archiveRegion(fs, getRootDir(), tabledir, regiondir);
+          HFileArchiver.archiveRegion(fs, getRootContainer().path, tabledir, regiondir);
         }
       }
       if (!fs.delete(tmpdir, true)) {
@@ -361,7 +444,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
    */
   private void checkStagingDir() throws IOException {
     final FileSystem fs = getFileSystem();
-    Path p = new Path(getRootDir(), HConstants.BULKLOAD_STAGING_DIR_NAME);
+    Path p = new Path(getRootContainer().path, HConstants.BULKLOAD_STAGING_DIR_NAME);
     try {
       if (!fs.exists(p)) {
         if (!fs.mkdirs(p, HiddenDirPerms)) {
@@ -380,27 +463,28 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
   // ==========================================================================
   //  PROTECTED Methods - Path
   // ==========================================================================
-  protected Path getNamespaceDir(FsContext ctx) {
+  protected Path getNamespaceDir(StorageContext ctx) {
     return getBaseDirFromContext(ctx);
   }
 
-  protected Path getNamespaceDir(FsContext ctx, String namespace) {
+  protected Path getNamespaceDir(StorageContext ctx, String namespace) {
     return LegacyLayout.getNamespaceDir(getBaseDirFromContext(ctx), namespace);
   }
 
-  protected Path getTableDir(FsContext ctx, TableName table) {
+  protected Path getTableDir(StorageContext ctx, TableName table) {
     return LegacyLayout.getTableDir(getBaseDirFromContext(ctx), table);
   }
 
-  protected Path getRegionDir(FsContext ctx, TableName table, HRegionInfo hri) {
+  protected Path getRegionDir(StorageContext ctx, TableName table, HRegionInfo hri) {
     return LegacyLayout.getRegionDir(getTableDir(ctx, table), hri);
   }
 
-  public Path getTempDir() {
-    return tmpDir;
+  @Override
+  public LegacyPathIdentifier getTempContainer() {
+    return new LegacyPathIdentifier(tmpDir);
   }
 
-  protected Path getBaseDirFromContext(FsContext ctx) {
+  protected Path getBaseDirFromContext(StorageContext ctx) {
     switch (ctx) {
       case TEMP: return tmpDataDir;
       case DATA: return dataDir;

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index afddaf6..6f958f0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
@@ -207,7 +208,7 @@ public class CatalogJanitor extends ScheduledChore {
           + regionB.getRegionNameAsString()
           + " from fs because merged region no longer holds references");
       // TODO update HFileArchiver to use RegionStorage
-      FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
+      FileSystem fs = this.services.getMasterStorage().getFileSystem();
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
       MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
@@ -353,7 +354,7 @@ public class CatalogJanitor extends ScheduledChore {
     if (hasNoReferences(a) && hasNoReferences(b)) {
       LOG.debug("Deleting region " + parent.getRegionNameAsString() +
         " because daughter splits no longer hold references");
-      FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
+      FileSystem fs = this.services.getMasterStorage().getFileSystem();
       if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
       MetaTableAccessor.deleteRegion(this.connection, parent);
@@ -389,8 +390,9 @@ public class CatalogJanitor extends ScheduledChore {
       return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
     }
 
-    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
-    Path rootdir = this.services.getMasterFileSystem().getRootDir();
+    FileSystem fs = this.services.getMasterStorage().getFileSystem();
+    Path rootdir = ((LegacyPathIdentifier) this.services.getMasterStorage().getRootContainer())
+        .path;
     Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());
 
     Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
index 194e023..41c25e9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
@@ -163,7 +163,7 @@ public class ClusterStatusPublisher extends ScheduledChore {
     // This could be extended in the future, for example if we want to send stuff like the
     //  hbase:meta server name.
     ClusterStatus cs = new ClusterStatus(VersionInfo.getVersion(),
-        master.getMasterFileSystem().getClusterId().toString(),
+        master.getMasterStorage().getClusterId().toString(),
         null,
         sns,
         master.getServerName(),

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c9056e3..9181579 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -89,8 +89,9 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.exceptions.MergeRegionException;
 import org.apache.hadoop.hbase.executor.ExecutorType;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.ipc.RpcServer;
 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
 import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
@@ -119,7 +120,6 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler.Procedu
 import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure;
 import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
 import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
-import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
 import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.mob.MobConstants;
@@ -142,8 +142,6 @@ import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
 import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy;
 import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesZKImpl;
 import org.apache.hadoop.hbase.replication.master.TableCFsUpdater;
 import org.apache.hadoop.hbase.replication.regionserver.Replication;
 import org.apache.hadoop.hbase.security.UserProvider;
@@ -164,7 +162,6 @@ import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
 import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
 import org.apache.hadoop.hbase.zookeeper.MasterMaintenanceModeTracker;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
 import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker;
 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
 import org.apache.hadoop.hbase.zookeeper.SplitOrMergeTracker;
@@ -275,8 +272,8 @@ public class HMaster extends HRegionServer implements MasterServices {
 
   // Metrics for the HMaster
   final MetricsMaster metricsMaster;
-  // file system manager for the master FS operations
-  private MasterFileSystem fileSystemManager;
+  // storage manager for the master storage operations
+  private MasterStorage storageManager;
   private MasterWalManager walManager;
 
   // server manager to deal with region server info
@@ -682,7 +679,7 @@ public class HMaster extends HRegionServer implements MasterServices {
 
     this.masterActiveTime = System.currentTimeMillis();
     // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
-    this.fileSystemManager = MasterFileSystem.open(conf, true);
+    this.storageManager = MasterStorage.open(conf, true);
     this.walManager = new MasterWalManager(this);
 
     // enable table descriptors cache
@@ -698,7 +695,7 @@ public class HMaster extends HRegionServer implements MasterServices {
 
     // publish cluster ID
     status.setStatus("Publishing Cluster ID in ZooKeeper");
-    ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
+    ZKClusterId.setClusterId(this.zooKeeper, storageManager.getClusterId());
     this.initLatch.countDown();
 
     this.serverManager = createServerManager(this);
@@ -922,8 +919,8 @@ public class HMaster extends HRegionServer implements MasterServices {
   }
 
   @Override
-  public MasterFileSystem getMasterFileSystem() {
-    return this.fileSystemManager;
+  public MasterStorage getMasterStorage() {
+    return this.storageManager;
   }
 
   @Override
@@ -976,7 +973,7 @@ public class HMaster extends HRegionServer implements MasterServices {
     Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
     Map<String, Object> params = new HashMap<String, Object>();
     params.put(MASTER, this);
-    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()
+    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterStorage()
         .getFileSystem(), archiveDir, params);
     getChoreService().scheduleChore(hfileCleaner);
     serviceStarted = true;
@@ -1030,10 +1027,10 @@ public class HMaster extends HRegionServer implements MasterServices {
 
   private void startProcedureExecutor() throws IOException {
     final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
-    final Path logDir = new Path(fileSystemManager.getRootDir(),
+    final Path logDir = new Path(((LegacyPathIdentifier) storageManager.getRootContainer()).path,
         MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR);
 
-    procedureStore = new WALProcedureStore(conf, fileSystemManager.getFileSystem(), logDir,
+    procedureStore = new WALProcedureStore(conf, storageManager.getFileSystem(), logDir,
         new MasterProcedureEnv.WALStoreLeaseRecovery(this));
     procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
     procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore,
@@ -2133,8 +2130,8 @@ public class HMaster extends HRegionServer implements MasterServices {
         }});
     }
 
-    String clusterId = fileSystemManager != null ?
-      fileSystemManager.getClusterId().toString() : null;
+    String clusterId = storageManager != null ?
+      storageManager.getClusterId().toString() : null;
     Set<RegionState> regionsInTransition = assignmentManager != null ?
       assignmentManager.getRegionStates().getRegionsInTransition() : null;
     String[] coprocessors = cpHost != null ? getMasterCoprocessors() : null;

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index 3dfe4a6..2a6704d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
@@ -68,9 +68,9 @@ public interface MasterServices extends Server {
   AssignmentManager getAssignmentManager();
 
   /**
-   * @return Master's filesystem {@link MasterFileSystem} utility class.
+   * @return Master's filesystem {@link MasterStorage} utility class.
    */
-  MasterFileSystem getMasterFileSystem();
+  MasterStorage getMasterStorage();
 
   /**
    * @return Master's WALs {@link MasterWalManager} utility class.

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
index e90b744..28b4c63 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -79,7 +81,7 @@ public class MasterWalManager {
 
   // The Path to the old logs dir
   private final Path oldLogDir;
-  private final Path rootDir;
+  private final StorageIdentifier rootContainer;
 
   // create the split log lock
   private final Lock splitLogLock = new ReentrantLock();
@@ -90,20 +92,21 @@ public class MasterWalManager {
   private volatile boolean fsOk = true;
 
   public MasterWalManager(MasterServices services) throws IOException {
-    this(services.getConfiguration(), services.getMasterFileSystem().getFileSystem(),
-      services.getMasterFileSystem().getRootDir(), services);
+    this(services.getConfiguration(), services.getMasterStorage().getFileSystem(),
+      services.getMasterStorage().getRootContainer(), services);
   }
 
-  public MasterWalManager(Configuration conf, FileSystem fs, Path rootDir, MasterServices services)
-      throws IOException {
+  public MasterWalManager(Configuration conf, FileSystem fs, StorageIdentifier rootContainer,
+                          MasterServices services) throws IOException {
     this.fs = fs;
     this.conf = conf;
-    this.rootDir = rootDir;
+    this.rootContainer = rootContainer;
     this.services = services;
     this.splitLogManager = new SplitLogManager(services, conf);
     this.distributedLogReplay = this.splitLogManager.isLogReplaying();
 
-    this.oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+    this.oldLogDir = new Path(((LegacyPathIdentifier) rootContainer).path, HConstants
+        .HREGION_OLDLOGDIR_NAME);
   }
 
   public void stop() {
@@ -156,7 +159,8 @@ public class MasterWalManager {
         WALSplitter.SPLIT_SKIP_ERRORS_DEFAULT);
 
     Set<ServerName> serverNames = new HashSet<ServerName>();
-    Path logsDirPath = new Path(this.rootDir, HConstants.HREGION_LOGDIR_NAME);
+    Path logsDirPath = new Path(((LegacyPathIdentifier) this.rootContainer).path, HConstants
+        .HREGION_LOGDIR_NAME);
 
     do {
       if (services.isStopped()) {
@@ -254,7 +258,7 @@ public class MasterWalManager {
     }
     try {
       for (ServerName serverName : serverNames) {
-        Path logDir = new Path(this.rootDir,
+        Path logDir = new Path(((LegacyPathIdentifier) this.rootContainer).path,
           AbstractFSWALProvider.getWALDirectoryName(serverName.toString()));
         Path splitDir = logDir.suffix(AbstractFSWALProvider.SPLITTING_EXT);
         // Rename the directory so a rogue RS doesn't create more WALs

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index e875f30..8b70685 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -20,13 +20,10 @@ package org.apache.hadoop.hbase.master;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -52,7 +49,6 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil;
 import org.apache.hadoop.hbase.master.RegionState.State;
 import org.apache.hadoop.hbase.client.TableState;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
@@ -679,7 +675,7 @@ public class RegionStates {
         // This is a cleanup task. Not critical.
         if (MetaTableAccessor.getRegion(server.getConnection(), hri.getEncodedNameAsBytes()) == null) {
           regionOffline(hri);
-          server.getMasterFileSystem().deleteRegion(hri);
+          server.getMasterStorage().deleteRegion(hri);
         }
       } catch (IOException e) {
         LOG.warn("Got exception while deleting " + hri + " directories from file system.", e);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
index f52e321..f3cabbd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.master.procedure;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -41,8 +40,10 @@ import org.apache.hadoop.hbase.TableExistsException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.fs.FsContext;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.StorageContext;
+import org.apache.hadoop.hbase.fs.MasterStorage;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
 import org.apache.hadoop.hbase.master.MetricsSnapshot;
 import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions;
@@ -52,7 +53,6 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CloneSnapshotState;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -308,13 +308,13 @@ public class CloneSnapshotProcedure
       throws IOException, InterruptedException {
     if (!getTableName().isSystemTable()) {
       // Check and update namespace quota
-      final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+      final MasterStorage ms = env.getMasterServices().getMasterStorage();
 
       SnapshotManifest manifest = SnapshotManifest.open(
         env.getMasterConfiguration(),
-        mfs.getFileSystem(),
-        SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()),
-        snapshot);
+        ms.getFileSystem(),
+        SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, ((LegacyPathIdentifier) ms
+            .getRootContainer()).path), snapshot);
 
       ProcedureSyncWait.getMasterQuotaManager(env)
         .checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size());
@@ -358,9 +358,9 @@ public class CloneSnapshotProcedure
         final Path tableRootDir, final TableName tableName,
         final List<HRegionInfo> newRegions) throws IOException {
 
-        final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
-        final FileSystem fs = mfs.getFileSystem();
-        final Path rootDir = mfs.getRootDir();
+        final MasterStorage ms = env.getMasterServices().getMasterStorage();
+        final FileSystem fs = ms.getFileSystem();
+        final StorageIdentifier rootContainer = ms.getRootContainer();
         final Configuration conf = env.getMasterConfiguration();
         final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher();
 
@@ -368,7 +368,8 @@ public class CloneSnapshotProcedure
 
         try {
           // 1. Execute the on-disk Clone
-          Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+          Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot,
+              ((LegacyPathIdentifier) rootContainer).path);
           SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
           RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
             conf, fs, manifest, hTableDescriptor, tableRootDir, monitorException, monitorStatus);
@@ -413,14 +414,14 @@ public class CloneSnapshotProcedure
     final HTableDescriptor hTableDescriptor,
     List<HRegionInfo> newRegions,
     final CreateHdfsRegions hdfsRegionHandler) throws IOException {
-    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
-    final Path tempdir = mfs.getTempDir();
+    final MasterStorage ms = env.getMasterServices().getMasterStorage();
+    final Path tempdir = ((LegacyPathIdentifier)ms.getTempContainer()).path;
 
     // 1. Create Table Descriptor
     // using a copy of descriptor, table will be created enabling first
     HTableDescriptor underConstruction = new HTableDescriptor(hTableDescriptor);
     final Path tempTableDir = FSUtils.getTableDir(tempdir, hTableDescriptor.getTableName());
-    mfs.createTableDescriptor(FsContext.TEMP, underConstruction, false);
+    ms.createTableDescriptor(StorageContext.TEMP, underConstruction, false);
 
     // 2. Create Regions
     newRegions = hdfsRegionHandler.createHdfsRegions(

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
index 7562a04..0b4dc5d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.hbase.master.TableNamespaceManager;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CreateNamespaceState;
-import org.apache.hadoop.hbase.util.FSUtils;
 
 /**
  * The procedure to create a new namespace.
@@ -202,7 +201,7 @@ public class CreateNamespaceProcedure
   protected static void createDirectory(
       final MasterProcedureEnv env,
       final NamespaceDescriptor nsDescriptor) throws IOException {
-    env.getMasterServices().getMasterFileSystem().createNamespace(nsDescriptor);
+    env.getMasterServices().getMasterStorage().createNamespace(nsDescriptor);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
index 196cb16..e44192d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.master.procedure;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -38,11 +37,11 @@ import org.apache.hadoop.hbase.TableExistsException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
 import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.fs.FsContext;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.StorageContext;
+import org.apache.hadoop.hbase.fs.MasterStorage;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
-import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
@@ -294,14 +293,14 @@ public class CreateTableProcedure
   protected static List<HRegionInfo> createFsLayout(final MasterProcedureEnv env,
       final HTableDescriptor hTableDescriptor, List<HRegionInfo> newRegions,
       final CreateHdfsRegions hdfsRegionHandler) throws IOException {
-    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
-    final Path tempdir = mfs.getTempDir();
+    final MasterStorage ms = env.getMasterServices().getMasterStorage();
+    final Path tempdir = ((LegacyPathIdentifier) ms.getTempContainer()).path;
 
     // 1. Create Table Descriptor
     // using a copy of descriptor, table will be created enabling first
     HTableDescriptor underConstruction = new HTableDescriptor(hTableDescriptor);
     final Path tempTableDir = FSUtils.getTableDir(tempdir, hTableDescriptor.getTableName());
-    mfs.createTableDescriptor(FsContext.TEMP, underConstruction, false);
+    ms.createTableDescriptor(StorageContext.TEMP, underConstruction, false);
 
     // 2. Create Regions
     newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir,
@@ -317,9 +316,10 @@ public class CreateTableProcedure
     final MasterProcedureEnv env,
     final HTableDescriptor hTableDescriptor,
     final Path tempTableDir) throws IOException {
-    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
-    final Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), hTableDescriptor.getTableName());
-    FileSystem fs = mfs.getFileSystem();
+    final MasterStorage ms = env.getMasterServices().getMasterStorage();
+    final Path tableDir = FSUtils.getTableDir(((LegacyPathIdentifier)ms.getRootContainer()).path,
+        hTableDescriptor.getTableName());
+    FileSystem fs = ms.getFileSystem();
 
     if (!fs.delete(tableDir, true) && fs.exists(tableDir)) {
       throw new IOException("Couldn't delete " + tableDir);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
index c6ca374..2069a6a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
@@ -25,20 +25,14 @@ import java.io.OutputStream;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.NamespaceNotFoundException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.constraint.ConstraintException;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.master.TableNamespaceManager;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.DeleteNamespaceState;
-import org.apache.hadoop.hbase.util.FSUtils;
 
 /**
  * The procedure to remove a namespace.
@@ -275,7 +269,7 @@ public class DeleteNamespaceProcedure
   protected static void deleteDirectory(
       final MasterProcedureEnv env,
       final String namespaceName) throws IOException {
-    env.getMasterServices().getMasterFileSystem().deleteNamespace(namespaceName);
+    env.getMasterServices().getMasterStorage().deleteNamespace(namespaceName);
   }
 
   /**


[05/18] hbase git commit: HBASE-14439 break everything HRegion

Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
deleted file mode 100644
index 12e93c2..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++ /dev/null
@@ -1,1109 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.regionserver;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSHDFSUtils;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
-
-/**
- * View to an on-disk Region.
- * Provides the set of methods necessary to interact with the on-disk region data.
- */
-@InterfaceAudience.Private
-public class HRegionFileSystem {
-  private static final Log LOG = LogFactory.getLog(HRegionFileSystem.class);
-
-  /** Name of the region info file that resides just under the region directory. */
-  public final static String REGION_INFO_FILE = ".regioninfo";
-
-  /** Temporary subdirectory of the region directory used for merges. */
-  public static final String REGION_MERGES_DIR = ".merges";
-
-  /** Temporary subdirectory of the region directory used for splits. */
-  public static final String REGION_SPLITS_DIR = ".splits";
-
-  /** Temporary subdirectory of the region directory used for compaction output. */
-  private static final String REGION_TEMP_DIR = ".tmp";
-
-  private final HRegionInfo regionInfo;
-  //regionInfo for interacting with FS (getting encodedName, etc)
-  private final HRegionInfo regionInfoForFs;
-  private final Configuration conf;
-  private final Path tableDir;
-  private final FileSystem fs;
-
-  /**
-   * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
-   * client level.
-   */
-  private final int hdfsClientRetriesNumber;
-  private final int baseSleepBeforeRetries;
-  private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
-  private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
-
-  /**
-   * Create a view to the on-disk region
-   * @param conf the {@link Configuration} to use
-   * @param fs {@link FileSystem} that contains the region
-   * @param tableDir {@link Path} to where the table is being stored
-   * @param regionInfo {@link HRegionInfo} for region
-   */
-  HRegionFileSystem(final Configuration conf, final FileSystem fs, final Path tableDir,
-      final HRegionInfo regionInfo) {
-    this.fs = fs;
-    this.conf = conf;
-    this.tableDir = tableDir;
-    this.regionInfo = regionInfo;
-    this.regionInfoForFs = ServerRegionReplicaUtil.getRegionInfoForFs(regionInfo);
-    this.hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
-      DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
-    this.baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
-      DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
- }
-
-  /** @return the underlying {@link FileSystem} */
-  public FileSystem getFileSystem() {
-    return this.fs;
-  }
-
-  /** @return the {@link HRegionInfo} that describe this on-disk region view */
-  public HRegionInfo getRegionInfo() {
-    return this.regionInfo;
-  }
-
-  public HRegionInfo getRegionInfoForFS() {
-    return this.regionInfoForFs;
-  }
-
-  /** @return {@link Path} to the region's root directory. */
-  public Path getTableDir() {
-    return this.tableDir;
-  }
-
-  /** @return {@link Path} to the region directory. */
-  public Path getRegionDir() {
-    return new Path(this.tableDir, this.regionInfoForFs.getEncodedName());
-  }
-
-  // ===========================================================================
-  //  Temp Helpers
-  // ===========================================================================
-  /** @return {@link Path} to the region's temp directory, used for file creations */
-  Path getTempDir() {
-    return new Path(getRegionDir(), REGION_TEMP_DIR);
-  }
-
-  /**
-   * Clean up any temp detritus that may have been left around from previous operation attempts.
-   */
-  void cleanupTempDir() throws IOException {
-    deleteDir(getTempDir());
-  }
-
-  // ===========================================================================
-  //  Store/StoreFile Helpers
-  // ===========================================================================
-  /**
-   * Returns the directory path of the specified family
-   * @param familyName Column Family Name
-   * @return {@link Path} to the directory of the specified family
-   */
-  public Path getStoreDir(final String familyName) {
-    return new Path(this.getRegionDir(), familyName);
-  }
-
-  /**
-   * Create the store directory for the specified family name
-   * @param familyName Column Family Name
-   * @return {@link Path} to the directory of the specified family
-   * @throws IOException if the directory creation fails.
-   */
-  Path createStoreDir(final String familyName) throws IOException {
-    Path storeDir = getStoreDir(familyName);
-    if(!fs.exists(storeDir) && !createDir(storeDir))
-      throw new IOException("Failed creating "+storeDir);
-    return storeDir;
-  }
-
-  /**
-   * Returns the store files available for the family.
-   * This methods performs the filtering based on the valid store files.
-   * @param familyName Column Family Name
-   * @return a set of {@link StoreFileInfo} for the specified family.
-   */
-  public Collection<StoreFileInfo> getStoreFiles(final byte[] familyName) throws IOException {
-    return getStoreFiles(Bytes.toString(familyName));
-  }
-
-  public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
-    return getStoreFiles(familyName, true);
-  }
-
-  /**
-   * Returns the store files available for the family.
-   * This methods performs the filtering based on the valid store files.
-   * @param familyName Column Family Name
-   * @return a set of {@link StoreFileInfo} for the specified family.
-   */
-  public Collection<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate)
-      throws IOException {
-    Path familyDir = getStoreDir(familyName);
-    FileStatus[] files = FSUtils.listStatus(this.fs, familyDir);
-    if (files == null) {
-      if (LOG.isTraceEnabled()) {
-        LOG.trace("No StoreFiles for: " + familyDir);
-      }
-      return null;
-    }
-
-    ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(files.length);
-    for (FileStatus status: files) {
-      if (validate && !StoreFileInfo.isValid(status)) {
-        LOG.warn("Invalid StoreFile: " + status.getPath());
-        continue;
-      }
-      StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo,
-        regionInfoForFs, familyName, status.getPath());
-      storeFiles.add(info);
-
-    }
-    return storeFiles;
-  }
-
-  /**
-   * Return Qualified Path of the specified family/file
-   *
-   * @param familyName Column Family Name
-   * @param fileName File Name
-   * @return The qualified Path for the specified family/file
-   */
-  Path getStoreFilePath(final String familyName, final String fileName) {
-    Path familyDir = getStoreDir(familyName);
-    return new Path(familyDir, fileName).makeQualified(this.fs);
-  }
-
-  /**
-   * Return the store file information of the specified family/file.
-   *
-   * @param familyName Column Family Name
-   * @param fileName File Name
-   * @return The {@link StoreFileInfo} for the specified family/file
-   */
-  StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
-      throws IOException {
-    Path familyDir = getStoreDir(familyName);
-    return ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo,
-      regionInfoForFs, familyName, new Path(familyDir, fileName));
-  }
-
-  /**
-   * Returns true if the specified family has reference files
-   * @param familyName Column Family Name
-   * @return true if family contains reference files
-   * @throws IOException
-   */
-  public boolean hasReferences(final String familyName) throws IOException {
-    FileStatus[] files = FSUtils.listStatus(fs, getStoreDir(familyName));
-    if (files != null) {
-      for(FileStatus stat: files) {
-        if(stat.isDirectory()) {
-          continue;
-        }
-        if(StoreFileInfo.isReference(stat.getPath())) {
-          return true;
-        }
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Check whether region has Reference file
-   * @param htd table desciptor of the region
-   * @return true if region has reference file
-   * @throws IOException
-   */
-  public boolean hasReferences(final HTableDescriptor htd) throws IOException {
-    for (HColumnDescriptor family : htd.getFamilies()) {
-      if (hasReferences(family.getNameAsString())) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /**
-   * @return the set of families present on disk
-   * @throws IOException
-   */
-  public Collection<String> getFamilies() throws IOException {
-    FileStatus[] fds = FSUtils.listStatus(fs, getRegionDir(), new FSUtils.FamilyDirFilter(fs));
-    if (fds == null) return null;
-
-    ArrayList<String> families = new ArrayList<String>(fds.length);
-    for (FileStatus status: fds) {
-      families.add(status.getPath().getName());
-    }
-
-    return families;
-  }
-
-  /**
-   * Remove the region family from disk, archiving the store files.
-   * @param familyName Column Family Name
-   * @throws IOException if an error occours during the archiving
-   */
-  public void deleteFamily(final String familyName) throws IOException {
-    // archive family store files
-    HFileArchiver.archiveFamily(fs, conf, regionInfoForFs, tableDir, Bytes.toBytes(familyName));
-
-    // delete the family folder
-    Path familyDir = getStoreDir(familyName);
-    if(fs.exists(familyDir) && !deleteDir(familyDir))
-      throw new IOException("Could not delete family " + familyName
-          + " from FileSystem for region " + regionInfoForFs.getRegionNameAsString() + "("
-          + regionInfoForFs.getEncodedName() + ")");
-  }
-
-  /**
-   * Generate a unique file name, used by createTempName() and commitStoreFile()
-   * @param suffix extra information to append to the generated name
-   * @return Unique file name
-   */
-  private static String generateUniqueName(final String suffix) {
-    String name = UUID.randomUUID().toString().replaceAll("-", "");
-    if (suffix != null) name += suffix;
-    return name;
-  }
-
-  /**
-   * Generate a unique temporary Path. Used in conjuction with commitStoreFile()
-   * to get a safer file creation.
-   * <code>
-   * Path file = fs.createTempName();
-   * ...StoreFile.Writer(file)...
-   * fs.commitStoreFile("family", file);
-   * </code>
-   *
-   * @return Unique {@link Path} of the temporary file
-   */
-  public Path createTempName() {
-    return createTempName(null);
-  }
-
-  /**
-   * Generate a unique temporary Path. Used in conjuction with commitStoreFile()
-   * to get a safer file creation.
-   * <code>
-   * Path file = fs.createTempName();
-   * ...StoreFile.Writer(file)...
-   * fs.commitStoreFile("family", file);
-   * </code>
-   *
-   * @param suffix extra information to append to the generated name
-   * @return Unique {@link Path} of the temporary file
-   */
-  public Path createTempName(final String suffix) {
-    return new Path(getTempDir(), generateUniqueName(suffix));
-  }
-
-  /**
-   * Move the file from a build/temp location to the main family store directory.
-   * @param familyName Family that will gain the file
-   * @param buildPath {@link Path} to the file to commit.
-   * @return The new {@link Path} of the committed file
-   * @throws IOException
-   */
-  public Path commitStoreFile(final String familyName, final Path buildPath) throws IOException {
-    return commitStoreFile(familyName, buildPath, -1, false);
-  }
-
-  /**
-   * Move the file from a build/temp location to the main family store directory.
-   * @param familyName Family that will gain the file
-   * @param buildPath {@link Path} to the file to commit.
-   * @param seqNum Sequence Number to append to the file name (less then 0 if no sequence number)
-   * @param generateNewName False if you want to keep the buildPath name
-   * @return The new {@link Path} of the committed file
-   * @throws IOException
-   */
-  private Path commitStoreFile(final String familyName, final Path buildPath,
-      final long seqNum, final boolean generateNewName) throws IOException {
-    Path storeDir = getStoreDir(familyName);
-    if(!fs.exists(storeDir) && !createDir(storeDir))
-      throw new IOException("Failed creating " + storeDir);
-
-    String name = buildPath.getName();
-    if (generateNewName) {
-      name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_");
-    }
-    Path dstPath = new Path(storeDir, name);
-    if (!fs.exists(buildPath)) {
-      throw new FileNotFoundException(buildPath.toString());
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Committing store file " + buildPath + " as " + dstPath);
-    }
-    // buildPath exists, therefore not doing an exists() check.
-    if (!rename(buildPath, dstPath)) {
-      throw new IOException("Failed rename of " + buildPath + " to " + dstPath);
-    }
-    return dstPath;
-  }
-
-
-  /**
-   * Moves multiple store files to the relative region's family store directory.
-   * @param storeFiles list of store files divided by family
-   * @throws IOException
-   */
-  void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException {
-    for (Map.Entry<byte[], List<StoreFile>> es: storeFiles.entrySet()) {
-      String familyName = Bytes.toString(es.getKey());
-      for (StoreFile sf: es.getValue()) {
-        commitStoreFile(familyName, sf.getPath());
-      }
-    }
-  }
-
-  /**
-   * Archives the specified store file from the specified family.
-   * @param familyName Family that contains the store files
-   * @param filePath {@link Path} to the store file to remove
-   * @throws IOException if the archiving fails
-   */
-  public void removeStoreFile(final String familyName, final Path filePath)
-      throws IOException {
-    HFileArchiver.archiveStoreFile(this.conf, this.fs, this.regionInfoForFs,
-        this.tableDir, Bytes.toBytes(familyName), filePath);
-  }
-
-  /**
-   * Closes and archives the specified store files from the specified family.
-   * @param familyName Family that contains the store files
-   * @param storeFiles set of store files to remove
-   * @throws IOException if the archiving fails
-   */
-  public void removeStoreFiles(final String familyName, final Collection<StoreFile> storeFiles)
-      throws IOException {
-    HFileArchiver.archiveStoreFiles(this.conf, this.fs, this.regionInfoForFs,
-        this.tableDir, Bytes.toBytes(familyName), storeFiles);
-  }
-
-  /**
-   * Bulk load: Add a specified store file to the specified family.
-   * If the source file is on the same different file-system is moved from the
-   * source location to the destination location, otherwise is copied over.
-   *
-   * @param familyName Family that will gain the file
-   * @param srcPath {@link Path} to the file to import
-   * @param seqNum Bulk Load sequence number
-   * @return The destination {@link Path} of the bulk loaded file
-   * @throws IOException
-   */
-  Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
-      throws IOException {
-    // Copy the file if it's on another filesystem
-    FileSystem srcFs = srcPath.getFileSystem(conf);
-    FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs;
-
-    // We can't compare FileSystem instances as equals() includes UGI instance
-    // as part of the comparison and won't work when doing SecureBulkLoad
-    // TODO deal with viewFS
-    if (!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)) {
-      LOG.info("Bulk-load file " + srcPath + " is on different filesystem than " +
-          "the destination store. Copying file over to destination filesystem.");
-      Path tmpPath = createTempName();
-      FileUtil.copy(srcFs, srcPath, fs, tmpPath, false, conf);
-      LOG.info("Copied " + srcPath + " to temporary path on destination filesystem: " + tmpPath);
-      srcPath = tmpPath;
-    }
-
-    return commitStoreFile(familyName, srcPath, seqNum, true);
-  }
-
-  // ===========================================================================
-  //  Splits Helpers
-  // ===========================================================================
-  /** @return {@link Path} to the temp directory used during split operations */
-  Path getSplitsDir() {
-    return new Path(getRegionDir(), REGION_SPLITS_DIR);
-  }
-
-  Path getSplitsDir(final HRegionInfo hri) {
-    return new Path(getSplitsDir(), hri.getEncodedName());
-  }
-
-  /**
-   * Clean up any split detritus that may have been left around from previous split attempts.
-   */
-  void cleanupSplitsDir() throws IOException {
-    deleteDir(getSplitsDir());
-  }
-
-  /**
-   * Clean up any split detritus that may have been left around from previous
-   * split attempts.
-   * Call this method on initial region deploy.
-   * @throws IOException
-   */
-  void cleanupAnySplitDetritus() throws IOException {
-    Path splitdir = this.getSplitsDir();
-    if (!fs.exists(splitdir)) return;
-    // Look at the splitdir.  It could have the encoded names of the daughter
-    // regions we tried to make.  See if the daughter regions actually got made
-    // out under the tabledir.  If here under splitdir still, then the split did
-    // not complete.  Try and do cleanup.  This code WILL NOT catch the case
-    // where we successfully created daughter a but regionserver crashed during
-    // the creation of region b.  In this case, there'll be an orphan daughter
-    // dir in the filesystem.  TOOD: Fix.
-    FileStatus[] daughters = FSUtils.listStatus(fs, splitdir, new FSUtils.DirFilter(fs));
-    if (daughters != null) {
-      for (FileStatus daughter: daughters) {
-        Path daughterDir = new Path(getTableDir(), daughter.getPath().getName());
-        if (fs.exists(daughterDir) && !deleteDir(daughterDir)) {
-          throw new IOException("Failed delete of " + daughterDir);
-        }
-      }
-    }
-    cleanupSplitsDir();
-    LOG.info("Cleaned up old failed split transaction detritus: " + splitdir);
-  }
-
-  /**
-   * Remove daughter region
-   * @param regionInfo daughter {@link HRegionInfo}
-   * @throws IOException
-   */
-  void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException {
-    Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName());
-    if (this.fs.exists(regionDir) && !deleteDir(regionDir)) {
-      throw new IOException("Failed delete of " + regionDir);
-    }
-  }
-
-  /**
-   * Commit a daughter region, moving it from the split temporary directory
-   * to the proper location in the filesystem.
-   *
-   * @param regionInfo                 daughter {@link org.apache.hadoop.hbase.HRegionInfo}
-   * @throws IOException
-   */
-  Path commitDaughterRegion(final HRegionInfo regionInfo)
-      throws IOException {
-    Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName());
-    Path daughterTmpDir = this.getSplitsDir(regionInfo);
-
-    if (fs.exists(daughterTmpDir)) {
-
-      // Write HRI to a file in case we need to recover hbase:meta
-      Path regionInfoFile = new Path(daughterTmpDir, REGION_INFO_FILE);
-      byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
-      writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
-
-      // Move the daughter temp dir to the table dir
-      if (!rename(daughterTmpDir, regionDir)) {
-        throw new IOException("Unable to rename " + daughterTmpDir + " to " + regionDir);
-      }
-    }
-
-    return regionDir;
-  }
-
-  /**
-   * Create the region splits directory.
-   */
-  void createSplitsDir() throws IOException {
-    Path splitdir = getSplitsDir();
-    if (fs.exists(splitdir)) {
-      LOG.info("The " + splitdir + " directory exists.  Hence deleting it to recreate it");
-      if (!deleteDir(splitdir)) {
-        throw new IOException("Failed deletion of " + splitdir
-            + " before creating them again.");
-      }
-    }
-    // splitDir doesn't exists now. No need to do an exists() call for it.
-    if (!createDir(splitdir)) {
-      throw new IOException("Failed create of " + splitdir);
-    }
-  }
-
-  /**
-   * Write out a split reference. Package local so it doesnt leak out of
-   * regionserver.
-   * @param hri {@link HRegionInfo} of the destination
-   * @param familyName Column Family Name
-   * @param f File to split.
-   * @param splitRow Split Row
-   * @param top True if we are referring to the top half of the hfile.
-   * @param splitPolicy
-   * @return Path to created reference.
-   * @throws IOException
-   */
-  Path splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f,
-      final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy)
-          throws IOException {
-    if (splitPolicy == null || !splitPolicy.skipStoreFileRangeCheck(familyName)) {
-      // Check whether the split row lies in the range of the store file
-      // If it is outside the range, return directly.
-      try {
-        if (top) {
-          //check if larger than last key.
-          Cell splitKey = CellUtil.createFirstOnRow(splitRow);
-          Cell lastKey = f.getLastKey();
-          // If lastKey is null means storefile is empty.
-          if (lastKey == null) {
-            return null;
-          }
-          if (f.getComparator().compare(splitKey, lastKey) > 0) {
-            return null;
-          }
-        } else {
-          //check if smaller than first key
-          Cell splitKey = CellUtil.createLastOnRow(splitRow);
-          Cell firstKey = f.getFirstKey();
-          // If firstKey is null means storefile is empty.
-          if (firstKey == null) {
-            return null;
-          }
-          if (f.getComparator().compare(splitKey, firstKey) < 0) {
-            return null;
-          }
-        }
-      } finally {
-        f.closeReader(f.getCacheConf() != null ? f.getCacheConf().shouldEvictOnClose() : true);
-      }
-    }
-
-    Path splitDir = new Path(getSplitsDir(hri), familyName);
-    // A reference to the bottom half of the hsf store file.
-    Reference r =
-      top ? Reference.createTopReference(splitRow): Reference.createBottomReference(splitRow);
-    // Add the referred-to regions name as a dot separated suffix.
-    // See REF_NAME_REGEX regex above.  The referred-to regions name is
-    // up in the path of the passed in <code>f</code> -- parentdir is family,
-    // then the directory above is the region name.
-    String parentRegionName = regionInfoForFs.getEncodedName();
-    // Write reference with same file id only with the other region name as
-    // suffix and into the new region location (under same family).
-    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
-    return r.write(fs, p);
-  }
-
-  // ===========================================================================
-  //  Merge Helpers
-  // ===========================================================================
-  /** @return {@link Path} to the temp directory used during merge operations */
-  Path getMergesDir() {
-    return new Path(getRegionDir(), REGION_MERGES_DIR);
-  }
-
-  Path getMergesDir(final HRegionInfo hri) {
-    return new Path(getMergesDir(), hri.getEncodedName());
-  }
-
-  /**
-   * Clean up any merge detritus that may have been left around from previous merge attempts.
-   */
-  void cleanupMergesDir() throws IOException {
-    deleteDir(getMergesDir());
-  }
-
-  /**
-   * Remove merged region
-   * @param mergedRegion {@link HRegionInfo}
-   * @throws IOException
-   */
-  void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException {
-    Path regionDir = new Path(this.tableDir, mergedRegion.getEncodedName());
-    if (this.fs.exists(regionDir) && !this.fs.delete(regionDir, true)) {
-      throw new IOException("Failed delete of " + regionDir);
-    }
-  }
-
-  /**
-   * Create the region merges directory.
-   * @throws IOException If merges dir already exists or we fail to create it.
-   * @see HRegionFileSystem#cleanupMergesDir()
-   */
-  void createMergesDir() throws IOException {
-    Path mergesdir = getMergesDir();
-    if (fs.exists(mergesdir)) {
-      LOG.info("The " + mergesdir
-          + " directory exists.  Hence deleting it to recreate it");
-      if (!fs.delete(mergesdir, true)) {
-        throw new IOException("Failed deletion of " + mergesdir
-            + " before creating them again.");
-      }
-    }
-    if (!fs.mkdirs(mergesdir))
-      throw new IOException("Failed create of " + mergesdir);
-  }
-
-  /**
-   * Write out a merge reference under the given merges directory. Package local
-   * so it doesnt leak out of regionserver.
-   * @param mergedRegion {@link HRegionInfo} of the merged region
-   * @param familyName Column Family Name
-   * @param f File to create reference.
-   * @param mergedDir
-   * @return Path to created reference.
-   * @throws IOException
-   */
-  Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
-      final StoreFile f, final Path mergedDir)
-      throws IOException {
-    Path referenceDir = new Path(new Path(mergedDir,
-        mergedRegion.getEncodedName()), familyName);
-    // A whole reference to the store file.
-    Reference r = Reference.createTopReference(regionInfoForFs.getStartKey());
-    // Add the referred-to regions name as a dot separated suffix.
-    // See REF_NAME_REGEX regex above. The referred-to regions name is
-    // up in the path of the passed in <code>f</code> -- parentdir is family,
-    // then the directory above is the region name.
-    String mergingRegionName = regionInfoForFs.getEncodedName();
-    // Write reference with same file id only with the other region name as
-    // suffix and into the new region location (under same family).
-    Path p = new Path(referenceDir, f.getPath().getName() + "."
-        + mergingRegionName);
-    return r.write(fs, p);
-  }
-
-  /**
-   * Commit a merged region, moving it from the merges temporary directory to
-   * the proper location in the filesystem.
-   * @param mergedRegionInfo merged region {@link HRegionInfo}
-   * @throws IOException
-   */
-  void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException {
-    Path regionDir = new Path(this.tableDir, mergedRegionInfo.getEncodedName());
-    Path mergedRegionTmpDir = this.getMergesDir(mergedRegionInfo);
-    // Move the tmp dir in the expected location
-    if (mergedRegionTmpDir != null && fs.exists(mergedRegionTmpDir)) {
-      if (!fs.rename(mergedRegionTmpDir, regionDir)) {
-        throw new IOException("Unable to rename " + mergedRegionTmpDir + " to "
-            + regionDir);
-      }
-    }
-  }
-
-  // ===========================================================================
-  //  Create/Open/Delete Helpers
-  // ===========================================================================
-  /**
-   * Log the current state of the region
-   * @param LOG log to output information
-   * @throws IOException if an unexpected exception occurs
-   */
-  void logFileSystemState(final Log LOG) throws IOException {
-    FSUtils.logFileSystemState(fs, this.getRegionDir(), LOG);
-  }
-
-  /**
-   * @param hri
-   * @return Content of the file we write out to the filesystem under a region
-   * @throws IOException
-   */
-  private static byte[] getRegionInfoFileContent(final HRegionInfo hri) throws IOException {
-    return hri.toDelimitedByteArray();
-  }
-
-  /**
-   * Create a {@link HRegionInfo} from the serialized version on-disk.
-   * @param fs {@link FileSystem} that contains the Region Info file
-   * @param regionDir {@link Path} to the Region Directory that contains the Info file
-   * @return An {@link HRegionInfo} instance gotten from the Region Info file.
-   * @throws IOException if an error occurred during file open/read operation.
-   */
-  public static HRegionInfo loadRegionInfoFileContent(final FileSystem fs, final Path regionDir)
-      throws IOException {
-    FSDataInputStream in = fs.open(new Path(regionDir, REGION_INFO_FILE));
-    try {
-      return HRegionInfo.parseFrom(in);
-    } finally {
-      in.close();
-    }
-  }
-
-  /**
-   * Write the .regioninfo file on-disk.
-   */
-  private static void writeRegionInfoFileContent(final Configuration conf, final FileSystem fs,
-      final Path regionInfoFile, final byte[] content) throws IOException {
-    // First check to get the permissions
-    FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
-    // Write the RegionInfo file content
-    FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null);
-    try {
-      out.write(content);
-    } finally {
-      out.close();
-    }
-  }
-
-  /**
-   * Write out an info file under the stored region directory. Useful recovering mangled regions.
-   * If the regionInfo already exists on-disk, then we fast exit.
-   */
-  void checkRegionInfoOnFilesystem() throws IOException {
-    // Compose the content of the file so we can compare to length in filesystem. If not same,
-    // rewrite it (it may have been written in the old format using Writables instead of pb). The
-    // pb version is much shorter -- we write now w/o the toString version -- so checking length
-    // only should be sufficient. I don't want to read the file every time to check if it pb
-    // serialized.
-    byte[] content = getRegionInfoFileContent(regionInfoForFs);
-    try {
-      Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
-
-      FileStatus status = fs.getFileStatus(regionInfoFile);
-      if (status != null && status.getLen() == content.length) {
-        // Then assume the content good and move on.
-        // NOTE: that the length is not sufficient to define the the content matches.
-        return;
-      }
-
-      LOG.info("Rewriting .regioninfo file at: " + regionInfoFile);
-      if (!fs.delete(regionInfoFile, false)) {
-        throw new IOException("Unable to remove existing " + regionInfoFile);
-      }
-    } catch (FileNotFoundException e) {
-      LOG.warn(REGION_INFO_FILE + " file not found for region: " + regionInfoForFs.getEncodedName() +
-          " on table " + regionInfo.getTable());
-    }
-
-    // Write HRI to a file in case we need to recover hbase:meta
-    writeRegionInfoOnFilesystem(content, true);
-  }
-
-  /**
-   * Write out an info file under the region directory. Useful recovering mangled regions.
-   * @param useTempDir indicate whether or not using the region .tmp dir for a safer file creation.
-   */
-  private void writeRegionInfoOnFilesystem(boolean useTempDir) throws IOException {
-    byte[] content = getRegionInfoFileContent(regionInfoForFs);
-    writeRegionInfoOnFilesystem(content, useTempDir);
-  }
-
-  /**
-   * Write out an info file under the region directory. Useful recovering mangled regions.
-   * @param regionInfoContent serialized version of the {@link HRegionInfo}
-   * @param useTempDir indicate whether or not using the region .tmp dir for a safer file creation.
-   */
-  private void writeRegionInfoOnFilesystem(final byte[] regionInfoContent,
-      final boolean useTempDir) throws IOException {
-    Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
-    if (useTempDir) {
-      // Create in tmpDir and then move into place in case we crash after
-      // create but before close. If we don't successfully close the file,
-      // subsequent region reopens will fail the below because create is
-      // registered in NN.
-
-      // And then create the file
-      Path tmpPath = new Path(getTempDir(), REGION_INFO_FILE);
-
-      // If datanode crashes or if the RS goes down just before the close is called while trying to
-      // close the created regioninfo file in the .tmp directory then on next
-      // creation we will be getting AlreadyCreatedException.
-      // Hence delete and create the file if exists.
-      if (FSUtils.isExists(fs, tmpPath)) {
-        FSUtils.delete(fs, tmpPath, true);
-      }
-
-      // Write HRI to a file in case we need to recover hbase:meta
-      writeRegionInfoFileContent(conf, fs, tmpPath, regionInfoContent);
-
-      // Move the created file to the original path
-      if (fs.exists(tmpPath) &&  !rename(tmpPath, regionInfoFile)) {
-        throw new IOException("Unable to rename " + tmpPath + " to " + regionInfoFile);
-      }
-    } else {
-      // Write HRI to a file in case we need to recover hbase:meta
-      writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
-    }
-  }
-
-  /**
-   * Create a new Region on file-system.
-   * @param conf the {@link Configuration} to use
-   * @param fs {@link FileSystem} from which to add the region
-   * @param tableDir {@link Path} to where the table is being stored
-   * @param regionInfo {@link HRegionInfo} for region to be added
-   * @throws IOException if the region creation fails due to a FileSystem exception.
-   */
-  public static HRegionFileSystem createRegionOnFileSystem(final Configuration conf,
-      final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
-    HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
-    Path regionDir = regionFs.getRegionDir();
-
-    if (fs.exists(regionDir)) {
-      LOG.warn("Trying to create a region that already exists on disk: " + regionDir);
-      throw new IOException("The specified region already exists on disk: " + regionDir);
-    }
-
-    // Create the region directory
-    if (!createDirOnFileSystem(fs, conf, regionDir)) {
-      LOG.warn("Unable to create the region directory: " + regionDir);
-      throw new IOException("Unable to create region directory: " + regionDir);
-    }
-
-    // Write HRI to a file in case we need to recover hbase:meta
-    regionFs.writeRegionInfoOnFilesystem(false);
-    return regionFs;
-  }
-
-  /**
-   * Open Region from file-system.
-   * @param conf the {@link Configuration} to use
-   * @param fs {@link FileSystem} from which to add the region
-   * @param tableDir {@link Path} to where the table is being stored
-   * @param regionInfo {@link HRegionInfo} for region to be added
-   * @param readOnly True if you don't want to edit the region data
-   * @throws IOException if the region creation fails due to a FileSystem exception.
-   */
-  public static HRegionFileSystem openRegionFromFileSystem(final Configuration conf,
-      final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo, boolean readOnly)
-      throws IOException {
-    HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
-    Path regionDir = regionFs.getRegionDir();
-
-    if (!fs.exists(regionDir)) {
-      LOG.warn("Trying to open a region that do not exists on disk: " + regionDir);
-      throw new IOException("The specified region do not exists on disk: " + regionDir);
-    }
-
-    if (!readOnly) {
-      // Cleanup temporary directories
-      regionFs.cleanupTempDir();
-      regionFs.cleanupSplitsDir();
-      regionFs.cleanupMergesDir();
-
-      // if it doesn't exists, Write HRI to a file, in case we need to recover hbase:meta
-      regionFs.checkRegionInfoOnFilesystem();
-    }
-
-    return regionFs;
-  }
-
-  /**
-   * Remove the region from the table directory, archiving the region's hfiles.
-   * @param conf the {@link Configuration} to use
-   * @param fs {@link FileSystem} from which to remove the region
-   * @param tableDir {@link Path} to where the table is being stored
-   * @param regionInfo {@link HRegionInfo} for region to be deleted
-   * @throws IOException if the request cannot be completed
-   */
-  public static void deleteRegionFromFileSystem(final Configuration conf,
-      final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
-    HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
-    Path regionDir = regionFs.getRegionDir();
-
-    if (!fs.exists(regionDir)) {
-      LOG.warn("Trying to delete a region that do not exists on disk: " + regionDir);
-      return;
-    }
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("DELETING region " + regionDir);
-    }
-
-    // Archive region
-    Path rootDir = FSUtils.getRootDir(conf);
-    HFileArchiver.archiveRegion(fs, rootDir, tableDir, regionDir);
-
-    // Delete empty region dir
-    if (!fs.delete(regionDir, true)) {
-      LOG.warn("Failed delete of " + regionDir);
-    }
-  }
-
-  /**
-   * Creates a directory. Assumes the user has already checked for this directory existence.
-   * @param dir
-   * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks
-   *         whether the directory exists or not, and returns true if it exists.
-   * @throws IOException
-   */
-  boolean createDir(Path dir) throws IOException {
-    int i = 0;
-    IOException lastIOE = null;
-    do {
-      try {
-        return fs.mkdirs(dir);
-      } catch (IOException ioe) {
-        lastIOE = ioe;
-        if (fs.exists(dir)) return true; // directory is present
-        try {
-          sleepBeforeRetry("Create Directory", i+1);
-        } catch (InterruptedException e) {
-          throw (InterruptedIOException)new InterruptedIOException().initCause(e);
-        }
-      }
-    } while (++i <= hdfsClientRetriesNumber);
-    throw new IOException("Exception in createDir", lastIOE);
-  }
-
-  /**
-   * Renames a directory. Assumes the user has already checked for this directory existence.
-   * @param srcpath
-   * @param dstPath
-   * @return true if rename is successful.
-   * @throws IOException
-   */
-  boolean rename(Path srcpath, Path dstPath) throws IOException {
-    IOException lastIOE = null;
-    int i = 0;
-    do {
-      try {
-        return fs.rename(srcpath, dstPath);
-      } catch (IOException ioe) {
-        lastIOE = ioe;
-        if (!fs.exists(srcpath) && fs.exists(dstPath)) return true; // successful move
-        // dir is not there, retry after some time.
-        try {
-          sleepBeforeRetry("Rename Directory", i+1);
-        } catch (InterruptedException e) {
-          throw (InterruptedIOException)new InterruptedIOException().initCause(e);
-        }
-      }
-    } while (++i <= hdfsClientRetriesNumber);
-
-    throw new IOException("Exception in rename", lastIOE);
-  }
-
-  /**
-   * Deletes a directory. Assumes the user has already checked for this directory existence.
-   * @param dir
-   * @return true if the directory is deleted.
-   * @throws IOException
-   */
-  boolean deleteDir(Path dir) throws IOException {
-    IOException lastIOE = null;
-    int i = 0;
-    do {
-      try {
-        return fs.delete(dir, true);
-      } catch (IOException ioe) {
-        lastIOE = ioe;
-        if (!fs.exists(dir)) return true;
-        // dir is there, retry deleting after some time.
-        try {
-          sleepBeforeRetry("Delete Directory", i+1);
-        } catch (InterruptedException e) {
-          throw (InterruptedIOException)new InterruptedIOException().initCause(e);
-        }
-      }
-    } while (++i <= hdfsClientRetriesNumber);
-
-    throw new IOException("Exception in DeleteDir", lastIOE);
-  }
-
-  /**
-   * sleeping logic; handles the interrupt exception.
-   */
-  private void sleepBeforeRetry(String msg, int sleepMultiplier) throws InterruptedException {
-    sleepBeforeRetry(msg, sleepMultiplier, baseSleepBeforeRetries, hdfsClientRetriesNumber);
-  }
-
-  /**
-   * Creates a directory for a filesystem and configuration object. Assumes the user has already
-   * checked for this directory existence.
-   * @param fs
-   * @param conf
-   * @param dir
-   * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks
-   *         whether the directory exists or not, and returns true if it exists.
-   * @throws IOException
-   */
-  private static boolean createDirOnFileSystem(FileSystem fs, Configuration conf, Path dir)
-      throws IOException {
-    int i = 0;
-    IOException lastIOE = null;
-    int hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
-      DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
-    int baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
-      DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
-    do {
-      try {
-        return fs.mkdirs(dir);
-      } catch (IOException ioe) {
-        lastIOE = ioe;
-        if (fs.exists(dir)) return true; // directory is present
-        try {
-          sleepBeforeRetry("Create Directory", i+1, baseSleepBeforeRetries, hdfsClientRetriesNumber);
-        } catch (InterruptedException e) {
-          throw (InterruptedIOException)new InterruptedIOException().initCause(e);
-        }
-      }
-    } while (++i <= hdfsClientRetriesNumber);
-
-    throw new IOException("Exception in createDir", lastIOE);
-  }
-
-  /**
-   * sleeping logic for static methods; handles the interrupt exception. Keeping a static version
-   * for this to avoid re-looking for the integer values.
-   */
-  private static void sleepBeforeRetry(String msg, int sleepMultiplier, int baseSleepBeforeRetries,
-      int hdfsClientRetriesNumber) throws InterruptedException {
-    if (sleepMultiplier > hdfsClientRetriesNumber) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug(msg + ", retries exhausted");
-      }
-      return;
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
-    }
-    Thread.sleep((long)baseSleepBeforeRetries * sleepMultiplier);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index c4bd849..e3b1a85 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.hbase.TagUtil;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.conf.ConfigurationManager;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.io.compress.Compression;
 import org.apache.hadoop.hbase.io.crypto.Encryption;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -127,7 +128,7 @@ public class HStore implements Store {
   // This stores directory in the filesystem.
   protected final HRegion region;
   private final HColumnDescriptor family;
-  private final HRegionFileSystem fs;
+  private final RegionFileSystem fs;
   protected Configuration conf;
   protected CacheConfig cacheConf;
   private long lastCompactSize = 0;
@@ -335,7 +336,7 @@ public class HStore implements Store {
     return this.fs.getFileSystem();
   }
 
-  public HRegionFileSystem getRegionFileSystem() {
+  public RegionFileSystem getRegionFileSystem() {
     return this.fs;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransaction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransaction.java
index 4a3f52f..3baa4d9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransaction.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransaction.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.hbase.security.User;
  * transaction, {@link #execute(Server, RegionServerServices)} to run the
  * transaction and {@link #rollback(Server, RegionServerServices)} to cleanup if
  * execute fails.
- * 
+ *
  * <p>Here is an example of how you would use this interface:
  * <pre>
  *  RegionMergeTransactionFactory factory = new RegionMergeTransactionFactory(conf);
@@ -158,7 +158,7 @@ public interface RegionMergeTransaction {
    * @param services
    * @return <code>true</code> if the regions are mergeable else
    *         <code>false</code> if they are not (e.g. its already closed, etc.).
-   * @throws IOException 
+   * @throws IOException
    */
   boolean prepare(RegionServerServices services) throws IOException;
 
@@ -217,7 +217,7 @@ public interface RegionMergeTransaction {
   /**
    * Register a listener for transaction preparation, execution, and possibly
    * rollback phases.
-   * <p>A listener can abort a transaction by throwing an exception. 
+   * <p>A listener can abort a transaction by throwing an exception.
    * @param listener the listener
    * @return 'this' for chaining
    */

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java
index 9e7f97b..61c3eda 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
 import org.apache.hadoop.hbase.regionserver.SplitTransactionImpl.LoggingProgressable;
 import org.apache.hadoop.hbase.security.User;
@@ -200,7 +201,7 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction {
           region_a.getRegionInfo().getRegionName());
       if (regionAHasMergeQualifier ||
           hasMergeQualifierInMeta(services, region_b.getRegionInfo().getRegionName())) {
-        LOG.debug("Region " + (regionAHasMergeQualifier ? 
+        LOG.debug("Region " + (regionAHasMergeQualifier ?
             region_a.getRegionInfo().getRegionNameAsString()
                 : region_b.getRegionInfo().getRegionNameAsString())
             + " is not mergeable because it has merge qualifier in META");
@@ -556,23 +557,19 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction {
       Map<byte[], List<StoreFile>> hstoreFilesOfRegionB)
       throws IOException {
     // Create reference file(s) of region A in mergdir
-    HRegionFileSystem fs_a = this.region_a.getRegionFileSystem();
-    for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionA
-        .entrySet()) {
+    RegionFileSystem fs_a = this.region_a.getRegionFileSystem();
+    for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionA.entrySet()) {
       String familyName = Bytes.toString(entry.getKey());
       for (StoreFile storeFile : entry.getValue()) {
-        fs_a.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile,
-            this.mergesdir);
+        fs_a.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile, this.mergesdir);
       }
     }
     // Create reference file(s) of region B in mergedir
-    HRegionFileSystem fs_b = this.region_b.getRegionFileSystem();
-    for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionB
-        .entrySet()) {
+    RegionFileSystem fs_b = this.region_b.getRegionFileSystem();
+    for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionB.entrySet()) {
       String familyName = Bytes.toString(entry.getKey());
       for (StoreFile storeFile : entry.getValue()) {
-        fs_b.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile,
-            this.mergesdir);
+        fs_b.mergeStoreFile(this.mergedRegionInfo, familyName, storeFile, this.mergesdir);
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
index 21c117f..e4aee69 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -662,7 +663,7 @@ public class SplitTransactionImpl implements SplitTransaction {
         LOG.debug("Splitting started for store file: " + sf.getPath() + " for region: " +
                   this.parent);
     }
-    HRegionFileSystem fs = this.parent.getRegionFileSystem();
+    RegionFileSystem fs = this.parent.getRegionFileSystem();
     String familyName = Bytes.toString(family);
     Path path_a =
         fs.splitStoreFile(this.hri_a, familyName, sf, this.splitrow, false,

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index a8542de..4cde73d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.MetaTableAccessor;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.mob.MobUtils;
@@ -55,7 +56,6 @@ import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -778,9 +778,9 @@ public class RestoreSnapshotHelper {
     FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
     if (regionDirs == null) return null;
 
-    List<HRegionInfo> regions = new ArrayList<HRegionInfo>(regionDirs.length);
-    for (int i = 0; i < regionDirs.length; ++i) {
-      HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath());
+    List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
+    for (FileStatus regionDir: regionDirs) {
+      HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath());
       regions.add(hri);
     }
     LOG.debug("found " + regions.size() + " regions for table=" +

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
index 0409441..dfb706b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
 import org.apache.hadoop.hbase.mob.MobUtils;
@@ -51,7 +52,6 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescriptio
 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDataManifest;
 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
@@ -250,8 +250,8 @@ public final class SnapshotManifest {
     boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
     try {
       // Open the RegionFS
-      HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
-            tableDir, regionInfo, true);
+      Path rootDir = null;
+      RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, rootDir, regionInfo, false);
       monitor.rethrowException();
 
       // 1. dump region meta info into the snapshot directory

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
index a5afb91..2c5d76a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.util.ByteStringer;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -62,8 +62,7 @@ public final class SnapshotManifestV1 {
   private SnapshotManifestV1() {
   }
 
-  static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
-                                                          HRegionFileSystem, Path> {
+  static class ManifestBuilder implements SnapshotManifest.RegionVisitor<RegionFileSystem, Path> {
     private final Configuration conf;
     private final Path snapshotDir;
     private final FileSystem fs;
@@ -74,24 +73,24 @@ public final class SnapshotManifestV1 {
       this.fs = fs;
     }
 
-    public HRegionFileSystem regionOpen(final HRegionInfo regionInfo) throws IOException {
-      HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf,
-        fs, snapshotDir, regionInfo);
+    public RegionFileSystem regionOpen(final HRegionInfo regionInfo) throws IOException {
+      RegionFileSystem snapshotRegionFs = RegionFileSystem.open(conf, fs,
+          snapshotDir, regionInfo, true);
       return snapshotRegionFs;
     }
 
-    public void regionClose(final HRegionFileSystem region) {
+    public void regionClose(final RegionFileSystem region) {
     }
 
-    public Path familyOpen(final HRegionFileSystem snapshotRegionFs, final byte[] familyName) {
+    public Path familyOpen(final RegionFileSystem snapshotRegionFs, final byte[] familyName) {
       Path familyDir = snapshotRegionFs.getStoreDir(Bytes.toString(familyName));
       return familyDir;
     }
 
-    public void familyClose(final HRegionFileSystem region, final Path family) {
+    public void familyClose(final RegionFileSystem region, final Path family) {
     }
 
-    public void storeFile(final HRegionFileSystem region, final Path familyDir,
+    public void storeFile(final RegionFileSystem region, final Path familyDir,
         final StoreFileInfo storeFile) throws IOException {
       Path referenceFile = new Path(familyDir, storeFile.getPath().getName());
       boolean success = true;
@@ -126,7 +125,7 @@ public final class SnapshotManifestV1 {
       completionService.submit(new Callable<SnapshotRegionManifest>() {
         @Override
         public SnapshotRegionManifest call() throws IOException {
-          HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, region.getPath());
+          HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, region.getPath());
           return buildManifestFromDisk(conf, fs, snapshotDir, hri);
         }
       });
@@ -156,8 +155,7 @@ public final class SnapshotManifestV1 {
 
   static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf,
       final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
-    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
-          tableDir, regionInfo, true);
+    RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, tableDir, regionInfo, true);
     SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
 
     // 1. dump region meta info into the snapshot directory

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index e1a3fa2..2f34291 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -111,7 +111,9 @@ import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableState;
 import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.legacy.LegacyLayout;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -120,7 +122,6 @@ import org.apache.hadoop.hbase.master.RegionState;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
@@ -236,6 +237,8 @@ public class HBaseFsck extends Configured implements Closeable {
   // successful
   private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
 
+  private final MasterFileSystem mfs;
+
   /***********
    * Options
    ***********/
@@ -348,8 +351,18 @@ public class HBaseFsck extends Configured implements Closeable {
   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
       ZooKeeperConnectionException, IOException, ClassNotFoundException {
     super(conf);
+    // make a copy, just to be sure we're not overriding someone else's config
+    setConf(HBaseConfiguration.create(getConf()));
+    // disable blockcache for tool invocation, see HBASE-10500
+    getConf().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
+    // Disable usage of meta replicas in hbck
+    getConf().setBoolean(HConstants.USE_META_REPLICAS, false);
+
+    mfs = MasterFileSystem.open(getConf(), false);
+
     errors = getErrorReporter(getConf());
     this.executor = exec;
+
     lockFileRetryCounterFactory = new RetryCounterFactory(
       getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
       getConf().getInt(
@@ -1075,7 +1088,6 @@ public class HBaseFsck extends Configured implements Closeable {
   private void offlineReferenceFileRepair() throws IOException {
     clearState();
     LOG.info("Validating mapping using HDFS state");
-    final MasterFileSystem mfs = MasterFileSystem.open(getConf(), false);
     mfs.visitStoreFiles(new StoreFileVisitor() {
       @Override
       public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
@@ -1179,7 +1191,7 @@ public class HBaseFsck extends Configured implements Closeable {
     }
 
     FileSystem fs = FileSystem.get(getConf());
-    HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
+    HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
     LOG.debug("HRegionInfo read: " + hri.toString());
     hbi.hdfsEntry.hri = hri;
   }
@@ -1412,7 +1424,7 @@ public class HBaseFsck extends Configured implements Closeable {
         Collections.<WALActionsListener>singletonList(new MetricsWAL()),
         "hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8))).
         getWAL(metaHRI.getEncodedNameAsBytes(), metaHRI.getTable().getNamespace());
-    HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal);
+    HRegion meta = HRegion.createHRegion(c, rootdir, metaDescriptor, metaHRI, wal);
     MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, true);
     return meta;
   }
@@ -2349,8 +2361,7 @@ public class HBaseFsck extends Configured implements Closeable {
                   LOG.warn(hri + " start and stop keys are in the range of " + region
                       + ". The region might not be cleaned up from hdfs when region " + region
                       + " split failed. Hence deleting from hdfs.");
-                  HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
-                    regionDir.getParent(), hri);
+                  RegionFileSystem.destroy(getConf(), fs, regionDir.getParent(), hri);
                   return;
                 }
               }
@@ -2612,7 +2623,7 @@ public class HBaseFsck extends Configured implements Closeable {
       Path src = cf.getPath();
       Path dst =  new Path(targetRegionDir, src.getName());
 
-      if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
+      if (src.getName().equals(LegacyLayout.REGION_INFO_FILE)) {
         // do not copy the old .regioninfo file.
         continue;
       }
@@ -4135,7 +4146,7 @@ public class HBaseFsck extends Configured implements Closeable {
               try {
                 LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
 
-                Path regioninfoFile = new Path(regionDir.getPath(), HRegionFileSystem.REGION_INFO_FILE);
+                Path regioninfoFile = new Path(he.hdfsRegionDir, LegacyLayout.REGION_INFO_FILE);
                 boolean regioninfoFileExists = fs.exists(regioninfoFile);
 
                 if (!regioninfoFileExists) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
index eaf8d54..a07fed9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
@@ -191,7 +191,7 @@ public class HBaseFsckRepair {
       HRegionInfo hri, HTableDescriptor htd) throws IOException {
     // Create HRegion
     Path root = FSUtils.getRootDir(conf);
-    HRegion region = HRegion.createHRegion(hri, root, conf, htd, null);
+    HRegion region = HRegion.createHRegion(conf, root, htd, hri, null);
 
     // Close the new region to flush to disk. Close log file too.
     region.close();

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
index a936fc2..4fe39ff 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
@@ -170,7 +170,7 @@ public abstract class ModifyRegionUtils {
     // unless I pass along via the conf.
     Configuration confForWAL = new Configuration(conf);
     confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
-    HRegion region = HRegion.createHRegion(newRegion, rootDir, conf, hTableDescriptor, null, false);
+    HRegion region = HRegion.createHRegion(conf, rootDir, hTableDescriptor, newRegion, null, false);
     try {
       // 2. Custom user code to interact with the created region
       if (task != null) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
index a6f70c3..cbc57dc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
@@ -61,7 +61,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.NoServerForRegionException;
 import org.apache.hadoop.hbase.client.RegionLocator;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
@@ -412,7 +412,7 @@ public class RegionSplitter {
    * Alternative getCurrentNrHRS which is no longer available.
    * @param connection
    * @return Rough count of regionservers out on cluster.
-   * @throws IOException 
+   * @throws IOException
    */
   private static int getRegionServerCount(final Connection connection) throws IOException {
     try (Admin admin = connection.getAdmin()) {
@@ -727,8 +727,8 @@ public class RegionSplitter {
             if (sk.length == 0)
               sk = splitAlgo.firstRow();
 
-            HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
-                connection.getConfiguration(), fs, tableDir, hri, true);
+            RegionFileSystem regionFs = RegionFileSystem.open(
+              connection.getConfiguration(), fs, tableDir, hri, true);
 
             // Check every Column Family for that region -- check does not have references.
             boolean refFound = false;
@@ -767,7 +767,7 @@ public class RegionSplitter {
    * @param conf
    * @param tableName
    * @return A Pair where first item is table dir and second is the split file.
-   * @throws IOException 
+   * @throws IOException
    */
   private static Pair<Path, Path> getTableDirAndSplitFile(final Configuration conf,
       final TableName tableName)
@@ -785,7 +785,7 @@ public class RegionSplitter {
       getTableDirAndSplitFile(connection.getConfiguration(), tableName);
     Path tableDir = tableDirAndSplitFile.getFirst();
     Path splitFile = tableDirAndSplitFile.getSecond();
- 
+
     FileSystem fs = tableDir.getFileSystem(connection.getConfiguration());
 
     // Using strings because (new byte[]{0}).equals(new byte[]{0}) == false

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 829661c..9af8427 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1826,7 +1826,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
    */
   public HRegion createLocalHRegion(HRegionInfo info, HTableDescriptor desc, WAL wal)
       throws IOException {
-    return HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), desc, wal);
+    return HRegion.createHRegion(getConfiguration(), getDataTestDir(), desc, info, wal);
   }
 
   /**
@@ -1839,7 +1839,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
    * @param families
    * @throws IOException
    * @return A region on which you must call
-             {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when done.
+   *         {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when done.
    * @deprecated use
    * {@link #createLocalHRegion(TableName, byte[], byte[], boolean, Durability, WAL, byte[]...)}
    */
@@ -2358,7 +2358,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
       final Configuration conf, final HTableDescriptor htd, boolean initialize)
       throws IOException {
     WAL wal = createWal(conf, rootDir, info);
-    return HRegion.createHRegion(info, rootDir, conf, htd, wal, initialize);
+    return HRegion.createHRegion(conf, rootDir, htd, info, wal, initialize);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
index 0217b41..6c2c882 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
@@ -93,11 +94,9 @@ public class TestIOFencing {
     volatile CountDownLatch compactionsBlocked = new CountDownLatch(0);
     volatile CountDownLatch compactionsWaiting = new CountDownLatch(0);
 
-    @SuppressWarnings("deprecation")
-    public CompactionBlockerRegion(Path tableDir, WAL log,
-        FileSystem fs, Configuration confParam, HRegionInfo info,
-        HTableDescriptor htd, RegionServerServices rsServices) {
-      super(tableDir, log, fs, confParam, info, htd, rsServices);
+    public CompactionBlockerRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+        final WAL wal, final RegionServerServices rsServices) {
+      super(rfs, htd, wal, rsServices);
     }
 
     public void stopCompactions() {
@@ -154,11 +153,11 @@ public class TestIOFencing {
    */
   public static class BlockCompactionsInPrepRegion extends CompactionBlockerRegion {
 
-    public BlockCompactionsInPrepRegion(Path tableDir, WAL log,
-        FileSystem fs, Configuration confParam, HRegionInfo info,
-        HTableDescriptor htd, RegionServerServices rsServices) {
-      super(tableDir, log, fs, confParam, info, htd, rsServices);
+    public BlockCompactionsInPrepRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+        final WAL wal, final RegionServerServices rsServices) {
+      super(rfs, htd, wal, rsServices);
     }
+
     @Override
     protected void doRegionCompactionPrep() throws IOException {
       compactionsWaiting.countDown();
@@ -177,11 +176,11 @@ public class TestIOFencing {
    * entry to go the WAL before blocking, but blocks afterwards
    */
   public static class BlockCompactionsInCompletionRegion extends CompactionBlockerRegion {
-    public BlockCompactionsInCompletionRegion(Path tableDir, WAL log,
-        FileSystem fs, Configuration confParam, HRegionInfo info,
-        HTableDescriptor htd, RegionServerServices rsServices) {
-      super(tableDir, log, fs, confParam, info, htd, rsServices);
+    public BlockCompactionsInCompletionRegion(final RegionFileSystem rfs,
+        final HTableDescriptor htd, final WAL wal, final RegionServerServices rsServices) {
+      super(rfs, htd, wal, rsServices);
     }
+
     @Override
     protected HStore instantiateHStore(final HColumnDescriptor family) throws IOException {
       return new BlockCompactionsInCompletionHStore(this, family, this.conf);

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index 53de8a0..a5b0002 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -31,11 +31,14 @@ import org.apache.hadoop.hbase.CategoryBasedTimeout;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.testclassification.ClientTests;
@@ -302,13 +305,14 @@ public class TestRestoreSnapshotFromClient {
 
   private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
     MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
-    Set<String> families = new HashSet<String>();
-    Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
-    for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
-      for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
-        families.add(familyDir.getName());
+    final Set<String> families = new HashSet<String>();
+    mfs.visitStoreFiles(tableName, new StoreFileVisitor() {
+      @Override
+      public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
+          throws IOException {
+        families.add(family);
       }
-    }
+    });
     return families;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
index 02bb0d3..fa4497f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.filter.FilterBase;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -153,7 +154,7 @@ public class TestRegionObserverScannerOpenHook {
     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
     Path path = new Path(DIR + callingMethod);
     WAL wal = HBaseTestingUtility.createWal(conf, path, info);
-    HRegion r = HRegion.createHRegion(info, path, conf, htd, wal);
+    HRegion r = HRegion.createHRegion(conf, path, htd, info, wal);
     // this following piece is a hack. currently a coprocessorHost
     // is secretly loaded at OpenRegionHandler. we don't really
     // start a region server here, so just manually create cphost
@@ -220,11 +221,9 @@ public class TestRegionObserverScannerOpenHook {
   public static class CompactionCompletionNotifyingRegion extends HRegion {
     private static volatile CountDownLatch compactionStateChangeLatch = null;
 
-    @SuppressWarnings("deprecation")
-    public CompactionCompletionNotifyingRegion(Path tableDir, WAL log,
-        FileSystem fs, Configuration confParam, HRegionInfo info,
-        HTableDescriptor htd, RegionServerServices rsServices) {
-      super(tableDir, log, fs, confParam, info, htd, rsServices);
+    public CompactionCompletionNotifyingRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+        final WAL wal, final RegionServerServices rsServices) {
+      super(rfs, htd, wal, rsServices);
     }
 
     public CountDownLatch getCompactionStateChangeLatch() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
index 66d290a..8aa4909 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
@@ -26,10 +26,12 @@ import org.apache.hadoop.hbase.CategoryBasedTimeout;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellScanner;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
@@ -125,26 +127,24 @@ public abstract class TableSnapshotInputFormatTestBase {
 
       testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir);
 
-      Path rootDir = FSUtils.getRootDir(UTIL.getConfiguration());
-      for (Path regionDir : FSUtils.getRegionDirs(fs, FSUtils.getTableDir(rootDir, tableName))) {
-        for (Path storeDir : FSUtils.getFamilyDirs(fs, regionDir)) {
-          for (FileStatus status : fs.listStatus(storeDir)) {
-            System.out.println(status.getPath());
-            if (StoreFileInfo.isValid(status)) {
-              Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(UTIL.getConfiguration(),
-                tableName, regionDir.getName(), storeDir.getName());
-
-              Path path = HFileLink.getBackReferencesDir(storeDir, status.getPath().getName());
-              // assert back references directory is empty
-              assertFalse("There is a back reference in " + path, fs.exists(path));
-
-              path = HFileLink.getBackReferencesDir(archiveStoreDir, status.getPath().getName());
-              // assert back references directory is empty
-              assertFalse("There is a back reference in " + path, fs.exists(path));
-            }
-          }
+      UTIL.getHBaseCluster().getMaster().getMasterFileSystem().visitStoreFiles(tableName,
+          new StoreFileVisitor() {
+        @Override
+        public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
+            throws IOException {
+          Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(UTIL.getConfiguration(),
+            region.getTable(), region.getEncodedName(), family);
+
+          // assert back references directory is empty
+          Path storeDir = storeFile.getPath().getParent();
+          Path path = HFileLink.getBackReferencesDir(storeDir, storeFile.getPath().getName());
+          assertFalse("There is a back reference in " + path, fs.exists(path));
+
+          // assert back references directory is empty
+          path = HFileLink.getBackReferencesDir(archiveStoreDir, storeFile.getPath().getName());
+          assertFalse("There is a back reference in " + path, fs.exists(path));
         }
-      }
+      });
     } finally {
       UTIL.getHBaseAdmin().deleteSnapshot(snapshotName);
       UTIL.deleteTable(tableName);

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index 9b69830..a6fe4ad 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -43,11 +43,13 @@ import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.TableStateManager;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -234,19 +236,20 @@ public class MasterProcedureTestingUtility {
   }
 
   public static void validateColumnFamilyDeletion(final HMaster master, final TableName tableName,
-      final String family) throws IOException {
+      final String deletedFamily) throws IOException {
     // verify htd
     HTableDescriptor htd = master.getTableDescriptors().get(tableName);
     assertTrue(htd != null);
-    assertFalse(htd.hasFamily(family.getBytes()));
+    assertFalse(htd.hasFamily(deletedFamily.getBytes()));
 
     // verify fs
-    final FileSystem fs = master.getMasterFileSystem().getFileSystem();
-    final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
-    for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
-      final Path familyDir = new Path(regionDir, family);
-      assertFalse(family + " family dir should not exist", fs.exists(familyDir));
-    }
+    master.getMasterFileSystem().visitStoreFiles(tableName, new StoreFileVisitor() {
+      @Override
+      public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
+          throws IOException {
+        assertFalse(family + " family dir should not exist", family.equals(deletedFamily));
+      }
+    });
   }
 
   public static void validateColumnFamilyModification(final HMaster master,


[12/18] hbase git commit: HBASE-14439 Move fs stuff out pt2 - region fs

Posted by bu...@apache.org.
HBASE-14439 Move fs stuff out pt2 - region fs


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7546190a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7546190a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7546190a

Branch: refs/heads/hbase-14439
Commit: 7546190a2f950ad99dffc2628b30fc92109be93b
Parents: 419800f
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Fri Oct 9 11:47:27 2015 -0700
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Sep 29 11:07:29 2016 -0500

----------------------------------------------------------------------
 .../apache/hadoop/hbase/HTableDescriptor.java   |  54 +-
 .../hadoop/hbase/backup/HFileArchiver.java      | 690 -----------------
 .../hadoop/hbase/fs/FSUtilsWithRetries.java     | 178 +++++
 .../hadoop/hbase/fs/MasterFileSystem.java       | 102 ++-
 .../hadoop/hbase/fs/RegionFileSystem.java       | 183 +++++
 .../hbase/fs/legacy/HFileArchiveUtil.java       | 156 ++++
 .../hadoop/hbase/fs/legacy/HFileArchiver.java   | 690 +++++++++++++++++
 .../hadoop/hbase/fs/legacy/LegacyArchiver.java  |  29 +
 .../hadoop/hbase/fs/legacy/LegacyLayout.java    |  52 +-
 .../hbase/fs/legacy/LegacyMasterFileSystem.java |  29 +-
 .../hbase/fs/legacy/LegacyRegionFileSystem.java | 758 +++++++++++++++++++
 .../hadoop/hbase/master/RegionStates.java       |   5 +-
 .../hbase/regionserver/StoreFileInfo.java       | 104 ++-
 .../hadoop/hbase/util/FSRegionScanner.java      | 104 +--
 .../org/apache/hadoop/hbase/util/FSUtils.java   | 388 ++--------
 .../org/apache/hadoop/hbase/util/FSVisitor.java | 109 ---
 .../org/apache/hadoop/hbase/util/HBaseFsck.java |  89 ++-
 .../hadoop/hbase/util/HFileArchiveUtil.java     | 156 ----
 .../TestCorruptedRegionStoreFile.java           |  15 +-
 .../TestSplitTransactionOnCluster.java          |  27 +-
 .../hbase/snapshot/SnapshotTestingUtils.java    |   1 -
 .../apache/hadoop/hbase/util/TestFSVisitor.java | 123 ---
 22 files changed, 2384 insertions(+), 1658 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 867dc0d..8897e35 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -1437,23 +1437,47 @@ public class HTableDescriptor implements Comparable<HTableDescriptor> {
       remove(match);
   }
 
-  /**
-   * Returns the {@link Path} object representing the table directory under
-   * path rootdir
-   *
-   * Deprecated use FSUtils.getTableDir() instead.
-   *
-   * @param rootdir qualified path of HBase root directory
-   * @param tableName name of table
-   * @return {@link Path} for table
+  /** Table descriptor for <code>hbase:meta</code> catalog table
+   * Deprecated, use TableDescriptors#get(TableName.META_TABLE) or
+   * Admin#getTableDescriptor(TableName.META_TABLE) instead.
    */
   @Deprecated
-  public static Path getTableDir(Path rootdir, final byte [] tableName) {
-    //This is bad I had to mirror code from FSUTils.getTableDir since
-    //there is no module dependency between hbase-client and hbase-server
-    TableName name = TableName.valueOf(tableName);
-    return new Path(rootdir, new Path(HConstants.BASE_NAMESPACE_DIR,
-              new Path(name.getNamespaceAsString(), new Path(name.getQualifierAsString()))));
+  public static final HTableDescriptor META_TABLEDESC = new HTableDescriptor(
+      TableName.META_TABLE_NAME,
+      new HColumnDescriptor[] {
+          new HColumnDescriptor(HConstants.CATALOG_FAMILY)
+              // Ten is arbitrary number.  Keep versions to help debugging.
+              .setMaxVersions(10)
+              .setInMemory(true)
+              .setBlocksize(8 * 1024)
+              .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+              // Disable blooms for meta.  Needs work.  Seems to mess w/ getClosestOrBefore.
+              .setBloomFilterType(BloomType.NONE)
+              // Enable cache of data blocks in L1 if more than one caching tier deployed:
+              // e.g. if using CombinedBlockCache (BucketCache).
+              .setCacheDataInL1(true),
+          new HColumnDescriptor(HConstants.TABLE_FAMILY)
+              // Ten is arbitrary number.  Keep versions to help debugging.
+              .setMaxVersions(10)
+              .setInMemory(true)
+              .setBlocksize(8 * 1024)
+              .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+                  // Disable blooms for meta.  Needs work.  Seems to mess w/ getClosestOrBefore.
+              .setBloomFilterType(BloomType.NONE)
+                  // Enable cache of data blocks in L1 if more than one caching tier deployed:
+                  // e.g. if using CombinedBlockCache (BucketCache).
+              .setCacheDataInL1(true)
+      });
+
+  static {
+    try {
+      META_TABLEDESC.addCoprocessor(
+          "org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint",
+          null, Coprocessor.PRIORITY_SYSTEM, null);
+    } catch (IOException ex) {
+      //LOG.warn("exception in loading coprocessor for the hbase:meta table");
+      throw new RuntimeException(ex);
+    }
   }
 
   public final static String NAMESPACE_FAMILY_INFO = "info";

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
deleted file mode 100644
index d682ccc..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
+++ /dev/null
@@ -1,690 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.HFileArchiveUtil;
-import org.apache.hadoop.io.MultipleIOException;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Lists;
-
-/**
- * Utility class to handle the removal of HFiles (or the respective {@link StoreFile StoreFiles})
- * for a HRegion from the {@link FileSystem}. The hfiles will be archived or deleted, depending on
- * the state of the system.
- */
-public class HFileArchiver {
-  private static final Log LOG = LogFactory.getLog(HFileArchiver.class);
-  private static final String SEPARATOR = ".";
-
-  /** Number of retries in case of fs operation failure */
-  private static final int DEFAULT_RETRIES_NUMBER = 3;
-
-  private HFileArchiver() {
-    // hidden ctor since this is just a util
-  }
-
-  /**
-   * Cleans up all the files for a HRegion by archiving the HFiles to the
-   * archive directory
-   * @param conf the configuration to use
-   * @param fs the file system object
-   * @param info HRegionInfo for region to be deleted
-   * @throws IOException
-   */
-  public static void archiveRegion(Configuration conf, FileSystem fs, HRegionInfo info)
-      throws IOException {
-    Path rootDir = FSUtils.getRootDir(conf);
-    archiveRegion(fs, rootDir, FSUtils.getTableDir(rootDir, info.getTable()),
-      HRegion.getRegionDir(rootDir, info));
-  }
-
-  /**
-   * Remove an entire region from the table directory via archiving the region's hfiles.
-   * @param fs {@link FileSystem} from which to remove the region
-   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
-   *          the archive path)
-   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
-   * @param regionDir {@link Path} to where a region is being stored (for building the archive path)
-   * @return <tt>true</tt> if the region was sucessfully deleted. <tt>false</tt> if the filesystem
-   *         operations could not complete.
-   * @throws IOException if the request cannot be completed
-   */
-  public static boolean archiveRegion(FileSystem fs, Path rootdir, Path tableDir, Path regionDir)
-      throws IOException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("ARCHIVING " + regionDir.toString());
-    }
-
-    // otherwise, we archive the files
-    // make sure we can archive
-    if (tableDir == null || regionDir == null) {
-      LOG.error("No archive directory could be found because tabledir (" + tableDir
-          + ") or regiondir (" + regionDir + "was null. Deleting files instead.");
-      deleteRegionWithoutArchiving(fs, regionDir);
-      // we should have archived, but failed to. Doesn't matter if we deleted
-      // the archived files correctly or not.
-      return false;
-    }
-
-    // make sure the regiondir lives under the tabledir
-    Preconditions.checkArgument(regionDir.toString().startsWith(tableDir.toString()));
-    Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(rootdir,
-        FSUtils.getTableName(tableDir),
-        regionDir.getName());
-
-    FileStatusConverter getAsFile = new FileStatusConverter(fs);
-    // otherwise, we attempt to archive the store files
-
-    // build collection of just the store directories to archive
-    Collection<File> toArchive = new ArrayList<File>();
-    final PathFilter dirFilter = new FSUtils.DirFilter(fs);
-    PathFilter nonHidden = new PathFilter() {
-      @Override
-      public boolean accept(Path file) {
-        return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
-      }
-    };
-    FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
-    // if there no files, we can just delete the directory and return;
-    if (storeDirs == null) {
-      LOG.debug("Region directory (" + regionDir + ") was empty, just deleting and returning!");
-      return deleteRegionWithoutArchiving(fs, regionDir);
-    }
-
-    // convert the files in the region to a File
-    toArchive.addAll(Lists.transform(Arrays.asList(storeDirs), getAsFile));
-    LOG.debug("Archiving " + toArchive);
-    boolean success = false;
-    try {
-      success = resolveAndArchive(fs, regionArchiveDir, toArchive);
-    } catch (IOException e) {
-      LOG.error("Failed to archive " + toArchive, e);
-      success = false;
-    }
-
-    // if that was successful, then we delete the region
-    if (success) {
-      return deleteRegionWithoutArchiving(fs, regionDir);
-    }
-
-    throw new IOException("Received error when attempting to archive files (" + toArchive
-        + "), cannot delete region directory. ");
-  }
-
-  /**
-   * Remove from the specified region the store files of the specified column family,
-   * either by archiving them or outright deletion
-   * @param fs the filesystem where the store files live
-   * @param conf {@link Configuration} to examine to determine the archive directory
-   * @param parent Parent region hosting the store files
-   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
-   * @param family the family hosting the store files
-   * @throws IOException if the files could not be correctly disposed.
-   */
-  public static void archiveFamily(FileSystem fs, Configuration conf,
-      HRegionInfo parent, Path tableDir, byte[] family) throws IOException {
-    Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family)));
-    FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir);
-    if (storeFiles == null) {
-      LOG.debug("No store files to dispose for region=" + parent.getRegionNameAsString() +
-          ", family=" + Bytes.toString(family));
-      return;
-    }
-
-    FileStatusConverter getAsFile = new FileStatusConverter(fs);
-    Collection<File> toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile);
-    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, tableDir, family);
-
-    // do the actual archive
-    if (!resolveAndArchive(fs, storeArchiveDir, toArchive)) {
-      throw new IOException("Failed to archive/delete all the files for region:"
-          + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family)
-          + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
-    }
-  }
-
-  /**
-   * Remove the store files, either by archiving them or outright deletion
-   * @param conf {@link Configuration} to examine to determine the archive directory
-   * @param fs the filesystem where the store files live
-   * @param regionInfo {@link HRegionInfo} of the region hosting the store files
-   * @param family the family hosting the store files
-   * @param compactedFiles files to be disposed of. No further reading of these files should be
-   *          attempted; otherwise likely to cause an {@link IOException}
-   * @throws IOException if the files could not be correctly disposed.
-   */
-  public static void archiveStoreFiles(Configuration conf, FileSystem fs, HRegionInfo regionInfo,
-      Path tableDir, byte[] family, Collection<StoreFile> compactedFiles) throws IOException {
-
-    // sometimes in testing, we don't have rss, so we need to check for that
-    if (fs == null) {
-      LOG.warn("Passed filesystem is null, so just deleting the files without archiving for region:"
-          + Bytes.toString(regionInfo.getRegionName()) + ", family:" + Bytes.toString(family));
-      deleteStoreFilesWithoutArchiving(compactedFiles);
-      return;
-    }
-
-    // short circuit if we don't have any files to delete
-    if (compactedFiles.size() == 0) {
-      LOG.debug("No store files to dispose, done!");
-      return;
-    }
-
-    // build the archive path
-    if (regionInfo == null || family == null) throw new IOException(
-        "Need to have a region and a family to archive from.");
-
-    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family);
-
-    // make sure we don't archive if we can't and that the archive dir exists
-    if (!fs.mkdirs(storeArchiveDir)) {
-      throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
-          + Bytes.toString(family) + ", deleting compacted files instead.");
-    }
-
-    // otherwise we attempt to archive the store files
-    if (LOG.isDebugEnabled()) LOG.debug("Archiving compacted store files.");
-
-    // Wrap the storefile into a File
-    StoreToFile getStorePath = new StoreToFile(fs);
-    Collection<File> storeFiles = Collections2.transform(compactedFiles, getStorePath);
-
-    // do the actual archive
-    if (!resolveAndArchive(fs, storeArchiveDir, storeFiles)) {
-      throw new IOException("Failed to archive/delete all the files for region:"
-          + Bytes.toString(regionInfo.getRegionName()) + ", family:" + Bytes.toString(family)
-          + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
-    }
-  }
-
-  /**
-   * Archive the store file
-   * @param fs the filesystem where the store files live
-   * @param regionInfo region hosting the store files
-   * @param conf {@link Configuration} to examine to determine the archive directory
-   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
-   * @param family the family hosting the store files
-   * @param storeFile file to be archived
-   * @throws IOException if the files could not be correctly disposed.
-   */
-  public static void archiveStoreFile(Configuration conf, FileSystem fs, HRegionInfo regionInfo,
-      Path tableDir, byte[] family, Path storeFile) throws IOException {
-    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family);
-    // make sure we don't archive if we can't and that the archive dir exists
-    if (!fs.mkdirs(storeArchiveDir)) {
-      throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
-          + Bytes.toString(family) + ", deleting compacted files instead.");
-    }
-
-    // do the actual archive
-    long start = EnvironmentEdgeManager.currentTime();
-    File file = new FileablePath(fs, storeFile);
-    if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) {
-      throw new IOException("Failed to archive/delete the file for region:"
-          + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family)
-          + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
-    }
-  }
-
-  /**
-   * Archive the given files and resolve any conflicts with existing files via appending the time
-   * archiving started (so all conflicts in the same group have the same timestamp appended).
-   * <p>
-   * If any of the passed files to archive are directories, archives all the files under that
-   * directory. Archive directory structure for children is the base archive directory name + the
-   * parent directory and is built recursively is passed files are directories themselves.
-   * @param fs {@link FileSystem} on which to archive the files
-   * @param baseArchiveDir base archive directory to archive the given files
-   * @param toArchive files to be archived
-   * @return <tt>true</tt> on success, <tt>false</tt> otherwise
-   * @throws IOException on unexpected failure
-   */
-  private static boolean resolveAndArchive(FileSystem fs, Path baseArchiveDir,
-      Collection<File> toArchive) throws IOException {
-    if (LOG.isTraceEnabled()) LOG.trace("Starting to archive " + toArchive);
-    long start = EnvironmentEdgeManager.currentTime();
-    List<File> failures = resolveAndArchive(fs, baseArchiveDir, toArchive, start);
-
-    // notify that some files were not archived.
-    // We can't delete the files otherwise snapshots or other backup system
-    // that relies on the archiver end up with data loss.
-    if (failures.size() > 0) {
-      LOG.warn("Failed to complete archive of: " + failures +
-        ". Those files are still in the original location, and they may slow down reads.");
-      return false;
-    }
-    return true;
-  }
-
-  /**
-   * Resolve any conflict with an existing archive file via timestamp-append
-   * renaming of the existing file and then archive the passed in files.
-   * @param fs {@link FileSystem} on which to archive the files
-   * @param baseArchiveDir base archive directory to store the files. If any of
-   *          the files to archive are directories, will append the name of the
-   *          directory to the base archive directory name, creating a parallel
-   *          structure.
-   * @param toArchive files/directories that need to be archvied
-   * @param start time the archiving started - used for resolving archive
-   *          conflicts.
-   * @return the list of failed to archive files.
-   * @throws IOException if an unexpected file operation exception occured
-   */
-  private static List<File> resolveAndArchive(FileSystem fs, Path baseArchiveDir,
-      Collection<File> toArchive, long start) throws IOException {
-    // short circuit if no files to move
-    if (toArchive.size() == 0) return Collections.emptyList();
-
-    if (LOG.isTraceEnabled()) LOG.trace("moving files to the archive directory: " + baseArchiveDir);
-
-    // make sure the archive directory exists
-    if (!fs.exists(baseArchiveDir)) {
-      if (!fs.mkdirs(baseArchiveDir)) {
-        throw new IOException("Failed to create the archive directory:" + baseArchiveDir
-            + ", quitting archive attempt.");
-      }
-      if (LOG.isTraceEnabled()) LOG.trace("Created archive directory:" + baseArchiveDir);
-    }
-
-    List<File> failures = new ArrayList<File>();
-    String startTime = Long.toString(start);
-    for (File file : toArchive) {
-      // if its a file archive it
-      try {
-        if (LOG.isTraceEnabled()) LOG.trace("Archiving: " + file);
-        if (file.isFile()) {
-          // attempt to archive the file
-          if (!resolveAndArchiveFile(baseArchiveDir, file, startTime)) {
-            LOG.warn("Couldn't archive " + file + " into backup directory: " + baseArchiveDir);
-            failures.add(file);
-          }
-        } else {
-          // otherwise its a directory and we need to archive all files
-          if (LOG.isTraceEnabled()) LOG.trace(file + " is a directory, archiving children files");
-          // so we add the directory name to the one base archive
-          Path parentArchiveDir = new Path(baseArchiveDir, file.getName());
-          // and then get all the files from that directory and attempt to
-          // archive those too
-          Collection<File> children = file.getChildren();
-          failures.addAll(resolveAndArchive(fs, parentArchiveDir, children, start));
-        }
-      } catch (IOException e) {
-        LOG.warn("Failed to archive " + file, e);
-        failures.add(file);
-      }
-    }
-    return failures;
-  }
-
-  /**
-   * Attempt to archive the passed in file to the archive directory.
-   * <p>
-   * If the same file already exists in the archive, it is moved to a timestamped directory under
-   * the archive directory and the new file is put in its place.
-   * @param archiveDir {@link Path} to the directory that stores the archives of the hfiles
-   * @param currentFile {@link Path} to the original HFile that will be archived
-   * @param archiveStartTime time the archiving started, to resolve naming conflicts
-   * @return <tt>true</tt> if the file is successfully archived. <tt>false</tt> if there was a
-   *         problem, but the operation still completed.
-   * @throws IOException on failure to complete {@link FileSystem} operations.
-   */
-  private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile,
-      String archiveStartTime) throws IOException {
-    // build path as it should be in the archive
-    String filename = currentFile.getName();
-    Path archiveFile = new Path(archiveDir, filename);
-    FileSystem fs = currentFile.getFileSystem();
-
-    // if the file already exists in the archive, move that one to a timestamped backup. This is a
-    // really, really unlikely situtation, where we get the same name for the existing file, but
-    // is included just for that 1 in trillion chance.
-    if (fs.exists(archiveFile)) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("File:" + archiveFile + " already exists in archive, moving to "
-            + "timestamped backup and overwriting current.");
-      }
-
-      // move the archive file to the stamped backup
-      Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime);
-      if (!fs.rename(archiveFile, backedupArchiveFile)) {
-        LOG.error("Could not rename archive file to backup: " + backedupArchiveFile
-            + ", deleting existing file in favor of newer.");
-        // try to delete the exisiting file, if we can't rename it
-        if (!fs.delete(archiveFile, false)) {
-          throw new IOException("Couldn't delete existing archive file (" + archiveFile
-              + ") or rename it to the backup file (" + backedupArchiveFile
-              + ") to make room for similarly named file.");
-        }
-      }
-      LOG.debug("Backed up archive file from " + archiveFile);
-    }
-
-    if (LOG.isTraceEnabled()) {
-      LOG.trace("No existing file in archive for: " + archiveFile +
-        ", free to archive original file.");
-    }
-
-    // at this point, we should have a free spot for the archive file
-    boolean success = false;
-    for (int i = 0; !success && i < DEFAULT_RETRIES_NUMBER; ++i) {
-      if (i > 0) {
-        // Ensure that the archive directory exists.
-        // The previous "move to archive" operation has failed probably because
-        // the cleaner has removed our archive directory (HBASE-7643).
-        // (we're in a retry loop, so don't worry too much about the exception)
-        try {
-          if (!fs.exists(archiveDir)) {
-            if (fs.mkdirs(archiveDir)) {
-              LOG.debug("Created archive directory:" + archiveDir);
-            }
-          }
-        } catch (IOException e) {
-          LOG.warn("Failed to create directory: " + archiveDir, e);
-        }
-      }
-
-      try {
-        success = currentFile.moveAndClose(archiveFile);
-      } catch (IOException e) {
-        LOG.warn("Failed to archive " + currentFile + " on try #" + i, e);
-        success = false;
-      }
-    }
-
-    if (!success) {
-      LOG.error("Failed to archive " + currentFile);
-      return false;
-    }
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Finished archiving from " + currentFile + ", to " + archiveFile);
-    }
-    return true;
-  }
-
-  /**
-   * Without regard for backup, delete a region. Should be used with caution.
-   * @param regionDir {@link Path} to the region to be deleted.
-   * @param fs FileSystem from which to delete the region
-   * @return <tt>true</tt> on successful deletion, <tt>false</tt> otherwise
-   * @throws IOException on filesystem operation failure
-   */
-  private static boolean deleteRegionWithoutArchiving(FileSystem fs, Path regionDir)
-      throws IOException {
-    if (fs.delete(regionDir, true)) {
-      LOG.debug("Deleted all region files in: " + regionDir);
-      return true;
-    }
-    LOG.debug("Failed to delete region directory:" + regionDir);
-    return false;
-  }
-
-  /**
-   * Just do a simple delete of the given store files
-   * <p>
-   * A best effort is made to delete each of the files, rather than bailing on the first failure.
-   * <p>
-   * This method is preferable to {@link #deleteFilesWithoutArchiving(Collection)} since it consumes
-   * less resources, but is limited in terms of usefulness
-   * @param compactedFiles store files to delete from the file system.
-   * @throws IOException if a file cannot be deleted. All files will be attempted to deleted before
-   *           throwing the exception, rather than failing at the first file.
-   */
-  private static void deleteStoreFilesWithoutArchiving(Collection<StoreFile> compactedFiles)
-      throws IOException {
-    LOG.debug("Deleting store files without archiving.");
-    List<IOException> errors = new ArrayList<IOException>(0);
-    for (StoreFile hsf : compactedFiles) {
-      try {
-        hsf.deleteReader();
-      } catch (IOException e) {
-        LOG.error("Failed to delete store file:" + hsf.getPath());
-        errors.add(e);
-      }
-    }
-    if (errors.size() > 0) {
-      throw MultipleIOException.createIOException(errors);
-    }
-  }
-
-  /**
-   * Adapt a type to match the {@link File} interface, which is used internally for handling
-   * archival/removal of files
-   * @param <T> type to adapt to the {@link File} interface
-   */
-  private static abstract class FileConverter<T> implements Function<T, File> {
-    protected final FileSystem fs;
-
-    public FileConverter(FileSystem fs) {
-      this.fs = fs;
-    }
-  }
-
-  /**
-   * Convert a FileStatus to something we can manage in the archiving
-   */
-  private static class FileStatusConverter extends FileConverter<FileStatus> {
-    public FileStatusConverter(FileSystem fs) {
-      super(fs);
-    }
-
-    @Override
-    public File apply(FileStatus input) {
-      return new FileablePath(fs, input.getPath());
-    }
-  }
-
-  /**
-   * Convert the {@link StoreFile} into something we can manage in the archive
-   * methods
-   */
-  private static class StoreToFile extends FileConverter<StoreFile> {
-    public StoreToFile(FileSystem fs) {
-      super(fs);
-    }
-
-    @Override
-    public File apply(StoreFile input) {
-      return new FileableStoreFile(fs, input);
-    }
-  }
-
-  /**
-   * Wrapper to handle file operations uniformly
-   */
-  private static abstract class File {
-    protected final FileSystem fs;
-
-    public File(FileSystem fs) {
-      this.fs = fs;
-    }
-
-    /**
-     * Delete the file
-     * @throws IOException on failure
-     */
-    abstract void delete() throws IOException;
-
-    /**
-     * Check to see if this is a file or a directory
-     * @return <tt>true</tt> if it is a file, <tt>false</tt> otherwise
-     * @throws IOException on {@link FileSystem} connection error
-     */
-    abstract boolean isFile() throws IOException;
-
-    /**
-     * @return if this is a directory, returns all the children in the
-     *         directory, otherwise returns an empty list
-     * @throws IOException
-     */
-    abstract Collection<File> getChildren() throws IOException;
-
-    /**
-     * close any outside readers of the file
-     * @throws IOException
-     */
-    abstract void close() throws IOException;
-
-    /**
-     * @return the name of the file (not the full fs path, just the individual
-     *         file name)
-     */
-    abstract String getName();
-
-    /**
-     * @return the path to this file
-     */
-    abstract Path getPath();
-
-    /**
-     * Move the file to the given destination
-     * @param dest
-     * @return <tt>true</tt> on success
-     * @throws IOException
-     */
-    public boolean moveAndClose(Path dest) throws IOException {
-      this.close();
-      Path p = this.getPath();
-      return FSUtils.renameAndSetModifyTime(fs, p, dest);
-    }
-
-    /**
-     * @return the {@link FileSystem} on which this file resides
-     */
-    public FileSystem getFileSystem() {
-      return this.fs;
-    }
-
-    @Override
-    public String toString() {
-      return this.getClass() + ", file:" + getPath().toString();
-    }
-  }
-
-  /**
-   * A {@link File} that wraps a simple {@link Path} on a {@link FileSystem}.
-   */
-  private static class FileablePath extends File {
-    private final Path file;
-    private final FileStatusConverter getAsFile;
-
-    public FileablePath(FileSystem fs, Path file) {
-      super(fs);
-      this.file = file;
-      this.getAsFile = new FileStatusConverter(fs);
-    }
-
-    @Override
-    public void delete() throws IOException {
-      if (!fs.delete(file, true)) throw new IOException("Failed to delete:" + this.file);
-    }
-
-    @Override
-    public String getName() {
-      return file.getName();
-    }
-
-    @Override
-    public Collection<File> getChildren() throws IOException {
-      if (fs.isFile(file)) return Collections.emptyList();
-      return Collections2.transform(Arrays.asList(fs.listStatus(file)), getAsFile);
-    }
-
-    @Override
-    public boolean isFile() throws IOException {
-      return fs.isFile(file);
-    }
-
-    @Override
-    public void close() throws IOException {
-      // NOOP - files are implicitly closed on removal
-    }
-
-    @Override
-    Path getPath() {
-      return file;
-    }
-  }
-
-  /**
-   * {@link File} adapter for a {@link StoreFile} living on a {@link FileSystem}
-   * .
-   */
-  private static class FileableStoreFile extends File {
-    StoreFile file;
-
-    public FileableStoreFile(FileSystem fs, StoreFile store) {
-      super(fs);
-      this.file = store;
-    }
-
-    @Override
-    public void delete() throws IOException {
-      file.deleteReader();
-    }
-
-    @Override
-    public String getName() {
-      return file.getPath().getName();
-    }
-
-    @Override
-    public boolean isFile() {
-      return true;
-    }
-
-    @Override
-    public Collection<File> getChildren() throws IOException {
-      // storefiles don't have children
-      return Collections.emptyList();
-    }
-
-    @Override
-    public void close() throws IOException {
-      file.closeReader(true);
-    }
-
-    @Override
-    Path getPath() {
-      return file.getPath();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FSUtilsWithRetries.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FSUtilsWithRetries.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FSUtilsWithRetries.java
new file mode 100644
index 0000000..5dad2fb
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FSUtilsWithRetries.java
@@ -0,0 +1,178 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class FSUtilsWithRetries {
+  private static final Log LOG = LogFactory.getLog(FSUtilsWithRetries.class);
+
+  /**
+   * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
+   * client level.
+   */
+  private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
+  private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
+  private final int hdfsClientRetriesNumber;
+  private final int baseSleepBeforeRetries;
+
+  private final FileSystem fs;
+
+  public FSUtilsWithRetries(final Configuration conf, final FileSystem fs) {
+    this.fs = fs;
+
+    this.hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
+      DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
+    this.baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
+      DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
+  }
+
+  public FileSystem getFileSystem() {
+    return fs;
+  }
+
+  public boolean exists(Path path) throws IOException {
+    IOException lastIOE = null;
+    int i = 0;
+    do {
+      try {
+        return fs.exists(path);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (fs.exists(path)) return true; // directory is present
+        try {
+          sleepBeforeRetry("Check existence", i+1);
+        } catch (InterruptedException e) {
+          throw (InterruptedIOException)new InterruptedIOException().initCause(e);
+        }
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+    throw new IOException("Exception in check existence", lastIOE);
+  }
+
+  /**
+   * Creates a directory. Assumes the user has already checked for this directory existence.
+   * @param dir
+   * @return the result of fs.mkdirs(). In case underlying fs throws an IOException, it checks
+   *         whether the directory exists or not, and returns true if it exists.
+   * @throws IOException
+   */
+  public boolean createDir(Path dir) throws IOException {
+    IOException lastIOE = null;
+    int i = 0;
+    do {
+      try {
+        return fs.mkdirs(dir);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (fs.exists(dir)) return true; // directory is present
+        try {
+          sleepBeforeRetry("Create Directory", i+1);
+        } catch (InterruptedException e) {
+          throw (InterruptedIOException)new InterruptedIOException().initCause(e);
+        }
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+    throw new IOException("Exception in createDir", lastIOE);
+  }
+
+  /**
+   * Renames a directory. Assumes the user has already checked for this directory existence.
+   * @param srcpath
+   * @param dstPath
+   * @return true if rename is successful.
+   * @throws IOException
+   */
+  public boolean rename(Path srcpath, Path dstPath) throws IOException {
+    IOException lastIOE = null;
+    int i = 0;
+    do {
+      try {
+        return fs.rename(srcpath, dstPath);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (!fs.exists(srcpath) && fs.exists(dstPath)) return true; // successful move
+        // dir is not there, retry after some time.
+        try {
+          sleepBeforeRetry("Rename Directory", i+1);
+        } catch (InterruptedException e) {
+          throw (InterruptedIOException)new InterruptedIOException().initCause(e);
+        }
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+
+    throw new IOException("Exception in rename", lastIOE);
+  }
+
+  /**
+   * Deletes a directory. Assumes the user has already checked for this directory existence.
+   * @param dir
+   * @return true if the directory is deleted.
+   * @throws IOException
+   */
+  public boolean deleteDir(Path dir) throws IOException {
+    IOException lastIOE = null;
+    int i = 0;
+    do {
+      try {
+        return fs.delete(dir, true);
+      } catch (IOException ioe) {
+        lastIOE = ioe;
+        if (!fs.exists(dir)) return true;
+        // dir is there, retry deleting after some time.
+        try {
+          sleepBeforeRetry("Delete Directory", i+1);
+        } catch (InterruptedException e) {
+          throw (InterruptedIOException)new InterruptedIOException().initCause(e);
+        }
+      }
+    } while (++i <= hdfsClientRetriesNumber);
+
+    throw new IOException("Exception in DeleteDir", lastIOE);
+  }
+
+  /**
+   * sleeping logic; handles the interrupt exception.
+   */
+  private void sleepBeforeRetry(String msg, int sleepMultiplier) throws InterruptedException {
+    sleepBeforeRetry(msg, sleepMultiplier, baseSleepBeforeRetries, hdfsClientRetriesNumber);
+  }
+
+  /**
+   * sleeping logic for static methods; handles the interrupt exception. Keeping a static version
+   * for this to avoid re-looking for the integer values.
+   */
+  private static void sleepBeforeRetry(String msg, int sleepMultiplier, int baseSleepBeforeRetries,
+      int hdfsClientRetriesNumber) throws InterruptedException {
+    if (sleepMultiplier > hdfsClientRetriesNumber) {
+      LOG.debug(msg + ", retries exhausted");
+      return;
+    }
+    LOG.debug(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
+    Thread.sleep((long)baseSleepBeforeRetries * sleepMultiplier);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
index 68a588e..d4323ac 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.fs.legacy.LegacyMasterFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 
@@ -143,6 +144,69 @@ public abstract class MasterFileSystem {
   }
 
   // ==========================================================================
+  //  PUBLIC Methods - Table Region related
+  // ==========================================================================
+  public void deleteRegion(HRegionInfo regionInfo) throws IOException {
+    RegionFileSystem.destroy(conf, fs, rootDir, regionInfo);
+  }
+
+  public Collection<HRegionInfo> getRegions(TableName tableName) throws IOException {
+    return getRegions(FsContext.DATA, tableName);
+  }
+
+  public abstract Collection<HRegionInfo> getRegions(FsContext ctx, TableName tableName)
+    throws IOException;
+
+  // TODO: Move in HRegionFileSystem
+  public void deleteFamilyFromFS(HRegionInfo regionInfo, byte[] familyName, boolean hasMob)
+      throws IOException {
+    getRegionFileSystem(regionInfo).deleteFamily(Bytes.toString(familyName), hasMob);
+  }
+
+  public RegionFileSystem getRegionFileSystem(HRegionInfo regionInfo) throws IOException {
+    return RegionFileSystem.open(conf, fs, rootDir, regionInfo, false);
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - visitors
+  // ==========================================================================
+  public void visitStoreFiles(StoreFileVisitor visitor)
+      throws IOException {
+    visitStoreFiles(FsContext.DATA, visitor);
+  }
+
+  public void visitStoreFiles(String namespace, StoreFileVisitor visitor)
+      throws IOException {
+    visitStoreFiles(FsContext.DATA, namespace, visitor);
+  }
+
+  public void visitStoreFiles(TableName table, StoreFileVisitor visitor)
+      throws IOException {
+    visitStoreFiles(FsContext.DATA, table, visitor);
+  }
+
+  public void visitStoreFiles(FsContext ctx, StoreFileVisitor visitor)
+      throws IOException {
+    for (String namespace: getNamespaces()) {
+      visitStoreFiles(ctx, namespace, visitor);
+    }
+  }
+
+  public void visitStoreFiles(FsContext ctx, String namespace, StoreFileVisitor visitor)
+      throws IOException {
+    for (TableName tableName: getTables(namespace)) {
+      visitStoreFiles(ctx, tableName, visitor);
+    }
+  }
+
+  public void visitStoreFiles(FsContext ctx, TableName table, StoreFileVisitor visitor)
+      throws IOException {
+    for (HRegionInfo hri: getRegions(ctx, table)) {
+      RegionFileSystem.open(conf, fs, rootDir, hri, false).visitStoreFiles(visitor);
+    }
+  }
+
+  // ==========================================================================
   //  PUBLIC Methods - bootstrap
   // ==========================================================================
   public abstract Path getTempDir();
@@ -228,44 +292,6 @@ public abstract class MasterFileSystem {
     bootstrapMeta();
   }
 
-  // TODO: Move in HRegionFileSystem
-  public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName, boolean hasMob)
-      throws IOException {
-    // archive family store files
-    Path tableDir = FSUtils.getTableDir(getRootDir(), region.getTable());
-    HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
-
-    // delete the family folder
-    Path familyDir = new Path(tableDir,
-      new Path(region.getEncodedName(), Bytes.toString(familyName)));
-    if (fs.delete(familyDir, true) == false) {
-      if (fs.exists(familyDir)) {
-        throw new IOException("Could not delete family "
-            + Bytes.toString(familyName) + " from FileSystem for region "
-            + region.getRegionNameAsString() + "(" + region.getEncodedName()
-            + ")");
-      }
-    }
-
-    // archive and delete mob files
-    if (hasMob) {
-      Path mobTableDir =
-          FSUtils.getTableDir(new Path(getRootDir(), MobConstants.MOB_DIR_NAME), region.getTable());
-      HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(region.getTable());
-      Path mobFamilyDir =
-          new Path(mobTableDir,
-              new Path(mobRegionInfo.getEncodedName(), Bytes.toString(familyName)));
-      // archive mob family store files
-      MobUtils.archiveMobStoreFiles(conf, fs, mobRegionInfo, mobFamilyDir, familyName);
-
-      if (!fs.delete(mobFamilyDir, true)) {
-        throw new IOException("Could not delete mob store files for family "
-            + Bytes.toString(familyName) + " from FileSystem region "
-            + mobRegionInfo.getRegionNameAsString() + "(" + mobRegionInfo.getEncodedName() + ")");
-      }
-    }
-  }
-
   // ==========================================================================
   //  PUBLIC
   // ==========================================================================

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
new file mode 100644
index 0000000..3b94355
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
@@ -0,0 +1,183 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.ClusterId;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.legacy.LegacyRegionFileSystem;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+
+@InterfaceAudience.Private
+public abstract class RegionFileSystem {
+  private static Log LOG = LogFactory.getLog(RegionFileSystem.class);
+
+  private final Configuration conf;
+  private final HRegionInfo hri;
+  private final FileSystem fs;
+  private final Path rootDir;
+
+  protected RegionFileSystem(Configuration conf, FileSystem fs, Path rootDir, HRegionInfo hri) {
+    this.conf = conf;
+    this.rootDir = rootDir;
+    this.hri = hri;
+    this.fs = fs;
+  }
+
+  public Configuration getConfiguration() { return conf; }
+  public FileSystem getFileSystem() { return fs; }
+  public Path getRootDir() { return rootDir; }
+
+  public HRegionInfo getRegionInfo() { return hri; }
+  public TableName getTable() { return getRegionInfo().getTable(); }
+
+  // ==========================================================================
+  //  PUBLIC Interfaces - Visitors
+  // ==========================================================================
+  public interface StoreFileVisitor {
+    void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
+       throws IOException;
+  }
+
+  public void visitStoreFiles(final StoreFileVisitor visitor) throws IOException {
+    for (String familyName: getFamilies()) {
+      for (StoreFileInfo storeFile: getStoreFiles(familyName)) {
+        visitor.storeFile(getRegionInfo(), familyName, storeFile);
+      }
+    }
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - Families Related
+  // ==========================================================================
+
+  /**
+   * @return the set of families present on disk
+   * @throws IOException
+   */
+  public abstract Collection<String> getFamilies() throws IOException;
+
+  public void deleteFamily(byte[] familyName, boolean hasMob) throws IOException {
+    deleteFamily(Bytes.toString(familyName), hasMob);
+  }
+
+  public abstract void deleteFamily(String familyName, boolean hasMob) throws IOException;
+
+  // ==========================================================================
+  //  PUBLIC Methods - Store Files related
+  // ==========================================================================
+
+  /**
+   * Returns the store files available for the family.
+   * This methods performs the filtering based on the valid store files.
+   * @param familyName Column Family Name
+   * @return a set of {@link StoreFileInfo} for the specified family.
+   */
+  public Collection<StoreFileInfo> getStoreFiles(final byte[] familyName) throws IOException {
+    return getStoreFiles(Bytes.toString(familyName));
+  }
+
+  public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
+    return getStoreFiles(familyName, true);
+  }
+
+  /**
+   * Returns the store files available for the family.
+   * This methods performs the filtering based on the valid store files.
+   * @param familyName Column Family Name
+   * @return a set of {@link StoreFileInfo} for the specified family.
+   */
+  public abstract Collection<StoreFileInfo> getStoreFiles(String familyName, boolean validate)
+      throws IOException;
+
+  // ==========================================================================
+  //  PUBLIC bootstrap
+  // ==========================================================================
+  protected abstract void bootstrap() throws IOException;
+  protected abstract void destroy() throws IOException;
+
+  // ==========================================================================
+  //  PUBLIC
+  // ==========================================================================
+  public static RegionFileSystem open(Configuration conf, HRegionInfo regionInfo, boolean bootstrap)
+      throws IOException {
+    return open(conf, FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf),
+        regionInfo, bootstrap);
+  }
+
+  public static RegionFileSystem open(Configuration conf, FileSystem fs, Path rootDir,
+      HRegionInfo regionInfo, boolean bootstrap) throws IOException {
+    // Cover both bases, the old way of setting default fs and the new.
+    // We're supposed to run on 0.20 and 0.21 anyways.
+    fs = rootDir.getFileSystem(conf);
+    FSUtils.setFsDefault(conf, new Path(fs.getUri()));
+    // make sure the fs has the same conf
+    fs.setConf(conf);
+
+    RegionFileSystem rfs = getInstance(conf, fs, rootDir, regionInfo);
+    if (bootstrap) {
+      // TODO: are bootstrap and create two different things?
+      // should switch to bootstrap & read-only 
+      // legacy region wants to recover the .regioninfo :(
+    }
+    return rfs;
+  }
+
+  public static void destroy(Configuration conf, HRegionInfo regionInfo) throws IOException {
+    destroy(conf, FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), regionInfo);
+  }
+
+  public static void destroy(Configuration conf, FileSystem fs,
+      Path rootDir, HRegionInfo regionInfo) throws IOException {
+    getInstance(conf, fs, rootDir, regionInfo).destroy();
+  }
+
+  private static RegionFileSystem getInstance(Configuration conf, FileSystem fs,
+      Path rootDir, HRegionInfo regionInfo) throws IOException {
+    String fsType = conf.get("hbase.fs.layout.type", "legacy").toLowerCase();
+    switch (fsType) {
+      case "legacy":
+        return new LegacyRegionFileSystem(conf, fs, rootDir, regionInfo);
+      default:
+        throw new IOException("Invalid filesystem type " + fsType);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
new file mode 100644
index 0000000..2fbaa92
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HStore;
+
+/**
+ * Helper class for all utilities related to archival/retrieval of HFiles
+ */
+public class HFileArchiveUtil {
+  private HFileArchiveUtil() {
+    // non-external instantiation - util class
+  }
+
+  /**
+   * Get the directory to archive a store directory
+   * @param conf {@link Configuration} to read for the archive directory name
+   * @param tableName table name under which the store currently lives
+   * @param regionName region encoded name under which the store currently lives
+   * @param familyName name of the family in the store
+   * @return {@link Path} to the directory to archive the given store or
+   *         <tt>null</tt> if it should not be archived
+   */
+  public static Path getStoreArchivePath(final Configuration conf,
+                                         final TableName tableName,
+      final String regionName, final String familyName) throws IOException {
+    Path tableArchiveDir = getTableArchivePath(conf, tableName);
+    return HStore.getStoreHomedir(tableArchiveDir, regionName, Bytes.toBytes(familyName));
+  }
+
+  /**
+   * Get the directory to archive a store directory
+   * @param conf {@link Configuration} to read for the archive directory name.
+   * @param region parent region information under which the store currently lives
+   * @param tabledir directory for the table under which the store currently lives
+   * @param family name of the family in the store
+   * @return {@link Path} to the directory to archive the given store or <tt>null</tt> if it should
+   *         not be archived
+   */
+  public static Path getStoreArchivePath(Configuration conf,
+                                         HRegionInfo region,
+                                         Path tabledir,
+      byte[] family) throws IOException {
+    Path rootDir = FSUtils.getRootDir(conf);
+    Path tableArchiveDir = getTableArchivePath(rootDir, region.getTable());
+    return HStore.getStoreHomedir(tableArchiveDir, region, family);
+  }
+
+  /**
+   * Get the archive directory for a given region under the specified table
+   * @param tableName the table name. Cannot be null.
+   * @param regiondir the path to the region directory. Cannot be null.
+   * @return {@link Path} to the directory to archive the given region, or <tt>null</tt> if it
+   *         should not be archived
+   */
+  public static Path getRegionArchiveDir(Path rootDir,
+                                         TableName tableName,
+                                         Path regiondir) {
+    // get the archive directory for a table
+    Path archiveDir = getTableArchivePath(rootDir, tableName);
+
+    // then add on the region path under the archive
+    String encodedRegionName = regiondir.getName();
+    return HRegion.getRegionDir(archiveDir, encodedRegionName);
+  }
+
+  /**
+   * Get the archive directory for a given region under the specified table
+   * @param rootDir {@link Path} to the root directory where hbase files are stored (for building
+   *          the archive path)
+   * @param tableName name of the table to archive. Cannot be null.
+   * @return {@link Path} to the directory to archive the given region, or <tt>null</tt> if it
+   *         should not be archived
+   */
+  public static Path getRegionArchiveDir(Path rootDir,
+                                         TableName tableName, String encodedRegionName) {
+    // get the archive directory for a table
+    Path archiveDir = getTableArchivePath(rootDir, tableName);
+    return HRegion.getRegionDir(archiveDir, encodedRegionName);
+  }
+
+  /**
+   * Get the path to the table archive directory based on the configured archive directory.
+   * <p>
+   * Get the path to the table's archive directory.
+   * <p>
+   * Generally of the form: /hbase/.archive/[tablename]
+   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
+   *          the archive path)
+   * @param tableName Name of the table to be archived. Cannot be null.
+   * @return {@link Path} to the archive directory for the table
+   */
+  public static Path getTableArchivePath(final Path rootdir, final TableName tableName) {
+    return FSUtils.getTableDir(getArchivePath(rootdir), tableName);
+  }
+
+  /**
+   * Get the path to the table archive directory based on the configured archive directory.
+   * <p>
+   * Assumed that the table should already be archived.
+   * @param conf {@link Configuration} to read the archive directory property. Can be null
+   * @param tableName Name of the table to be archived. Cannot be null.
+   * @return {@link Path} to the archive directory for the table
+   */
+  public static Path getTableArchivePath(final Configuration conf,
+                                         final TableName tableName)
+      throws IOException {
+    return FSUtils.getTableDir(getArchivePath(conf), tableName);
+  }
+
+  /**
+   * Get the full path to the archive directory on the configured 
+   * {@link org.apache.hadoop.hbase.fs.MasterFileSystem}
+   * @param conf to look for archive directory name and root directory. Cannot be null. Notes for
+   *          testing: requires a FileSystem root directory to be specified.
+   * @return the full {@link Path} to the archive directory, as defined by the configuration
+   * @throws IOException if an unexpected error occurs
+   */
+  public static Path getArchivePath(Configuration conf) throws IOException {
+    return getArchivePath(FSUtils.getRootDir(conf));
+  }
+
+  /**
+   * Get the full path to the archive directory on the configured 
+   * {@link org.apache.hadoop.hbase.fs.MasterFileSystem}
+   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
+   *          the archive path)
+   * @return the full {@link Path} to the archive directory, as defined by the configuration
+   */
+  private static Path getArchivePath(final Path rootdir) {
+    return new Path(rootdir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+  }
+}


[17/18] hbase git commit: HBASE-14439 API cleanup

Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionStorage.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionStorage.java
new file mode 100644
index 0000000..2785950
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionStorage.java
@@ -0,0 +1,807 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs.legacy;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ipc.RemoteException;
+
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.fs.FSUtilsWithRetries;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.MetaUtils;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.regionserver.*;
+import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSHDFSUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
+
+@InterfaceAudience.Private
+public class LegacyRegionStorage extends RegionStorage<LegacyPathIdentifier> {
+  private static final Log LOG = LogFactory.getLog(LegacyRegionStorage.class);
+
+  private final Path tableDir;
+  private final Path regionDir;
+  private final Path mobDir;
+
+  // regionInfo for interacting with FS (getting encodedName, etc)
+  private final HRegionInfo regionInfoForFs;
+
+  private final FSUtilsWithRetries fsWithRetries;
+
+  public LegacyRegionStorage(Configuration conf, FileSystem fs, LegacyPathIdentifier rootContainer, LegacyPathIdentifier regionContainer) throws IOException {
+    this(conf, fs, rootContainer, loadRegionInfoFileContent(fs, regionContainer.path));
+  }
+
+  public LegacyRegionStorage(Configuration conf, FileSystem fs, LegacyPathIdentifier rootContainer, HRegionInfo hri) {
+    super(conf, fs, rootContainer, hri);
+
+    Path dataDir = LegacyLayout.getDataDir(rootContainer.path);
+    this.tableDir = LegacyLayout.getTableDir(dataDir, hri.getTable());
+    this.regionDir = LegacyLayout.getRegionDir(tableDir, hri);
+    this.mobDir = LegacyLayout.getDataDir(LegacyLayout.getMobDir(rootContainer.path));
+    this.fsWithRetries = new FSUtilsWithRetries(conf, fs);
+
+    this.regionInfoForFs = ServerRegionReplicaUtil.getRegionInfoForFs(hri);
+  }
+
+  @Override
+  public LegacyPathIdentifier getRegionContainer() {
+    return new LegacyPathIdentifier(regionDir);
+  }
+
+  @Override
+  public LegacyPathIdentifier getTableContainer() {
+    return new LegacyPathIdentifier(tableDir);
+  }
+
+  public static HRegionInfo loadRegionInfoFileContent(FileSystem fs, Path regionContainer)
+      throws IOException {
+    FSDataInputStream in = fs.open(new Path(regionContainer, ".regioninfo"));
+    try {
+      return HRegionInfo.parseFrom(in);
+    } finally {
+      in.close();
+    }
+  }
+  // ==========================================================================
+  //  PUBLIC Methods - Families Related
+  // ==========================================================================
+  @Override
+  public Collection<String> getFamilies() throws IOException {
+    FileSystem fs = getFileSystem();
+    FileStatus[] fds = FSUtils.listStatus(fs, regionDir, new FSUtils.FamilyDirFilter(fs));
+    if (fds == null) return Collections.emptyList();
+
+    ArrayList<String> families = new ArrayList<String>(fds.length);
+    for (FileStatus status: fds) {
+      families.add(status.getPath().getName());
+    }
+    return families;
+  }
+
+  @Override
+  public void deleteFamily(String familyName, boolean hasMob) throws IOException {
+    // archive family store files
+    byte[] bFamilyName = Bytes.toBytes(familyName);
+
+    FileSystem fs = getFileSystem();
+    HFileArchiver.archiveFamily(fs, getConfiguration(), getRegionInfo(), tableDir, bFamilyName);
+
+    // delete the family folder
+    HRegionInfo region = getRegionInfo();
+    Path familyDir = new Path(tableDir, new Path(region.getEncodedName(), familyName));
+    if (!fsWithRetries.deleteDir(familyDir)) {
+      throw new IOException("Could not delete family "
+          + familyName + " from FileSystem for region "
+          + region.getRegionNameAsString() + "(" + region.getEncodedName()
+          + ")");
+    }
+
+    // archive and delete mob files
+    if (hasMob) {
+      Path mobTableDir = LegacyLayout.getTableDir(mobDir, getTable());
+      HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(getTable());
+      Path mobRegionDir = LegacyLayout.getRegionDir(mobTableDir, mobRegionInfo);
+      Path mobFamilyDir = LegacyLayout.getFamilyDir(mobRegionDir, familyName);
+      // archive mob family store files
+      MobUtils.archiveMobStoreFiles(getConfiguration(), getFileSystem(),
+          mobRegionInfo, mobFamilyDir, bFamilyName);
+
+      if (!fsWithRetries.deleteDir(mobFamilyDir)) {
+        throw new IOException("Could not delete mob store files for family "
+            + familyName + " from FileSystem region "
+            + mobRegionInfo.getRegionNameAsString() + "(" + mobRegionInfo.getEncodedName() + ")");
+      }
+    }
+  }
+
+  // ===========================================================================
+  //  Temp Helpers
+  // ===========================================================================
+  /** @return {@link Path} to the region's temp directory, used for file creations */
+  @Override
+  public LegacyPathIdentifier getTempContainer() {
+    return new LegacyPathIdentifier(LegacyLayout.getRegionTempDir(regionDir));
+  }
+
+  /**
+   * Clean up any temp detritus that may have been left around from previous operation attempts.
+   */
+  public void cleanupTempContainer() throws IOException {
+    fsWithRetries.deleteDir(getTempContainer().path);
+  }
+
+  // ===========================================================================
+  //  Store/StoreFile Helpers
+  // ===========================================================================
+  /**
+   * Returns the directory path of the specified family
+   * @param familyName Column Family Name
+   * @return {@link Path} to the directory of the specified family
+   */
+  public LegacyPathIdentifier getStoreContainer(final String familyName) {
+    return new LegacyPathIdentifier(LegacyLayout.getFamilyDir(getRegionContainer().path, familyName));
+  }
+
+  /**
+   * Create the store directory for the specified family name
+   * @param familyName Column Family Name
+   * @return {@link Path} to the directory of the specified family
+   * @throws IOException if the directory creation fails.
+   */
+  public LegacyPathIdentifier createStoreContainer(final String familyName) throws IOException {
+    Path storeDir = getStoreContainer(familyName).path;
+    if (!fsWithRetries.createDir(storeDir))
+      throw new IOException("Failed creating "+storeDir);
+    return new LegacyPathIdentifier(storeDir);
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - Store Files related
+  // ==========================================================================
+
+  @Override
+  public Collection<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate)
+      throws IOException {
+    Path familyDir = getStoreContainer(familyName).path;
+    FileStatus[] files = FSUtils.listStatus(getFileSystem(), familyDir);
+    if (files == null) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("No StoreFiles for: " + familyDir);
+      }
+      return null;
+    }
+
+    ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(files.length);
+    for (FileStatus status: files) {
+      if (validate && !StoreFileInfo.isValid(status)) {
+        LOG.warn("Invalid StoreFile: " + status.getPath());
+        continue;
+      }
+      StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(getConfiguration(),
+          getFileSystem(), getRegionInfo(), regionInfoForFs, familyName, status.getPath());
+      storeFiles.add(info);
+
+    }
+    return storeFiles;
+  }
+
+
+  /**
+   * Return Qualified Path of the specified family/file
+   *
+   * @param familyName Column Family Name
+   * @param fileName File Name
+   * @return The qualified Path for the specified family/file
+   */
+  public LegacyPathIdentifier getStoreFileStorageIdentifier(final String familyName, final String fileName) {
+    Path familyDir = getStoreContainer(familyName).path;
+    return new LegacyPathIdentifier(LegacyLayout.getStoreFile(familyDir, fileName).makeQualified(getFileSystem()));
+  }
+
+  /**
+   * Return the store file information of the specified family/file.
+   *
+   * @param familyName Column Family Name
+   * @param fileName File Name
+   * @return The {@link StoreFileInfo} for the specified family/file
+   */
+  public StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
+      throws IOException {
+    Path familyDir = getStoreContainer(familyName).path;
+    return ServerRegionReplicaUtil.getStoreFileInfo(getConfiguration(),
+      getFileSystem(), getRegionInfo(), regionInfoForFs, familyName,
+      LegacyLayout.getStoreFile(familyDir, fileName));
+  }
+
+  @Override
+  public long getStoreFileLen(final StoreFile store) throws IOException {
+    return store.getFileInfo().getFileStatus().getLen();
+  }
+
+  /**
+   * Returns true if the specified family has reference files
+   * @param familyName Column Family Name
+   * @return true if family contains reference files
+   * @throws IOException
+   */
+  public boolean hasReferences(final String familyName) throws IOException {
+    FileStatus[] files = FSUtils.listStatus(getFileSystem(), getStoreContainer(familyName).path);
+    if (files != null) {
+      for(FileStatus stat: files) {
+        if(stat.isDirectory()) {
+          continue;
+        }
+        if(StoreFileInfo.isReference(stat.getPath())) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Check whether region has Reference file
+   * @param htd table desciptor of the region
+   * @return true if region has reference file
+   * @throws IOException
+   */
+  public boolean hasReferences(final HTableDescriptor htd) throws IOException {
+    for (HColumnDescriptor family : htd.getFamilies()) {
+      if (hasReferences(family.getNameAsString())) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+
+  /**
+   * Generate a unique file name, used by createTempName() and commitStoreFile()
+   * @param suffix extra information to append to the generated name
+   * @return Unique file name
+   */
+  private static String generateUniqueName(final String suffix) {
+    String name = UUID.randomUUID().toString().replaceAll("-", "");
+    if (suffix != null) name += suffix;
+    return name;
+  }
+
+  /**
+   * Generate a unique temporary Path. Used in conjuction with commitStoreFile()
+   * to get a safer file creation.
+   * <code>
+   * Path file = fs.createTempName();
+   * ...StoreFile.Writer(file)...
+   * fs.commitStoreFile("family", file);
+   * </code>
+   *
+   * @return Unique {@link Path} of the temporary file
+   */
+  public LegacyPathIdentifier getTempIdentifier() {
+    return new LegacyPathIdentifier(createTempName(null));
+  }
+
+  /**
+   * Generate a unique temporary Path. Used in conjuction with commitStoreFile()
+   * to get a safer file creation.
+   * <code>
+   * Path file = fs.createTempName();
+   * ...StoreFile.Writer(file)...
+   * fs.commitStoreFile("family", file);
+   * </code>
+   *
+   * @param suffix extra information to append to the generated name
+   * @return Unique {@link Path} of the temporary file
+   */
+  public Path createTempName(final String suffix) {
+    return new Path(getTempContainer().path, generateUniqueName(suffix));
+  }
+
+  /**
+   * Move the file from a build/temp location to the main family store directory.
+   * @param familyName Family that will gain the file
+   * @param buildPath {@link Path} to the file to commit.
+   * @return The StoreFile representing the newly committed file
+   * @throws IOException
+   */
+  @Override
+  public StoreFile commitStoreFile(final String familyName, final LegacyPathIdentifier buildPath, final CacheConfig cacheConf,  final BloomType cfBloomType, final RegionCoprocessorHost coprocessorHost) throws IOException {
+    final Path path = commitStoreFile(familyName, buildPath.path, -1, false);
+    return createStoreFileAndReader(path, cacheConf, cfBloomType, coprocessorHost);
+  }
+
+  /**
+   * Move the file from a build/temp location to the main family store directory.
+   * @param familyName Family that will gain the file
+   * @param buildPath {@link Path} to the file to commit.
+   * @param seqNum Sequence Number to append to the file name (less then 0 if no sequence number)
+   * @param generateNewName False if you want to keep the buildPath name
+   * @return The new {@link Path} of the committed file
+   * @throws IOException
+   */
+  private Path commitStoreFile(final String familyName, final Path buildPath,
+      final long seqNum, final boolean generateNewName) throws IOException {
+    Path storeDir = getStoreContainer(familyName).path;
+    if(!fsWithRetries.createDir(storeDir))
+      throw new IOException("Failed creating " + storeDir);
+
+    String name = buildPath.getName();
+    if (generateNewName) {
+      name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_");
+    }
+    Path dstPath = new Path(storeDir, name);
+    if (!fsWithRetries.exists(buildPath)) {
+      throw new FileNotFoundException(buildPath.toString());
+    }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Committing store file " + buildPath + " as " + dstPath);
+    }
+    // buildPath exists, therefore not doing an exists() check.
+    if (!fsWithRetries.rename(buildPath, dstPath)) {
+      throw new IOException("Failed rename of " + buildPath + " to " + dstPath);
+    }
+    return dstPath;
+  }
+
+
+  /**
+   * Moves multiple store files to the relative region's family store directory.
+   * @param storeFiles list of store files divided by family
+   * @throws IOException
+   */
+  public void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException {
+    for (Map.Entry<byte[], List<StoreFile>> es: storeFiles.entrySet()) {
+      String familyName = Bytes.toString(es.getKey());
+      for (StoreFile sf: es.getValue()) {
+        commitStoreFile(familyName, sf.getPath(), -1, false);
+      }
+    }
+  }
+
+  /**
+   * Closes and archives the specified store files from the specified family.
+   * @param familyName Family that contains the store files
+   * @param storeFiles set of store files to remove
+   * @throws IOException if the archiving fails
+   */
+  public void removeStoreFiles(final String familyName, final Collection<StoreFile> storeFiles)
+      throws IOException {
+    HFileArchiver.archiveStoreFiles(getConfiguration(), getFileSystem(), this.regionInfoForFs,
+        this.tableDir, Bytes.toBytes(familyName), storeFiles);
+  }
+
+  /**
+   * Bulk load: Add a specified store file to the specified family.
+   * If the source file is on the same file-system is moved from the
+   * source location to the destination location, otherwise is copied over.
+   *
+   * @param familyName Family that will gain the file
+   * @param srcPath {@link Path} to the file to import
+   * @param seqNum Bulk Load sequence number
+   * @return a StoreFile representation of the bulk loaded file
+   * @throws IOException
+   */
+  @Override
+  public StoreFile bulkLoadStoreFile(final String familyName, LegacyPathIdentifier src, long seqNum, final CacheConfig cacheConf, final BloomType cfBloomType, final RegionCoprocessorHost coprocessorHost)
+      throws IOException {
+    // Copy the file if it's on another filesystem
+    FileSystem fs = getFileSystem();
+    FileSystem srcFs = src.path.getFileSystem(getConfiguration());
+    FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs;
+    Path commitPath = src.path;
+
+    // We can't compare FileSystem instances as equals() includes UGI instance
+    // as part of the comparison and won't work when doing SecureBulkLoad
+    // TODO deal with viewFS
+    if (!FSHDFSUtils.isSameHdfs(getConfiguration(), srcFs, desFs)) {
+      LOG.info("Bulk-load file " + src+ " is on different filesystem than " +
+          "the destination store. Copying file over to destination filesystem.");
+      Path tmpPath = getTempIdentifier().path;
+      FileUtil.copy(srcFs, src.path, fs, tmpPath, false, getConfiguration());
+      LOG.info("Copied " + src + " to temporary path on destination filesystem: " + tmpPath);
+      commitPath = tmpPath;
+    }
+
+    final Path result = commitStoreFile(familyName, commitPath, seqNum, true);
+    return createStoreFileAndReader(result, cacheConf, cfBloomType, coprocessorHost);
+  }
+
+  private StoreFile createStoreFileAndReader(final Path p, final CacheConfig cacheConf,  final BloomType cfBloomType, final RegionCoprocessorHost coprocessorHost) throws IOException {
+    final Configuration conf = getConfiguration();
+    final StoreFileInfo info = new StoreFileInfo(conf, this.getFileSystem(), p);
+    info.setRegionCoprocessorHost(coprocessorHost);
+    StoreFile storeFile = new StoreFile(this.getFileSystem(), info, conf, cacheConf, cfBloomType);
+    StoreFileReader r = storeFile.createReader();
+    HRegionInfo region = getRegionInfo();
+    r.setReplicaStoreFile(region.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID);
+    return storeFile;
+  }
+
+  // ===========================================================================
+  //  Splits Helpers
+  // ===========================================================================
+  /** @return {@link Path} to the temp directory used during split operations */
+  public LegacyPathIdentifier getSplitsContainer() {
+    return new LegacyPathIdentifier(LegacyLayout.getRegionSplitsDir(getRegionContainer().path));
+  }
+
+  public LegacyPathIdentifier getSplitsContainer(final HRegionInfo hri) {
+    return new LegacyPathIdentifier(LegacyLayout.getRegionSplitsDir(getSplitsContainer().path, hri));
+  }
+
+  /**
+   * Clean up any split detritus that may have been left around from previous split attempts.
+   */
+  public void cleanupSplitsContainer() throws IOException {
+    fsWithRetries.deleteDir(getSplitsContainer().path);
+  }
+
+  /**
+   * Clean up any split detritus that may have been left around from previous
+   * split attempts.
+   * Call this method on initial region deploy.
+   * @throws IOException
+   */
+  public void cleanupAnySplitDetritus() throws IOException {
+    Path splitdir = this.getSplitsContainer().path;
+    if (!fsWithRetries.exists(splitdir)) return;
+    // Look at the splitdir.  It could have the encoded names of the daughter
+    // regions we tried to make.  See if the daughter regions actually got made
+    // out under the tabledir.  If here under splitdir still, then the split did
+    // not complete.  Try and do cleanup.  This code WILL NOT catch the case
+    // where we successfully created daughter a but regionserver crashed during
+    // the creation of region b.  In this case, there'll be an orphan daughter
+    // dir in the filesystem.  TOOD: Fix.
+    FileSystem fs = getFileSystem();
+    FileStatus[] daughters = FSUtils.listStatus(fs, splitdir, new FSUtils.DirFilter(fs));
+    if (daughters != null) {
+      for (int i = 0; i < daughters.length; ++i) {
+        Path daughterDir = new Path(this.tableDir, daughters[i].getPath().getName());
+        if (!fsWithRetries.deleteDir(daughterDir)) {
+          throw new IOException("Failed delete of " + daughterDir);
+        }
+      }
+    }
+    cleanupSplitsContainer();
+    LOG.info("Cleaned up old failed split transaction detritus: " + splitdir);
+  }
+
+  /**
+   * Remove daughter region
+   * @param regionInfo daughter {@link HRegionInfo}
+   * @throws IOException
+   */
+  public void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException {
+    Path regionDir = LegacyLayout.getRegionDir(tableDir, regionInfo);
+    if (!fsWithRetries.deleteDir(regionDir)) {
+      throw new IOException("Failed delete of " + regionDir);
+    }
+  }
+
+  /**
+   * Commit a daughter region, moving it from the split temporary directory
+   * to the proper location in the filesystem.
+   *
+   * @param regionInfo                 daughter {@link org.apache.hadoop.hbase.HRegionInfo}
+   * @throws IOException
+   */
+  public LegacyPathIdentifier commitDaughterRegion(final HRegionInfo regionInfo)
+      throws IOException {
+    Path regionDir = LegacyLayout.getRegionDir(tableDir, regionInfo);
+    Path daughterTmpDir = this.getSplitsContainer(regionInfo).path;
+
+    if (fsWithRetries.exists(daughterTmpDir)) {
+
+      // Write HRI to a file in case we need to recover hbase:meta
+      Path regionInfoFile = LegacyLayout.getRegionInfoFile(daughterTmpDir);
+      byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
+      writeRegionInfoFileContent(getConfiguration(), getFileSystem(), regionInfoFile, regionInfoContent);
+
+      // Move the daughter temp dir to the table dir
+      if (!fsWithRetries.rename(daughterTmpDir, regionDir)) {
+        throw new IOException("Unable to rename " + daughterTmpDir + " to " + regionDir);
+      }
+    }
+
+    return new LegacyPathIdentifier(regionDir);
+  }
+
+  /**
+   * Create the region splits directory.
+   */
+  @Override
+  public void createSplitsContainer() throws IOException {
+    createTransientDir(getSplitsContainer().path);
+  }
+
+  void createTransientDir(Path dir) throws IOException {
+    if (fsWithRetries.exists(dir)) {
+      LOG.info("The " + dir + " directory exists.  Hence deleting it to recreate it");
+      if (!fsWithRetries.deleteDir(dir)) {
+        throw new IOException("Failed deletion of " + dir + " before creating them again.");
+      }
+    }
+    // dir doesn't exists now. No need to do an exists() call for it.
+    if (!fsWithRetries.createDir(dir)) {
+      throw new IOException("Failed create of " + dir);
+    }
+  }
+
+  /**
+   * Write out a split reference. Package local so it doesnt leak out of
+   * regionserver.
+   * @param hri {@link HRegionInfo} of the destination
+   * @param familyName Column Family Name
+   * @param f File to split.
+   * @param splitRow Split Row
+   * @param top True if we are referring to the top half of the hfile.
+   * @param splitPolicy
+   * @return Path to created reference.
+   * @throws IOException
+   */
+  public LegacyPathIdentifier splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f,
+      final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy)
+          throws IOException {
+
+    if (splitPolicy == null || !splitPolicy.skipStoreFileRangeCheck(familyName)) {
+      // Check whether the split row lies in the range of the store file
+      // If it is outside the range, return directly.
+      try {
+        if (top) {
+          //check if larger than last key.
+          Cell splitKey = CellUtil.createFirstOnRow(splitRow);
+          Cell lastKey = f.getLastKey();
+          // If lastKey is null means storefile is empty.
+          if (lastKey == null) {
+            return null;
+          }
+          if (f.getComparator().compare(splitKey, lastKey) > 0) {
+            return null;
+          }
+        } else {
+          //check if smaller than first key
+          Cell splitKey = CellUtil.createLastOnRow(splitRow);
+          Cell firstKey = f.getFirstKey();
+          // If firstKey is null means storefile is empty.
+          if (firstKey == null) {
+            return null;
+          }
+          if (f.getComparator().compare(splitKey, firstKey) < 0) {
+            return null;
+          }
+        }
+      } finally {
+        f.closeReader(true);
+      }
+    }
+
+    Path splitDir = new Path(getSplitsContainer(hri).path, familyName);
+    // A reference to the bottom half of the hsf store file.
+    Reference r =
+      top ? Reference.createTopReference(splitRow): Reference.createBottomReference(splitRow);
+    // Add the referred-to regions name as a dot separated suffix.
+    // See REF_NAME_REGEX regex above.  The referred-to regions name is
+    // up in the path of the passed in <code>f</code> -- parentdir is family,
+    // then the directory above is the region name.
+    String parentRegionName = regionInfoForFs.getEncodedName();
+    // Write reference with same file id only with the other region name as
+    // suffix and into the new region location (under same family).
+    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
+    p = r.write(getFileSystem(), p);
+    return p == null ? null : new LegacyPathIdentifier(p);
+  }
+
+  // ===========================================================================
+  //  Merge Helpers
+  // ===========================================================================
+  /** @return {@link Path} to the temp directory used during merge operations */
+  public LegacyPathIdentifier getMergesContainer() {
+    return new LegacyPathIdentifier(LegacyLayout.getRegionMergesDir(getRegionContainer().path));
+  }
+
+  public LegacyPathIdentifier getMergesContainer(final HRegionInfo hri) {
+    return new LegacyPathIdentifier(LegacyLayout.getRegionMergesDir(getMergesContainer().path, hri));
+  }
+
+  /**
+   * Clean up any merge detritus that may have been left around from previous merge attempts.
+   */
+  public void cleanupMergesContainer() throws IOException {
+    fsWithRetries.deleteDir(getMergesContainer().path);
+  }
+
+  /**
+   * Remove merged region
+   * @param mergedRegion {@link HRegionInfo}
+   * @throws IOException
+   */
+  public void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException {
+    Path regionDir = LegacyLayout.getRegionDir(tableDir, mergedRegion);
+    if (fsWithRetries.deleteDir(regionDir)) {
+      throw new IOException("Failed delete of " + regionDir);
+    }
+  }
+
+  /**
+   * Create the region merges directory.
+   * @throws IOException If merges dir already exists or we fail to create it.
+   * @see RegionStorage#cleanupMergesContainer()
+   */
+  @Override
+  public void createMergesContainer() throws IOException {
+    createTransientDir(getMergesContainer().path);
+  }
+
+  /**
+   * Write out a merge reference under the given merges directory. Package local
+   * so it doesnt leak out of regionserver.
+   * @param mergedRegion {@link HRegionInfo} of the merged region
+   * @param familyName Column Family Name
+   * @param f File to create reference.
+   * @param mergedDir
+   * @return Path to created reference.
+   * @throws IOException
+   */
+  @Override
+  public LegacyPathIdentifier mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
+      final StoreFile f, final LegacyPathIdentifier mergedDir)
+      throws IOException {
+    Path referenceDir = new Path(new Path(mergedDir.path,
+        mergedRegion.getEncodedName()), familyName);
+    // A whole reference to the store file.
+    Reference r = Reference.createTopReference(regionInfoForFs.getStartKey());
+    // Add the referred-to regions name as a dot separated suffix.
+    // See REF_NAME_REGEX regex above. The referred-to regions name is
+    // up in the path of the passed in <code>f</code> -- parentdir is family,
+    // then the directory above is the region name.
+    String mergingRegionName = regionInfoForFs.getEncodedName();
+    // Write reference with same file id only with the other region name as
+    // suffix and into the new region location (under same family).
+    Path p = new Path(referenceDir, f.getPath().getName() + "."
+        + mergingRegionName);
+    p = r.write(getFileSystem(), p);
+    return p == null ? null : new LegacyPathIdentifier(p);
+  }
+
+  /**
+   * Commit a merged region, moving it from the merges temporary directory to
+   * the proper location in the filesystem.
+   * @param mergedRegionInfo merged region {@link HRegionInfo}
+   * @throws IOException
+   */
+  @Override
+  public void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException {
+    Path regionDir = new Path(this.tableDir, mergedRegionInfo.getEncodedName());
+    Path mergedRegionTmpDir = this.getMergesContainer(mergedRegionInfo).path;
+    // Move the tmp dir in the expected location
+    if (mergedRegionTmpDir != null && fsWithRetries.exists(mergedRegionTmpDir)) {
+      if (!fsWithRetries.rename(mergedRegionTmpDir, regionDir)) {
+        throw new IOException("Unable to rename " + mergedRegionTmpDir + " to "
+            + regionDir);
+      }
+    }
+  }
+
+  // ===========================================================================
+  //  Create/Open/Delete Helpers
+  // ===========================================================================
+  /**
+   * Log the current state of the region
+   * // TODO refactor to "logStorageState"
+   * @param LOG log to output information
+   * @throws IOException if an unexpected exception occurs
+   */
+  @Override
+  public void logFileSystemState(final Log LOG) throws IOException {
+    FSUtils.logFileSystemState(getFileSystem(), this.getRegionContainer().path, LOG);
+  }
+
+  /**
+   * @param hri
+   * @return Content of the file we write out to the filesystem under a region
+   * @throws IOException
+   */
+  private static byte[] getRegionInfoFileContent(final HRegionInfo hri) throws IOException {
+    return hri.toDelimitedByteArray();
+  }
+
+  /**
+   * Write the .regioninfo file on-disk.
+   */
+  private static void writeRegionInfoFileContent(final Configuration conf, final FileSystem fs,
+      final Path regionInfoFile, final byte[] content) throws IOException {
+    // First check to get the permissions
+    FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
+    // Write the RegionInfo file content
+    FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null);
+    try {
+      out.write(content);
+    } finally {
+      out.close();
+    }
+  }
+
+  // ==========================================================================
+  //  PUBLIC bootstrap
+  // ==========================================================================
+  @Override
+  protected void bootstrap() throws IOException {
+    fsWithRetries.createDir(getRegionContainer().path);
+
+    // Cleanup temporary directories
+    cleanupTempContainer();
+    cleanupSplitsContainer();
+    cleanupMergesContainer();
+
+    // if it doesn't exists, Write HRI to a file, in case we need to recover hbase:meta
+    checkRegionInfoOnFilesystem();
+  }
+
+  public void checkRegionInfoOnFilesystem() throws IOException {
+    writeRegionInfoFileContent(getConfiguration(), getFileSystem(),
+      LegacyLayout.getRegionInfoFile(getRegionContainer().path),
+      getRegionInfoFileContent(getRegionInfo()));
+  }
+
+  @Override
+  protected void destroy() throws IOException {
+    fsWithRetries.deleteDir(regionDir);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index 1005449..afddaf6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
@@ -195,14 +195,10 @@ public class CatalogJanitor extends ScheduledChore {
    */
   boolean cleanMergeRegion(final HRegionInfo mergedRegion,
       final HRegionInfo regionA, final HRegionInfo regionB) throws IOException {
-    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
-    Path rootdir = this.services.getMasterFileSystem().getRootDir();
-    Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());
     HTableDescriptor htd = getTableDescriptor(mergedRegion.getTable());
-    RegionFileSystem regionFs = null;
+    RegionStorage regionFs = null;
     try {
-      regionFs = RegionFileSystem.open(
-          this.services.getConfiguration(), fs, tabledir, mergedRegion, false);
+      regionFs = RegionStorage.open(this.services.getConfiguration(), mergedRegion, false);
     } catch (IOException e) {
       LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
     }
@@ -210,6 +206,8 @@ public class CatalogJanitor extends ScheduledChore {
       LOG.debug("Deleting region " + regionA.getRegionNameAsString() + " and "
           + regionB.getRegionNameAsString()
           + " from fs because merged region no longer holds references");
+      // TODO update HFileArchiver to use RegionStorage
+      FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
       MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
@@ -409,8 +407,7 @@ public class CatalogJanitor extends ScheduledChore {
 
     boolean references = false;
     try {
-      final RegionFileSystem regionFs = RegionFileSystem.open(this.services.getConfiguration(),
-          fs, tabledir, daughter, false);
+      final RegionStorage regionFs = RegionStorage.open(this.services.getConfiguration(), daughter, false);
       final HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());
 
       for (HColumnDescriptor family: parentDescriptor.getFamilies()) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
index 047f1a0..196cb16 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
@@ -284,6 +284,7 @@ public class CreateTableProcedure
           final List<HRegionInfo> newRegions) throws IOException {
         HRegionInfo[] regions = newRegions != null ?
           newRegions.toArray(new HRegionInfo[newRegions.size()]) : null;
+        //TODO this should be RegionStorage
         return ModifyRegionUtils.createRegions(env.getMasterConfiguration(),
             tableRootDir, hTableDescriptor, regions, null);
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ServerProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ServerProtobufUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ServerProtobufUtil.java
new file mode 100644
index 0000000..416ffee
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ServerProtobufUtil.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.protobuf;
+
+// TODO remove unused imports
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.nio.ByteBuffer;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellScanner;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.ClusterId;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.ServerLoad;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.TagUtil;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.CompactionState;
+import org.apache.hadoop.hbase.client.Consistency;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionLoadStats;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotType;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
+import org.apache.hadoop.hbase.client.security.SecurityCapability;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.filter.ByteArrayComparable;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
+import org.apache.hadoop.hbase.io.LimitInputStream;
+import org.apache.hadoop.hbase.io.TimeRange;
+import org.apache.hadoop.hbase.master.RegionState;
+import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
+import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
+import org.apache.hadoop.hbase.protobuf.generated.CellProtos;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Column;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
+import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
+import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.LiveServerInfo;
+import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionInTransition;
+import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
+import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
+import org.apache.hadoop.hbase.protobuf.generated.FSProtos.HBaseVersionFileContent;
+import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ColumnFamilySchema;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema;
+import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
+import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos;
+import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
+import org.apache.hadoop.hbase.util.ByteStringer;
+
+/**
+ * Protobufs utility for server side only.
+ */
+@InterfaceAudience.Private
+public final class ServerProtobufUtil {
+
+  private ServerProtobufUtil() {
+  }
+
+  public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family,
+      List<Path> inputPaths, List<Path> outputPaths, StorageIdentifier storeDir) {
+    return toCompactionDescriptor(info, null, family, inputPaths, outputPaths, storeDir);
+  }
+
+  public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] regionName,
+      byte[] family, List<Path> inputPaths, List<Path> outputPaths, StorageIdentifier storeDir) {
+    // compaction descriptor contains relative paths.
+    // input / output paths are relative to the store dir
+    // store dir is relative to region dir
+    CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder()
+        .setTableName(ByteStringer.wrap(info.getTable().toBytes()))
+        .setEncodedRegionName(ByteStringer.wrap(
+          regionName == null ? info.getEncodedNameAsBytes() : regionName))
+        .setFamilyName(ByteStringer.wrap(family))
+        // TODO need an equivalent to getName as unique name for StorageIdentifier
+        .setStoreHomeDir(((LegacyPathIdentifier)storeDir).path.getName()); //make relative
+    for (Path inputPath : inputPaths) {
+      builder.addCompactionInput(inputPath.getName()); //relative path
+    }
+    for (Path outputPath : outputPaths) {
+      builder.addCompactionOutput(outputPath.getName());
+    }
+    builder.setRegionName(ByteStringer.wrap(info.getRegionName()));
+    return builder.build();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
index 73ed9e9..f071f27 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
@@ -52,7 +52,8 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
@@ -103,6 +104,8 @@ public class CompactionTool extends Configured implements Tool {
     /**
      * Execute the compaction on the specified path.
      *
+     * TODO either retool in terms of region info or remove outright
+     *
      * @param path Directory path on which to run compaction.
      * @param compactOnce Execute just a single step of compaction.
      * @param major Request major compaction.
@@ -112,8 +115,8 @@ public class CompactionTool extends Configured implements Tool {
         Path regionDir = path.getParent();
         Path tableDir = regionDir.getParent();
         HTableDescriptor htd = LegacyTableDescriptor.getTableDescriptorFromFs(fs, tableDir);
-        HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
-        compactStoreFiles(tableDir, htd, hri,
+        final RegionStorage rs = RegionStorage.open(conf, new LegacyPathIdentifier(regionDir), false);
+        compactStoreFiles(tableDir, htd, rs.getRegionInfo(),
             path.getName(), compactOnce, major);
       } else if (isRegionDir(fs, path)) {
         Path tableDir = path.getParent();
@@ -138,9 +141,10 @@ public class CompactionTool extends Configured implements Tool {
     private void compactRegion(final Path tableDir, final HTableDescriptor htd,
         final Path regionDir, final boolean compactOnce, final boolean major)
         throws IOException {
-      HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
+      final RegionStorage rs = RegionStorage.open(conf, new LegacyPathIdentifier(regionDir), false);
+      // todo use RegionStorage to iterate instead of FSUtils
       for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
-        compactStoreFiles(tableDir, htd, hri, familyDir.getName(), compactOnce, major);
+        compactStoreFiles(tableDir, htd, rs.getRegionInfo(), familyDir.getName(), compactOnce, major);
       }
     }
 
@@ -181,7 +185,7 @@ public class CompactionTool extends Configured implements Tool {
     private static HStore getStore(final Configuration conf, final FileSystem fs,
         final Path tableDir, final HTableDescriptor htd, final HRegionInfo hri,
         final String familyName, final Path tempDir) throws IOException {
-      RegionFileSystem regionFs = null;
+      RegionStorage regionFs = null;
       HRegion region = new HRegion(regionFs, htd, null, null);
       return new HStore(region, htd.getFamily(Bytes.toBytes(familyName)), conf);
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index c284082..6d46cba 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -133,7 +133,9 @@ import org.apache.hadoop.hbase.filter.ByteArrayComparable;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.FilterWrapper;
 import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.io.TimeRange;
 import org.apache.hadoop.hbase.io.hfile.HFile;
@@ -288,7 +290,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
   final AtomicLong compactionNumBytesCompacted = new AtomicLong(0L);
 
   private final WAL wal;
-  private final RegionFileSystem fs;
+  private final RegionStorage fs;
   protected final Configuration conf;
   private final Configuration baseConf;
   private final int rowLockWaitDuration;
@@ -621,7 +623,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
       final Configuration confParam, final HRegionInfo regionInfo,
       final HTableDescriptor htd, final RegionServerServices rsServices)
       throws IOException {
-    this(RegionFileSystem.open(confParam, fs, tableDir, regionInfo, false), htd, wal, rsServices);
+    this(RegionStorage.open(confParam, fs, new LegacyPathIdentifier(tableDir), regionInfo, false), htd, wal, rsServices);
   }
 
   /**
@@ -640,12 +642,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
    * @param htd the table descriptor
    * @param rsServices reference to {@link RegionServerServices} or null
    */
-  public HRegion(final RegionFileSystem rfs, final HTableDescriptor htd, final WAL wal,
+  public HRegion(final RegionStorage rfs, final HTableDescriptor htd, final WAL wal,
       final RegionServerServices rsServices) {
     this(rfs, wal, rfs.getConfiguration(), htd, rsServices);
   }
 
-  private HRegion(final RegionFileSystem fs, final WAL wal, final Configuration confParam,
+  private HRegion(final RegionStorage fs, final WAL wal, final Configuration confParam,
       final HTableDescriptor htd, final RegionServerServices rsServices) {
     if (htd == null) {
       throw new IllegalArgumentException("Need table descriptor");
@@ -832,11 +834,22 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     long maxSeqId = initializeStores(reporter, status);
     this.mvcc.advanceTo(maxSeqId);
     if (ServerRegionReplicaUtil.shouldReplayRecoveredEdits(this)) {
-      // Recover any edits if available.
-      maxSeqId = Math.max(maxSeqId,
-        replayRecoveredEditsIfAny(this.fs.getRegionDir(), maxSeqIdInStores, reporter, status));
-      // Make sure mvcc is up to max.
-      this.mvcc.advanceTo(maxSeqId);
+      final StorageIdentifier regionContainer = this.fs.getRegionContainer();
+      /* 
+       * TODO either move wal replay stuff to not rely on details from RegionStorage,
+       *      implement a WALStorage abstraction, or make a "Recovered Edits visitor".
+       */
+      if (regionContainer instanceof LegacyPathIdentifier) {
+        // Recover any edits if available.
+        maxSeqId = Math.max(maxSeqId,
+            replayRecoveredEditsIfAny(((LegacyPathIdentifier)this.fs.getRegionContainer()).path, maxSeqIdInStores,
+            reporter, status));
+        // Make sure mvcc is up to max.
+        this.mvcc.advanceTo(maxSeqId);
+      } else {
+        LOG.debug("Skipping check for recovered edits, because RegionStorage implementation '" + this.fs.getClass() +
+            "' doesn't return Paths for the region container.");
+      }
     }
     this.lastReplayedOpenRegionSeqId = maxSeqId;
 
@@ -847,14 +860,14 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     if (this.writestate.writesEnabled) {
       // Remove temporary data left over from old regions
       status.setStatus("Cleaning up temporary data from old regions");
-      fs.cleanupTempDir();
+      fs.cleanupTempContainer();
 
       status.setStatus("Cleaning up detritus from prior splits");
       // Get rid of any splits or merges that were lost in-progress.  Clean out
       // these directories here on open.  We may be opening a region that was
       // being split but we crashed in the middle of it all.
       fs.cleanupAnySplitDetritus();
-      fs.cleanupMergesDir();
+      fs.cleanupMergesContainer();
     }
 
     // Initialize split policy
@@ -876,8 +889,19 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     // is opened before recovery completes. So we add a safety bumper to avoid new sequence number
     // overlaps used sequence numbers
     if (this.writestate.writesEnabled) {
-      nextSeqid = WALSplitter.writeRegionSequenceIdFile(this.fs.getFileSystem(), this.fs
-          .getRegionDir(), nextSeqid, (this.recovering ? (this.flushPerChanges + 10000000) : 1));
+      final StorageIdentifier regionContainer = this.fs.getRegionContainer();
+      /* 
+       * TODO more WAL replay stuff that needs to get pulled out of the notion of region storage
+       */
+      if (regionContainer instanceof LegacyPathIdentifier) {
+        nextSeqid = WALSplitter.writeRegionSequenceIdFile(this.fs.getFileSystem(),
+            ((LegacyPathIdentifier)regionContainer).path, nextSeqid,
+            (this.recovering ? (this.flushPerChanges + 10000000) : 1));
+      } else {
+        LOG.debug("Skipping region sequence id checkpointing, because RegionStorage implementation '" +
+            this.fs.getClass() + "' doesn't return Paths for the region container.");
+        nextSeqid++;
+      }
     } else {
       nextSeqid++;
     }
@@ -1027,12 +1051,21 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     WALUtil.writeRegionEventMarker(wal, getReplicationScope(), getRegionInfo(), regionEventDesc,
         mvcc);
 
-    // Store SeqId in HDFS when a region closes
-    // checking region folder exists is due to many tests which delete the table folder while a
-    // table is still online
-    if (this.fs.getFileSystem().exists(this.fs.getRegionDir())) {
-      WALSplitter.writeRegionSequenceIdFile(this.fs.getFileSystem(), this.fs.getRegionDir(),
-        mvcc.getReadPoint(), 0);
+    final StorageIdentifier regionContainer = this.fs.getRegionContainer();
+    /*
+     * TODO more WAL stuff to move out of region storage
+     */
+    if (regionContainer instanceof LegacyPathIdentifier) {
+      // Store SeqId in HDFS when a region closes
+      // checking region folder exists is due to many tests which delete the table folder while a
+      // table is still online
+      if (this.fs.getFileSystem().exists(((LegacyPathIdentifier)regionContainer).path)) {
+        WALSplitter.writeRegionSequenceIdFile(this.fs.getFileSystem(), ((LegacyPathIdentifier)regionContainer).path,
+          mvcc.getReadPoint(), 0);
+      }
+    } else {
+      LOG.debug("skipping WAL sequence ID checkpointing because the RegionStorage implementation, '" +
+          this.fs.getClass() + "' doesn't return Paths for the region container.");
     }
   }
 
@@ -1054,6 +1087,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     this.updatesLock.writeLock().unlock();
   }
 
+  /* TODO move these into RegionStorage, make it something like host locality so it generalizes */
+
   @Override
   public HDFSBlocksDistribution getHDFSBlocksDistribution() {
     HDFSBlocksDistribution hdfsBlocksDistribution =
@@ -1101,7 +1136,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
     FileSystem fs = tablePath.getFileSystem(conf);
 
-    RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, tablePath, regionInfo, false);
+    RegionStorage regionFs = RegionStorage.open(conf, regionInfo, false);
     for (HColumnDescriptor family: tableDescriptor.getFamilies()) {
       Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family.getNameAsString());
       if (storeFiles == null) continue;
@@ -1116,6 +1151,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     return hdfsBlocksDistribution;
   }
 
+  /* /end TODO move these into RegionStorage, make it something like host locality so it generalizes */
+
   /**
    * Increase the size of mem store in this region and the size of global mem
    * store
@@ -1707,8 +1744,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     return fs.getFileSystem();
   }
 
-  /** @return the {@link HRegionFileSystem} used by this region */
-  public RegionFileSystem getRegionFileSystem() {
+  /** @return the {@link HRegionStorage} used by this region */
+  public RegionStorage getRegionStorage() {
     return this.fs;
   }
 
@@ -4045,10 +4082,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
       String fakeFamilyName = WALSplitter.getRegionDirRecoveredEditsDir(regiondir).getName();
       Set<StoreFile> fakeStoreFiles = new HashSet<StoreFile>(files.size());
       for (Path file: files) {
-        fakeStoreFiles.add(new StoreFile(getRegionFileSystem().getFileSystem(), file, this.conf,
+        fakeStoreFiles.add(new StoreFile(getRegionStorage().getFileSystem(), file, this.conf,
           null, null));
       }
-      getRegionFileSystem().removeStoreFiles(fakeFamilyName, fakeStoreFiles);
+      getRegionStorage().removeStoreFiles(fakeFamilyName, fakeStoreFiles);
     } else {
       for (Path file: files) {
         if (!fs.delete(file, false)) {
@@ -5515,23 +5552,22 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
           if (bulkLoadListener != null) {
             finalPath = bulkLoadListener.prepareBulkLoad(familyName, path, copyFile);
           }
-          Path commitedStoreFile = store.bulkLoadHFile(finalPath, seqId);
+          final StoreFile commitedStoreFile = store.bulkLoadHFile(finalPath, seqId);
+          final String name = commitedStoreFile.getFileInfo().getPath().getName();
 
           // Note the size of the store file
           try {
-            FileSystem fs = commitedStoreFile.getFileSystem(baseConf);
-            storeFilesSizes.put(commitedStoreFile.getName(), fs.getFileStatus(commitedStoreFile)
-                .getLen());
+            storeFilesSizes.put(name, this.fs.getStoreFileLen(commitedStoreFile));
           } catch (IOException e) {
             LOG.warn("Failed to find the size of hfile " + commitedStoreFile);
-            storeFilesSizes.put(commitedStoreFile.getName(), 0L);
+            storeFilesSizes.put(name, 0L);
           }
 
           if(storeFiles.containsKey(familyName)) {
-            storeFiles.get(familyName).add(commitedStoreFile);
+            storeFiles.get(familyName).add(commitedStoreFile.getFileInfo().getPath());
           } else {
             List<Path> storeFileNames = new ArrayList<Path>();
-            storeFileNames.add(commitedStoreFile);
+            storeFileNames.add(commitedStoreFile.getFileInfo().getPath());
             storeFiles.put(familyName, storeFileNames);
           }
           if (bulkLoadListener != null) {
@@ -6300,14 +6336,14 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
    * @param htd the table descriptor
    * @return the new instance
    */
-  static HRegion newHRegion(Configuration conf, FileSystem fs, Path rootDir,
+  static HRegion newHRegion(Configuration conf,
       HTableDescriptor htd, HRegionInfo regionInfo, WAL wal, RegionServerServices rsServices)
       throws IOException {
-    RegionFileSystem rfs = RegionFileSystem.open(conf, fs, rootDir, regionInfo, false);
+    RegionStorage rfs = RegionStorage.open(conf, regionInfo, false);
     return newHRegion(rfs, htd, wal, rsServices);
   }
 
-  private static HRegion newHRegion(RegionFileSystem rfs, HTableDescriptor htd, WAL wal,
+  private static HRegion newHRegion(RegionStorage rfs, HTableDescriptor htd, WAL wal,
       RegionServerServices rsServices) throws IOException {
     try {
       Configuration conf = rfs.getConfiguration();
@@ -6317,7 +6353,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
           (Class<? extends HRegion>) conf.getClass(HConstants.REGION_IMPL, HRegion.class);
 
       Constructor<? extends HRegion> c =
-          regionClass.getConstructor(RegionFileSystem.class, HTableDescriptor.class,
+          regionClass.getConstructor(RegionStorage.class, HTableDescriptor.class,
               WAL.class, RegionServerServices.class);
 
       return c.newInstance(rfs, htd, wal, rsServices);
@@ -6337,25 +6373,34 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
    * @return new HRegion
    * @throws IOException
    */
-  public static HRegion createHRegion(final Configuration conf, final Path rootDir,
+  public static HRegion createHRegion(final Configuration conf,
         final HTableDescriptor hTableDescriptor, final HRegionInfo info,
         final WAL wal, final boolean initialize) throws IOException {
     LOG.info("creating HRegion " + info.getTable().getNameAsString()
-        + " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
+        + " HTD == " + hTableDescriptor +
         " Table name == " + info.getTable().getNameAsString());
-    FileSystem fs = rootDir.getFileSystem(conf);
-    RegionFileSystem rfs = RegionFileSystem.open(conf, fs, rootDir, info, true);
+    RegionStorage rfs = RegionStorage.open(conf, info, true);
     HRegion region = HRegion.newHRegion(rfs, hTableDescriptor, wal, null);
     if (initialize) region.initialize(null);
     return region;
   }
 
-  public static HRegion createHRegion(final Configuration conf, final Path rootDir,
+  /**
+   * Create an HRegion and initialize it.
+   */
+  public static HRegion createHRegion(final Configuration conf,
       final HTableDescriptor hTableDescriptor, final HRegionInfo info, final WAL wal)
       throws IOException {
-    return createHRegion(conf, rootDir, hTableDescriptor, info, wal, true);
+    return createHRegion(conf, hTableDescriptor, info, wal, true);
   }
 
+  /**
+   * TODO remove after refactoring ModifyRegionUtils to use a RegionStorage impl instead of specifying a different root dir manually.
+   */
+  public static HRegion createHRegion(final Configuration conf, final Path rootDir, final HTableDescriptor htd, final HRegionInfo info) throws IOException {
+    RegionStorage rfs = RegionStorage.open(conf, rootDir.getFileSystem(conf), new LegacyPathIdentifier(rootDir), info, true);
+    return HRegion.newHRegion(rfs, htd, null, null);
+  }
 
   /**
    * Open a Region.
@@ -6377,7 +6422,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
 
   /**
    * Open a Region.
-   * @param info Info for region to be opened
+   * @param info Info for region to be opened, must not be null
    * @param htd the table descriptor
    * @param wal WAL for region to use. This method will call
    * WAL#setSequenceNumber(long) passing the result of the call to
@@ -6389,110 +6434,32 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
    * @return new HRegion
    *
    * @throws IOException
+   * @throws IllegalArgumentException if info is null
    */
   public static HRegion openHRegion(final HRegionInfo info,
     final HTableDescriptor htd, final WAL wal, final Configuration conf,
     final RegionServerServices rsServices,
     final CancelableProgressable reporter)
   throws IOException {
-    return openHRegion(FSUtils.getRootDir(conf), info, htd, wal, conf, rsServices, reporter);
-  }
-
-  /**
-   * Open a Region.
-   * @param rootDir Root directory for HBase instance
-   * @param info Info for region to be opened.
-   * @param htd the table descriptor
-   * @param wal WAL for region to use. This method will call
-   * WAL#setSequenceNumber(long) passing the result of the call to
-   * HRegion#getMinSequenceId() to ensure the wal id is properly kept
-   * up.  HRegionStore does this every time it opens a new region.
-   * @param conf The Configuration object to use.
-   * @return new HRegion
-   * @throws IOException
-   */
-  public static HRegion openHRegion(Path rootDir, final HRegionInfo info,
-      final HTableDescriptor htd, final WAL wal, final Configuration conf)
-  throws IOException {
-    return openHRegion(rootDir, info, htd, wal, conf, null, null);
-  }
-
-  /**
-   * Open a Region.
-   * @param rootDir Root directory for HBase instance
-   * @param info Info for region to be opened.
-   * @param htd the table descriptor
-   * @param wal WAL for region to use. This method will call
-   * WAL#setSequenceNumber(long) passing the result of the call to
-   * HRegion#getMinSequenceId() to ensure the wal id is properly kept
-   * up.  HRegionStore does this every time it opens a new region.
-   * @param conf The Configuration object to use.
-   * @param rsServices An interface we can request flushes against.
-   * @param reporter An interface we can report progress against.
-   * @return new HRegion
-   * @throws IOException
-   */
-  public static HRegion openHRegion(final Path rootDir, final HRegionInfo info,
-      final HTableDescriptor htd, final WAL wal, final Configuration conf,
-      final RegionServerServices rsServices,
-      final CancelableProgressable reporter)
-  throws IOException {
-    FileSystem fs = null;
-    if (rsServices != null) {
-      fs = rsServices.getFileSystem();
-    }
-    if (fs == null) {
-      fs = FileSystem.get(conf);
+    if (info == null) throw new IllegalArgumentException("Passed region info is null");
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Opening region: " + info);
     }
-    return openHRegion(conf, fs, rootDir, info, htd, wal, rsServices, reporter);
-  }
-
-  /**
-   * Open a Region.
-   * @param conf The Configuration object to use.
-   * @param fs Filesystem to use
-   * @param rootDir Root directory for HBase instance
-   * @param info Info for region to be opened.
-   * @param htd the table descriptor
-   * @param wal WAL for region to use. This method will call
-   * WAL#setSequenceNumber(long) passing the result of the call to
-   * HRegion#getMinSequenceId() to ensure the wal id is properly kept
-   * up.  HRegionStore does this every time it opens a new region.
-   * @return new HRegion
-   * @throws IOException
-   */
-  public static HRegion openHRegion(final Configuration conf, final FileSystem fs,
-      final Path rootDir, final HRegionInfo info, final HTableDescriptor htd, final WAL wal)
-      throws IOException {
-    return openHRegion(conf, fs, rootDir, info, htd, wal, null, null);
+    HRegion r = HRegion.newHRegion(conf, htd, info, wal, rsServices);
+    return r.openHRegion(reporter);
   }
 
   /**
-   * Open a Region.
-   * @param conf The Configuration object to use.
-   * @param fs Filesystem to use
-   * @param rootDir Root directory for HBase instance
-   * @param info Info for region to be opened.
-   * @param htd the table descriptor
-   * @param wal WAL for region to use. This method will call
-   * WAL#setSequenceNumber(long) passing the result of the call to
-   * HRegion#getMinSequenceId() to ensure the wal id is properly kept
-   * up.  HRegionStore does this every time it opens a new region.
-   * @param rsServices An interface we can request flushes against.
-   * @param reporter An interface we can report progress against.
-   * @return new HRegion
-   * @throws IOException
+   * TODO remove after refactoring TableSnapshotScanner and TableSnapshotInputFormatImpl to use a RegionStorage impl instead of specifying a different root dir manually.
    */
-  public static HRegion openHRegion(final Configuration conf, final FileSystem fs,
-      final Path rootDir, final HRegionInfo info, final HTableDescriptor htd, final WAL wal,
-      final RegionServerServices rsServices, final CancelableProgressable reporter)
-      throws IOException {
-    if (info == null) throw new NullPointerException("Passed region info is null");
+  public static HRegion openHRegion(final FileSystem fs, final Path rootDir, final HRegionInfo info, HTableDescriptor htd, Configuration conf) throws IOException {
+    if (info == null) throw new IllegalArgumentException("Passed region info is null");
     if (LOG.isDebugEnabled()) {
       LOG.debug("Opening region: " + info);
     }
-    HRegion r = HRegion.newHRegion(conf, fs, rootDir, htd, info, wal, rsServices);
-    return r.openHRegion(reporter);
+    RegionStorage rfs = RegionStorage.open(conf, fs, new LegacyPathIdentifier(rootDir), info, false);
+    HRegion r = newHRegion(rfs, htd, null, null);
+    return r.openHRegion(null);
   }
 
   @VisibleForTesting
@@ -6509,10 +6476,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
    */
   public static HRegion openHRegion(final HRegion other, final CancelableProgressable reporter)
       throws IOException {
-    RegionFileSystem regionFs = other.getRegionFileSystem();
-    HRegion r = newHRegion(other.baseConf, regionFs.getFileSystem(),
-      regionFs.getRootDir(), other.getTableDesc(), other.getRegionInfo(),
-      other.getWAL(), null);
+    HRegion r = newHRegion(other.getRegionStorage(), other.htableDescriptor, other.getWAL(), null);
     return r.openHRegion(reporter);
   }
 
@@ -6560,17 +6524,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
       LOG.debug("HRegion.Warming up region: " + info);
     }
 
-    Path rootDir = FSUtils.getRootDir(conf);
-
-    FileSystem fs = null;
-    if (rsServices != null) {
-      fs = rsServices.getFileSystem();
-    }
-    if (fs == null) {
-      fs = FileSystem.get(conf);
-    }
-
-    HRegion r = HRegion.newHRegion(tableDir, wal, fs, conf, info, htd, null);
+    HRegion r = HRegion.newHRegion(conf, htd, info, wal, null);
     r.initializeWarmup(reporter);
   }
 
@@ -6603,7 +6557,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     fs.commitDaughterRegion(hri);
 
     // Create the daughter HRegion instance
-    HRegion r = HRegion.newHRegion(this.getBaseConf(), fs.getFileSystem(), this.fs.getRootDir(),
+    HRegion r = HRegion.newHRegion(this.getBaseConf(),
         this.getTableDesc(), hri, this.getWAL(), rsServices);
     r.readRequestsCount.add(this.getReadRequestsCount() / 2);
     r.filteredReadRequestsCount.add(this.getFilteredReadRequestsCount() / 2);
@@ -6619,7 +6573,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
    */
   HRegion createMergedRegionFromMerges(final HRegionInfo mergedRegionInfo,
       final HRegion region_b) throws IOException {
-    HRegion r = HRegion.newHRegion(this.getBaseConf(), fs.getFileSystem(), this.fs.getRootDir(),
+    HRegion r = HRegion.newHRegion(this.getBaseConf(),
         this.getTableDesc(), mergedRegionInfo, this.getWAL(), this.rsServices);
     r.readRequestsCount.add(this.getReadRequestsCount()
         + region_b.getReadRequestsCount());
@@ -6756,7 +6710,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
       throw new IOException("Regions do not belong to the same table");
     }
 
-    FileSystem fs = a.getRegionFileSystem().getFileSystem();
+    FileSystem fs = a.getRegionStorage().getFileSystem();
     // Make sure each region's cache is empty
     a.flush(true);
     b.flush(true);
@@ -6765,12 +6719,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     a.compact(true);
     if (LOG.isDebugEnabled()) {
       LOG.debug("Files for region: " + a);
-      a.getRegionFileSystem().logFileSystemState(LOG);
+      a.getRegionStorage().logFileSystemState(LOG);
     }
     b.compact(true);
     if (LOG.isDebugEnabled()) {
       LOG.debug("Files for region: " + b);
-      b.getRegionFileSystem().logFileSystemState(LOG);
+      b.getRegionStorage().logFileSystemState(LOG);
     }
 
     RegionMergeTransactionImpl rmt = new RegionMergeTransactionImpl(a, b, true);
@@ -6796,14 +6750,14 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("Files for new region");
-      dstRegion.getRegionFileSystem().logFileSystemState(LOG);
+      dstRegion.getRegionStorage().logFileSystemState(LOG);
     }
 
     // clear the compacted files if any
     for (Store s : dstRegion.getStores()) {
       s.closeAndArchiveCompactedFiles();
     }
-    if (dstRegion.getRegionFileSystem().hasReferences(dstRegion.getTableDesc())) {
+    if (dstRegion.getRegionStorage().hasReferences(dstRegion.getTableDesc())) {
       throw new IOException("Merged region " + dstRegion
           + " still has references after the compaction, is compaction canceled?");
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index e3b1a85..ba66ded 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -68,7 +68,9 @@ import org.apache.hadoop.hbase.TagUtil;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.conf.ConfigurationManager;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.io.compress.Compression;
 import org.apache.hadoop.hbase.io.crypto.Encryption;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -80,7 +82,7 @@ import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.io.hfile.InvalidHFileException;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.ServerProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
@@ -128,7 +130,7 @@ public class HStore implements Store {
   // This stores directory in the filesystem.
   protected final HRegion region;
   private final HColumnDescriptor family;
-  private final RegionFileSystem fs;
+  private final RegionStorage fs;
   protected Configuration conf;
   protected CacheConfig cacheConf;
   private long lastCompactSize = 0;
@@ -201,10 +203,10 @@ public class HStore implements Store {
   protected HStore(final HRegion region, final HColumnDescriptor family,
       final Configuration confParam) throws IOException {
 
-    this.fs = region.getRegionFileSystem();
+    this.fs = region.getRegionStorage();
 
     // Assemble the store's home directory and Ensure it exists.
-    fs.createStoreDir(family.getNameAsString());
+    fs.createStoreContainer(family.getNameAsString());
     this.region = region;
     this.family = family;
     // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
@@ -336,7 +338,7 @@ public class HStore implements Store {
     return this.fs.getFileSystem();
   }
 
-  public RegionFileSystem getRegionFileSystem() {
+  public RegionStorage getRegionStorage() {
     return this.fs;
   }
 
@@ -734,21 +736,21 @@ public class HStore implements Store {
     }
   }
 
+  /* TODO bulkload rephrased in terms of transfer between RegionStorage impls. */
   @Override
-  public Path bulkLoadHFile(String srcPathStr, long seqNum) throws IOException {
-    Path srcPath = new Path(srcPathStr);
-    Path dstPath = fs.bulkLoadStoreFile(getColumnFamilyName(), srcPath, seqNum);
+  public StoreFile bulkLoadHFile(String srcPathStr, long seqNum) throws IOException {
+    LegacyPathIdentifier srcPath = new LegacyPathIdentifier(new Path(srcPathStr));
+    StoreFile sf = fs.bulkLoadStoreFile(getColumnFamilyName(), srcPath, seqNum, cacheConf, family.getBloomFilterType(), this.region.getCoprocessorHost());
 
     LOG.info("Loaded HFile " + srcPath + " into store '" + getColumnFamilyName() + "' as "
-        + dstPath + " - updating store file list.");
+        + sf + " - updating store file list.");
 
-    StoreFile sf = createStoreFileAndReader(dstPath);
     bulkLoadHFile(sf);
 
     LOG.info("Successfully loaded store file " + srcPath + " into store " + this
-        + " (new location: " + dstPath + ")");
+        + " (new location: " + sf + ")");
 
-    return dstPath;
+    return sf;
   }
 
   @Override
@@ -921,10 +923,11 @@ public class HStore implements Store {
   private StoreFile commitFile(final Path path, final long logCacheFlushId, MonitoredTask status)
       throws IOException {
     // Write-out finished successfully, move into the right spot
-    Path dstPath = fs.commitStoreFile(getColumnFamilyName(), path);
+    // TODO move from Path to StoreFileWriter or StorageIdentifier that goes ends-to-end 
+    final StorageIdentifier id = new LegacyPathIdentifier(path);
+    final StoreFile sf = fs.commitStoreFile(getColumnFamilyName(), id, this.cacheConf, this.family.getBloomFilterType(), this.region.getCoprocessorHost());
 
     status.setStatus("Flushing " + this + ": reopening flushed file");
-    StoreFile sf = createStoreFileAndReader(dstPath);
 
     StoreFileReader r = sf.getReader();
     this.storeSize += r.length();
@@ -992,9 +995,10 @@ public class HStore implements Store {
     }
     HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag,
       cryptoContext);
+    // TODO move StoreFileWriter to use RegionStorage directly instead of Path
     StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf,
         this.getFileSystem())
-            .withFilePath(fs.createTempName())
+            .withFilePath(((LegacyPathIdentifier)fs.getTempIdentifier()).path)
             .withComparator(comparator)
             .withBloomType(family.getBloomFilterType())
             .withMaxKeyCount(maxKeyCount)
@@ -1223,7 +1227,7 @@ public class HStore implements Store {
       // Ready to go. Have list of files to compact.
       LOG.info("Starting compaction of " + filesToCompact.size() + " file(s) in "
           + this + " of " + this.getRegionInfo().getRegionNameAsString()
-          + " into tmpdir=" + fs.getTempDir() + ", totalSize="
+          + " into tmpdir=" + fs.getTempContainer() + ", totalSize="
           + TraditionalBinaryPrefix.long2String(cr.getSize(), "", 1));
 
       // Commence the compaction.
@@ -1298,8 +1302,10 @@ public class HStore implements Store {
   StoreFile moveFileIntoPlace(final Path newFile) throws IOException {
     validateStoreFile(newFile);
     // Move the file into the right spot
-    Path destPath = fs.commitStoreFile(getColumnFamilyName(), newFile);
-    return createStoreFileAndReader(destPath);
+    // TODO work backwards to end up with a StorageIdentifier instead of Path as arg.
+    final StorageIdentifier id = new LegacyPathIdentifier(newFile);
+    final StoreFile sf = fs.commitStoreFile(getColumnFamilyName(), id, this.cacheConf, this.family.getBloomFilterType(), this.region.getCoprocessorHost());
+    return sf;
   }
 
   /**
@@ -1319,8 +1325,8 @@ public class HStore implements Store {
       outputPaths.add(f.getPath());
     }
     HRegionInfo info = this.region.getRegionInfo();
-    CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
-        family.getName(), inputPaths, outputPaths, fs.getStoreDir(getFamily().getNameAsString()));
+    CompactionDescriptor compactionDescriptor = ServerProtobufUtil.toCompactionDescriptor(info,
+        family.getName(), inputPaths, outputPaths, fs.getStoreContainer(getFamily().getNameAsString()));
     // Fix reaching into Region to get the maxWaitForSeqId.
     // Does this method belong in Region altogether given it is making so many references up there?
     // Could be Region#writeCompactionMarker(compactionDescriptor);
@@ -1382,6 +1388,7 @@ public class HStore implements Store {
     }
   }
 
+  // TODO move this into RegionStorage
   /**
    * Call to complete a compaction. Its for the case where we find in the WAL a compaction
    * that was not finished.  We could find one recovering a WAL after a regionserver crash.
@@ -1413,7 +1420,7 @@ public class HStore implements Store {
     String familyName = this.getColumnFamilyName();
     List<String> inputFiles = new ArrayList<String>(compactionInputs.size());
     for (String compactionInput : compactionInputs) {
-      Path inputPath = fs.getStoreFilePath(familyName, compactionInput);
+      Path inputPath = ((LegacyPathIdentifier)fs.getStoreFileStorageIdentifier(familyName, compactionInput)).path;
       inputFiles.add(inputPath.getName());
     }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
index 4e76e76..27fd941 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
@@ -220,6 +220,7 @@ public interface Region extends ConfigurationObserver {
   /** @return the MetricsRegion for this region */
   MetricsRegion getMetrics();
 
+  /* TODO make this something like host locality so it generalizes */
   /** @return the block distribution for all Stores managed by this region */
   HDFSBlocksDistribution getHDFSBlocksDistribution();
 


[10/18] hbase git commit: HBASE-14439 Move fs stuff out pt2 - region fs

Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 85a8080..80fbb76 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -32,6 +32,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -72,6 +73,9 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
@@ -275,21 +279,6 @@ public abstract class FSUtils {
   }
 
   /**
-   * Delete the region directory if exists.
-   * @param conf
-   * @param hri
-   * @return True if deleted the region directory.
-   * @throws IOException
-   */
-  public static boolean deleteRegionDir(final Configuration conf, final HRegionInfo hri)
-  throws IOException {
-    Path rootDir = getRootDir(conf);
-    FileSystem fs = rootDir.getFileSystem(conf);
-    return deleteDirectory(fs,
-      new Path(getTableDir(rootDir, hri.getTable()), hri.getEncodedName()));
-  }
-
-  /**
    * Return the number of bytes that large input files should be optimally
    * be split into to minimize i/o time.
    *
@@ -1078,7 +1067,7 @@ public abstract class FSUtils {
    * @throws IOException When scanning the directory fails.
    */
   public static int getTotalTableFragmentation(final HMaster master)
-  throws IOException {
+      throws IOException {
     Map<String, Integer> map = getTableFragmentation(master);
     return map != null && map.size() > 0 ? map.get("-TOTAL-") : -1;
   }
@@ -1093,56 +1082,29 @@ public abstract class FSUtils {
    *
    * @throws IOException When scanning the directory fails.
    */
-  public static Map<String, Integer> getTableFragmentation(
-    final HMaster master)
-  throws IOException {
-    Path path = getRootDir(master.getConfiguration());
-    // since HMaster.getFileSystem() is package private
-    FileSystem fs = path.getFileSystem(master.getConfiguration());
-    return getTableFragmentation(fs, path);
-  }
-
-  /**
-   * Runs through the HBase rootdir and checks how many stores for each table
-   * have more than one file in them. Checks -ROOT- and hbase:meta too. The total
-   * percentage across all tables is stored under the special key "-TOTAL-".
-   *
-   * @param fs  The file system to use.
-   * @param hbaseRootDir  The root directory to scan.
-   * @return A map for each table and its percentage.
-   * @throws IOException When scanning the directory fails.
-   */
-  public static Map<String, Integer> getTableFragmentation(
-    final FileSystem fs, final Path hbaseRootDir)
-  throws IOException {
-    Map<String, Integer> frags = new HashMap<String, Integer>();
+  public static Map<String, Integer> getTableFragmentation(final HMaster master)
+      throws IOException {
+    final Map<String, Integer> frags = new HashMap<String, Integer>();
     int cfCountTotal = 0;
     int cfFragTotal = 0;
-    PathFilter regionFilter = new RegionDirFilter(fs);
-    PathFilter familyFilter = new FamilyDirFilter(fs);
-    List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
-    for (Path d : tableDirs) {
+
+    MasterFileSystem mfs = master.getMasterFileSystem();
+    for (TableName table: mfs.getTables()) {
       int cfCount = 0;
       int cfFrag = 0;
-      FileStatus[] regionDirs = fs.listStatus(d, regionFilter);
-      for (FileStatus regionDir : regionDirs) {
-        Path dd = regionDir.getPath();
-        // else its a region name, now look in region for families
-        FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);
-        for (FileStatus familyDir : familyDirs) {
+      for (HRegionInfo hri: mfs.getRegions(table)) {
+        RegionFileSystem rfs = mfs.getRegionFileSystem(hri);
+        for (String family: rfs.getFamilies()) {
           cfCount++;
           cfCountTotal++;
-          Path family = familyDir.getPath();
-          // now in family make sure only one file
-          FileStatus[] familyStatus = fs.listStatus(family);
-          if (familyStatus.length > 1) {
+          if (rfs.getStoreFiles(family).size() > 1) {
             cfFrag++;
             cfFragTotal++;
           }
         }
       }
       // compute percentage per table and store in result list
-      frags.put(FSUtils.getTableName(d).getNameAsString(),
+      frags.put(table.getNameAsString(),
         cfCount == 0? 0: Math.round((float) cfFrag / cfCount * 100));
     }
     // set overall percentage for all tables
@@ -1459,6 +1421,18 @@ public abstract class FSUtils {
     return referenceFiles;
   }
 
+  public static int getRegionReferenceFileCount(final FileSystem fs, final Path p) {
+    int result = 0;
+    try {
+      for (Path familyDir:getFamilyDirs(fs, p)){
+        result += getReferenceFilePaths(fs, familyDir).size();
+      }
+    } catch (IOException e) {
+      LOG.warn("Error Counting reference files.", e);
+    }
+    return result;
+  }
+
   /**
    * Filter for HFiles that excludes reference files.
    */
@@ -1521,232 +1495,6 @@ public abstract class FSUtils {
     return getRootDir(conf).getFileSystem(conf);
   }
 
-
-  /**
-   * Runs through the HBase rootdir/tablename and creates a reverse lookup map for
-   * table StoreFile names to the full Path.
-   * <br>
-   * Example...<br>
-   * Key = 3944417774205889744  <br>
-   * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
-   *
-   * @param map map to add values.  If null, this method will create and populate one to return
-   * @param fs  The file system to use.
-   * @param hbaseRootDir  The root directory to scan.
-   * @param tableName name of the table to scan.
-   * @return Map keyed by StoreFile name with a value of the full Path.
-   * @throws IOException When scanning the directory fails.
-   * @throws InterruptedException
-   */
-  public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
-  final FileSystem fs, final Path hbaseRootDir, TableName tableName)
-  throws IOException, InterruptedException {
-    return getTableStoreFilePathMap(map, fs, hbaseRootDir, tableName, null, null, null);
-  }
-
-  /**
-   * Runs through the HBase rootdir/tablename and creates a reverse lookup map for
-   * table StoreFile names to the full Path.  Note that because this method can be called
-   * on a 'live' HBase system that we will skip files that no longer exist by the time
-   * we traverse them and similarly the user of the result needs to consider that some
-   * entries in this map may not exist by the time this call completes.
-   * <br>
-   * Example...<br>
-   * Key = 3944417774205889744  <br>
-   * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
-   *
-   * @param resultMap map to add values.  If null, this method will create and populate one to return
-   * @param fs  The file system to use.
-   * @param hbaseRootDir  The root directory to scan.
-   * @param tableName name of the table to scan.
-   * @param sfFilter optional path filter to apply to store files
-   * @param executor optional executor service to parallelize this operation
-   * @param errors ErrorReporter instance or null
-   * @return Map keyed by StoreFile name with a value of the full Path.
-   * @throws IOException When scanning the directory fails.
-   * @throws InterruptedException
-   */
-  public static Map<String, Path> getTableStoreFilePathMap(
-      Map<String, Path> resultMap,
-      final FileSystem fs, final Path hbaseRootDir, TableName tableName, final PathFilter sfFilter,
-      ExecutorService executor, final ErrorReporter errors) throws IOException, InterruptedException {
-
-    final Map<String, Path> finalResultMap =
-        resultMap == null ? new ConcurrentHashMap<String, Path>(128, 0.75f, 32) : resultMap;
-
-    // only include the directory paths to tables
-    Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName);
-    // Inside a table, there are compaction.dir directories to skip.  Otherwise, all else
-    // should be regions.
-    final FamilyDirFilter familyFilter = new FamilyDirFilter(fs);
-    final Vector<Exception> exceptions = new Vector<Exception>();
-
-    try {
-      List<FileStatus> regionDirs = FSUtils.listStatusWithStatusFilter(fs, tableDir, new RegionDirFilter(fs));
-      if (regionDirs == null) {
-        return finalResultMap;
-      }
-
-      final List<Future<?>> futures = new ArrayList<Future<?>>(regionDirs.size());
-
-      for (FileStatus regionDir : regionDirs) {
-        if (null != errors) {
-          errors.progress();
-        }
-        final Path dd = regionDir.getPath();
-
-        if (!exceptions.isEmpty()) {
-          break;
-        }
-
-        Runnable getRegionStoreFileMapCall = new Runnable() {
-          @Override
-          public void run() {
-            try {
-              HashMap<String,Path> regionStoreFileMap = new HashMap<String, Path>();
-              List<FileStatus> familyDirs = FSUtils.listStatusWithStatusFilter(fs, dd, familyFilter);
-              if (familyDirs == null) {
-                if (!fs.exists(dd)) {
-                  LOG.warn("Skipping region because it no longer exists: " + dd);
-                } else {
-                  LOG.warn("Skipping region because it has no family dirs: " + dd);
-                }
-                return;
-              }
-              for (FileStatus familyDir : familyDirs) {
-                if (null != errors) {
-                  errors.progress();
-                }
-                Path family = familyDir.getPath();
-                if (family.getName().equals(HConstants.RECOVERED_EDITS_DIR)) {
-                  continue;
-                }
-                // now in family, iterate over the StoreFiles and
-                // put in map
-                FileStatus[] familyStatus = fs.listStatus(family);
-                for (FileStatus sfStatus : familyStatus) {
-                  if (null != errors) {
-                    errors.progress();
-                  }
-                  Path sf = sfStatus.getPath();
-                  if (sfFilter == null || sfFilter.accept(sf)) {
-                    regionStoreFileMap.put( sf.getName(), sf);
-                  }
-                }
-              }
-              finalResultMap.putAll(regionStoreFileMap);
-            } catch (Exception e) {
-              LOG.error("Could not get region store file map for region: " + dd, e);
-              exceptions.add(e);
-            }
-          }
-        };
-
-        // If executor is available, submit async tasks to exec concurrently, otherwise
-        // just do serial sync execution
-        if (executor != null) {
-          Future<?> future = executor.submit(getRegionStoreFileMapCall);
-          futures.add(future);
-        } else {
-          FutureTask<?> future = new FutureTask<Object>(getRegionStoreFileMapCall, null);
-          future.run();
-          futures.add(future);
-        }
-      }
-
-      // Ensure all pending tasks are complete (or that we run into an exception)
-      for (Future<?> f : futures) {
-        if (!exceptions.isEmpty()) {
-          break;
-        }
-        try {
-          f.get();
-        } catch (ExecutionException e) {
-          LOG.error("Unexpected exec exception!  Should've been caught already.  (Bug?)", e);
-          // Shouldn't happen, we already logged/caught any exceptions in the Runnable
-        }
-      }
-    } catch (IOException e) {
-      LOG.error("Cannot execute getTableStoreFilePathMap for " + tableName, e);
-      exceptions.add(e);
-    } finally {
-      if (!exceptions.isEmpty()) {
-        // Just throw the first exception as an indication something bad happened
-        // Don't need to propagate all the exceptions, we already logged them all anyway
-        Throwables.propagateIfInstanceOf(exceptions.firstElement(), IOException.class);
-        throw Throwables.propagate(exceptions.firstElement());
-      }
-    }
-
-    return finalResultMap;
-  }
-
-  public static int getRegionReferenceFileCount(final FileSystem fs, final Path p) {
-    int result = 0;
-    try {
-      for (Path familyDir:getFamilyDirs(fs, p)){
-        result += getReferenceFilePaths(fs, familyDir).size();
-      }
-    } catch (IOException e) {
-      LOG.warn("Error Counting reference files.", e);
-    }
-    return result;
-  }
-
-  /**
-   * Runs through the HBase rootdir and creates a reverse lookup map for
-   * table StoreFile names to the full Path.
-   * <br>
-   * Example...<br>
-   * Key = 3944417774205889744  <br>
-   * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
-   *
-   * @param fs  The file system to use.
-   * @param hbaseRootDir  The root directory to scan.
-   * @return Map keyed by StoreFile name with a value of the full Path.
-   * @throws IOException When scanning the directory fails.
-   * @throws InterruptedException
-   */
-  public static Map<String, Path> getTableStoreFilePathMap(
-    final FileSystem fs, final Path hbaseRootDir)
-  throws IOException, InterruptedException {
-    return getTableStoreFilePathMap(fs, hbaseRootDir, null, null, null);
-  }
-
-  /**
-   * Runs through the HBase rootdir and creates a reverse lookup map for
-   * table StoreFile names to the full Path.
-   * <br>
-   * Example...<br>
-   * Key = 3944417774205889744  <br>
-   * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
-   *
-   * @param fs  The file system to use.
-   * @param hbaseRootDir  The root directory to scan.
-   * @param sfFilter optional path filter to apply to store files
-   * @param executor optional executor service to parallelize this operation
-   * @param errors ErrorReporter instance or null
-   * @return Map keyed by StoreFile name with a value of the full Path.
-   * @throws IOException When scanning the directory fails.
-   * @throws InterruptedException
-   */
-  public static Map<String, Path> getTableStoreFilePathMap(
-    final FileSystem fs, final Path hbaseRootDir, PathFilter sfFilter,
-    ExecutorService executor, ErrorReporter errors)
-  throws IOException, InterruptedException {
-    ConcurrentHashMap<String, Path> map = new ConcurrentHashMap<String, Path>(1024, 0.75f, 32);
-
-    // if this method looks similar to 'getTableFragmentation' that is because
-    // it was borrowed from it.
-
-    // only include the directory paths to tables
-    for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) {
-      getTableStoreFilePathMap(map, fs, hbaseRootDir,
-          FSUtils.getTableName(tableDir), sfFilter, executor, errors);
-    }
-    return map;
-  }
-
   /**
    * Filters FileStatuses in an array and returns a list
    *
@@ -1999,13 +1747,12 @@ public abstract class FSUtils {
    * @throws IOException
    *           in case of file system errors or interrupts
    */
-  public static Map<String, Map<String, Float>> getRegionDegreeLocalityMappingFromFS(
+  private static Map<String, Map<String, Float>> getRegionDegreeLocalityMappingFromFS(
       final Configuration conf, final String desiredTable, int threadPoolSize)
       throws IOException {
     Map<String, Map<String, Float>> regionDegreeLocalityMapping =
         new ConcurrentHashMap<String, Map<String, Float>>();
-    getRegionLocalityMappingFromFS(conf, desiredTable, threadPoolSize, null,
-        regionDegreeLocalityMapping);
+    getRegionLocalityMappingFromFS(conf, desiredTable, threadPoolSize, regionDegreeLocalityMapping);
     return regionDegreeLocalityMapping;
   }
 
@@ -2021,8 +1768,6 @@ public abstract class FSUtils {
    *          the table you wish to scan locality for
    * @param threadPoolSize
    *          the thread pool size to use
-   * @param regionToBestLocalityRSMapping
-   *          the map into which to put the best locality mapping or null
    * @param regionDegreeLocalityMapping
    *          the map into which to put the locality degree mapping or null,
    *          must be a thread-safe implementation
@@ -2032,81 +1777,36 @@ public abstract class FSUtils {
   private static void getRegionLocalityMappingFromFS(
       final Configuration conf, final String desiredTable,
       int threadPoolSize,
-      Map<String, String> regionToBestLocalityRSMapping,
       Map<String, Map<String, Float>> regionDegreeLocalityMapping)
       throws IOException {
-    FileSystem fs =  FileSystem.get(conf);
-    Path rootPath = FSUtils.getRootDir(conf);
     long startTime = EnvironmentEdgeManager.currentTime();
-    Path queryPath;
-    // The table files are in ${hbase.rootdir}/data/<namespace>/<table>/*
-    if (null == desiredTable) {
-      queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR).toString() + "/*/*/*/");
-    } else {
-      queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)).toString() + "/*/");
-    }
-
-    // reject all paths that are not appropriate
-    PathFilter pathFilter = new PathFilter() {
-      @Override
-      public boolean accept(Path path) {
-        // this is the region name; it may get some noise data
-        if (null == path) {
-          return false;
-        }
-
-        // no parent?
-        Path parent = path.getParent();
-        if (null == parent) {
-          return false;
-        }
-
-        String regionName = path.getName();
-        if (null == regionName) {
-          return false;
-        }
 
-        if (!regionName.toLowerCase(Locale.ROOT).matches("[0-9a-f]+")) {
-          return false;
-        }
-        return true;
+    MasterFileSystem mfs = MasterFileSystem.open(conf, false);
+    Collection<HRegionInfo> hris;
+    if (desiredTable != null) {
+      hris = mfs.getRegions(TableName.valueOf(desiredTable));
+    } else {
+      hris = new ArrayList<HRegionInfo>();
+      for (TableName tableName: mfs.getTables()) {
+        hris.addAll(mfs.getRegions(tableName));
       }
-    };
-
-    FileStatus[] statusList = fs.globStatus(queryPath, pathFilter);
+    }
 
-    if (null == statusList) {
+    if (hris.isEmpty()) {
       return;
-    } else {
-      LOG.debug("Query Path: " + queryPath + " ; # list of files: " +
-          statusList.length);
     }
 
     // lower the number of threads in case we have very few expected regions
-    threadPoolSize = Math.min(threadPoolSize, statusList.length);
+    threadPoolSize = Math.min(threadPoolSize, hris.size());
 
     // run in multiple threads
     ThreadPoolExecutor tpe = new ThreadPoolExecutor(threadPoolSize,
         threadPoolSize, 60, TimeUnit.SECONDS,
-        new ArrayBlockingQueue<Runnable>(statusList.length));
+        new ArrayBlockingQueue<Runnable>(hris.size()));
     try {
       // ignore all file status items that are not of interest
-      for (FileStatus regionStatus : statusList) {
-        if (null == regionStatus) {
-          continue;
-        }
-
-        if (!regionStatus.isDirectory()) {
-          continue;
-        }
-
-        Path regionPath = regionStatus.getPath();
-        if (null == regionPath) {
-          continue;
-        }
-
-        tpe.execute(new FSRegionScanner(fs, regionPath,
-            regionToBestLocalityRSMapping, regionDegreeLocalityMapping));
+      for (HRegionInfo hri: hris) {
+        tpe.execute(new FSRegionScanner(conf, hri, regionDegreeLocalityMapping));
       }
     } finally {
       tpe.shutdown();

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
deleted file mode 100644
index d5197cf..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSVisitor.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.util;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-
-/**
- * Utility methods for interacting with the hbase.root file system.
- */
-@InterfaceAudience.Private
-public final class FSVisitor {
-  private static final Log LOG = LogFactory.getLog(FSVisitor.class);
-
-  public interface StoreFileVisitor {
-    void storeFile(final String region, final String family, final String hfileName)
-       throws IOException;
-  }
-
-  private FSVisitor() {
-    // private constructor for utility class
-  }
-
-  /**
-   * Iterate over the table store files
-   *
-   * @param fs {@link FileSystem}
-   * @param tableDir {@link Path} to the table directory
-   * @param visitor callback object to get the store files
-   * @throws IOException if an error occurred while scanning the directory
-   */
-  public static void visitTableStoreFiles(final FileSystem fs, final Path tableDir,
-      final StoreFileVisitor visitor) throws IOException {
-    List<FileStatus> regions = FSUtils.listStatusWithStatusFilter(fs, tableDir, new FSUtils.RegionDirFilter(fs));
-    if (regions == null) {
-      if (LOG.isTraceEnabled()) {
-        LOG.trace("No regions under directory:" + tableDir);
-      }
-      return;
-    }
-
-    for (FileStatus region: regions) {
-      visitRegionStoreFiles(fs, region.getPath(), visitor);
-    }
-  }
-
-  /**
-   * Iterate over the region store files
-   *
-   * @param fs {@link FileSystem}
-   * @param regionDir {@link Path} to the region directory
-   * @param visitor callback object to get the store files
-   * @throws IOException if an error occurred while scanning the directory
-   */
-  public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
-      final StoreFileVisitor visitor) throws IOException {
-    List<FileStatus> families = FSUtils.listStatusWithStatusFilter(fs, regionDir, new FSUtils.FamilyDirFilter(fs));
-    if (families == null) {
-      if (LOG.isTraceEnabled()) {
-        LOG.trace("No families under region directory:" + regionDir);
-      }
-      return;
-    }
-
-    PathFilter fileFilter = new FSUtils.FileFilter(fs);
-    for (FileStatus family: families) {
-      Path familyDir = family.getPath();
-      String familyName = familyDir.getName();
-
-      // get all the storeFiles in the family
-      FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir, fileFilter);
-      if (storeFiles == null) {
-        if (LOG.isTraceEnabled()) {
-          LOG.trace("No hfiles found for family: " + familyDir + ", skipping.");
-        }
-        continue;
-      }
-
-      for (FileStatus hfile: storeFiles) {
-        Path hfilePath = hfile.getPath();
-        visitor.storeFile(regionDir.getName(), familyName, hfilePath.getName());
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index a12425d..e1a3fa2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -110,6 +110,8 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -1070,51 +1072,54 @@ public class HBaseFsck extends Configured implements Closeable {
    * Lingering reference file prevents a region from opening. It has to
    * be fixed before a cluster can start properly.
    */
-  private void offlineReferenceFileRepair() throws IOException, InterruptedException {
+  private void offlineReferenceFileRepair() throws IOException {
     clearState();
-    Configuration conf = getConf();
-    Path hbaseRoot = FSUtils.getRootDir(conf);
-    FileSystem fs = hbaseRoot.getFileSystem(conf);
-    LOG.info("Computing mapping of all store files");
-    Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot,
-      new FSUtils.ReferenceFileFilter(fs), executor, errors);
-    errors.print("");
     LOG.info("Validating mapping using HDFS state");
-    for (Path path: allFiles.values()) {
-      Path referredToFile = StoreFileInfo.getReferredToFile(path);
-      if (fs.exists(referredToFile)) continue;  // good, expected
-
-      // Found a lingering reference file
-      errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
-        "Found lingering reference file " + path);
-      if (!shouldFixReferenceFiles()) continue;
-
-      // Now, trying to fix it since requested
-      boolean success = false;
-      String pathStr = path.toString();
-
-      // A reference file path should be like
-      // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
-      // Up 5 directories to get the root folder.
-      // So the file will be sidelined to a similar folder structure.
-      int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
-      for (int i = 0; index > 0 && i < 5; i++) {
-        index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
-      }
-      if (index > 0) {
-        Path rootDir = getSidelineDir();
-        Path dst = new Path(rootDir, pathStr.substring(index + 1));
-        fs.mkdirs(dst.getParent());
-        LOG.info("Trying to sildeline reference file "
-          + path + " to " + dst);
-        setShouldRerun();
-
-        success = fs.rename(path, dst);
-      }
-      if (!success) {
-        LOG.error("Failed to sideline reference file " + path);
+    final MasterFileSystem mfs = MasterFileSystem.open(getConf(), false);
+    mfs.visitStoreFiles(new StoreFileVisitor() {
+      @Override
+      public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
+          throws IOException {
+        if (errors != null) errors.progress();
+        if (!storeFile.isReference()) return;
+
+        FileSystem fs = mfs.getFileSystem();
+        Path path = storeFile.getPath();
+        Path referredToFile = StoreFileInfo.getReferredToFile(path);
+        if (fs.exists(referredToFile)) return;  // good, expected
+
+        // Found a lingering reference file
+        errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
+          "Found lingering reference file " + path);
+        if (!shouldFixReferenceFiles()) return;
+
+        // Now, trying to fix it since requested
+        boolean success = false;
+        String pathStr = path.toString();
+
+        // A reference file path should be like
+        // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
+        // Up 5 directories to get the root folder.
+        // So the file will be sidelined to a similar folder structure.
+        int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
+        for (int i = 0; index > 0 && i < 5; i++) {
+          index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
+        }
+        if (index > 0) {
+          Path rootDir = getSidelineDir();
+          Path dst = new Path(rootDir, pathStr.substring(index + 1));
+          fs.mkdirs(dst.getParent());
+          LOG.info("Trying to sildeline reference file "
+            + path + " to " + dst);
+          setShouldRerun();
+
+          success = fs.rename(path, dst);
+        }
+        if (!success) {
+          LOG.error("Failed to sideline reference file " + path);
+        }
       }
-    }
+    });
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
deleted file mode 100644
index 2fbaa92..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.util;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HStore;
-
-/**
- * Helper class for all utilities related to archival/retrieval of HFiles
- */
-public class HFileArchiveUtil {
-  private HFileArchiveUtil() {
-    // non-external instantiation - util class
-  }
-
-  /**
-   * Get the directory to archive a store directory
-   * @param conf {@link Configuration} to read for the archive directory name
-   * @param tableName table name under which the store currently lives
-   * @param regionName region encoded name under which the store currently lives
-   * @param familyName name of the family in the store
-   * @return {@link Path} to the directory to archive the given store or
-   *         <tt>null</tt> if it should not be archived
-   */
-  public static Path getStoreArchivePath(final Configuration conf,
-                                         final TableName tableName,
-      final String regionName, final String familyName) throws IOException {
-    Path tableArchiveDir = getTableArchivePath(conf, tableName);
-    return HStore.getStoreHomedir(tableArchiveDir, regionName, Bytes.toBytes(familyName));
-  }
-
-  /**
-   * Get the directory to archive a store directory
-   * @param conf {@link Configuration} to read for the archive directory name.
-   * @param region parent region information under which the store currently lives
-   * @param tabledir directory for the table under which the store currently lives
-   * @param family name of the family in the store
-   * @return {@link Path} to the directory to archive the given store or <tt>null</tt> if it should
-   *         not be archived
-   */
-  public static Path getStoreArchivePath(Configuration conf,
-                                         HRegionInfo region,
-                                         Path tabledir,
-      byte[] family) throws IOException {
-    Path rootDir = FSUtils.getRootDir(conf);
-    Path tableArchiveDir = getTableArchivePath(rootDir, region.getTable());
-    return HStore.getStoreHomedir(tableArchiveDir, region, family);
-  }
-
-  /**
-   * Get the archive directory for a given region under the specified table
-   * @param tableName the table name. Cannot be null.
-   * @param regiondir the path to the region directory. Cannot be null.
-   * @return {@link Path} to the directory to archive the given region, or <tt>null</tt> if it
-   *         should not be archived
-   */
-  public static Path getRegionArchiveDir(Path rootDir,
-                                         TableName tableName,
-                                         Path regiondir) {
-    // get the archive directory for a table
-    Path archiveDir = getTableArchivePath(rootDir, tableName);
-
-    // then add on the region path under the archive
-    String encodedRegionName = regiondir.getName();
-    return HRegion.getRegionDir(archiveDir, encodedRegionName);
-  }
-
-  /**
-   * Get the archive directory for a given region under the specified table
-   * @param rootDir {@link Path} to the root directory where hbase files are stored (for building
-   *          the archive path)
-   * @param tableName name of the table to archive. Cannot be null.
-   * @return {@link Path} to the directory to archive the given region, or <tt>null</tt> if it
-   *         should not be archived
-   */
-  public static Path getRegionArchiveDir(Path rootDir,
-                                         TableName tableName, String encodedRegionName) {
-    // get the archive directory for a table
-    Path archiveDir = getTableArchivePath(rootDir, tableName);
-    return HRegion.getRegionDir(archiveDir, encodedRegionName);
-  }
-
-  /**
-   * Get the path to the table archive directory based on the configured archive directory.
-   * <p>
-   * Get the path to the table's archive directory.
-   * <p>
-   * Generally of the form: /hbase/.archive/[tablename]
-   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
-   *          the archive path)
-   * @param tableName Name of the table to be archived. Cannot be null.
-   * @return {@link Path} to the archive directory for the table
-   */
-  public static Path getTableArchivePath(final Path rootdir, final TableName tableName) {
-    return FSUtils.getTableDir(getArchivePath(rootdir), tableName);
-  }
-
-  /**
-   * Get the path to the table archive directory based on the configured archive directory.
-   * <p>
-   * Assumed that the table should already be archived.
-   * @param conf {@link Configuration} to read the archive directory property. Can be null
-   * @param tableName Name of the table to be archived. Cannot be null.
-   * @return {@link Path} to the archive directory for the table
-   */
-  public static Path getTableArchivePath(final Configuration conf,
-                                         final TableName tableName)
-      throws IOException {
-    return FSUtils.getTableDir(getArchivePath(conf), tableName);
-  }
-
-  /**
-   * Get the full path to the archive directory on the configured 
-   * {@link org.apache.hadoop.hbase.fs.MasterFileSystem}
-   * @param conf to look for archive directory name and root directory. Cannot be null. Notes for
-   *          testing: requires a FileSystem root directory to be specified.
-   * @return the full {@link Path} to the archive directory, as defined by the configuration
-   * @throws IOException if an unexpected error occurs
-   */
-  public static Path getArchivePath(Configuration conf) throws IOException {
-    return getArchivePath(FSUtils.getRootDir(conf));
-  }
-
-  /**
-   * Get the full path to the archive directory on the configured 
-   * {@link org.apache.hadoop.hbase.fs.MasterFileSystem}
-   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
-   *          the archive path)
-   * @return the full {@link Path} to the archive directory, as defined by the configuration
-   */
-  private static Path getArchivePath(final Path rootdir) {
-    return new Path(rootdir, HConstants.HFILE_ARCHIVE_DIRECTORY);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
index 68b0ba3..1325c85 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
@@ -28,19 +28,22 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.FsContext;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSVisitor;
 import org.apache.hadoop.hbase.util.TestTableName;
 
 import org.junit.After;
@@ -103,13 +106,11 @@ public class TestCorruptedRegionStoreFile {
 
     // get the store file paths
     storeFiles.clear();
-    tableDir = FSUtils.getTableDir(getRootDir(), tableName);
-    FSVisitor.visitTableStoreFiles(getFileSystem(), tableDir, new FSVisitor.StoreFileVisitor() {
-      @Override
-      public void storeFile(final String region, final String family, final String hfile)
+    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().visitStoreFiles(
+        FsContext.DATA, tableName, new StoreFileVisitor() {
+      public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
           throws IOException {
-        HFileLink link = HFileLink.build(UTIL.getConfiguration(), tableName, region, family, hfile);
-        storeFiles.add(link.getOriginPath());
+        storeFiles.add(storeFile.getPath());
       }
     });
     assertTrue("Expected at least " + NUM_FILES + " store files", storeFiles.size() >= NUM_FILES);

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 7fbcfea..c318fe4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -29,6 +29,7 @@ import static org.junit.Assert.fail;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.Collection;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -75,6 +76,7 @@ import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterRpcServices;
@@ -767,12 +769,8 @@ public class TestSplitTransactionOnCluster {
     try {
       // Precondition: we created a table with no data, no store files.
       printOutRegions(regionServer, "Initial regions: ");
-      Configuration conf = cluster.getConfiguration();
-      HBaseFsck.debugLsr(conf, new Path("/"));
-      Path rootDir = FSUtils.getRootDir(conf);
-      FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
-      Map<String, Path> storefiles =
-          FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
+      cluster.getMaster().getMasterFileSystem().logFSTree(LOG);
+      List<StoreFileInfo> storefiles = getStoreFiles(tableName);
       assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
 
       // find a splittable region.  Refresh the regions list
@@ -795,9 +793,8 @@ public class TestSplitTransactionOnCluster {
       assertTrue(daughters.size() == 2);
 
       // check dirs
-      HBaseFsck.debugLsr(conf, new Path("/"));
-      Map<String, Path> storefilesAfter =
-          FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
+      cluster.getMaster().getMasterFileSystem().logFSTree(LOG);
+      List<StoreFileInfo> storefilesAfter = getStoreFiles(tableName);
       assertEquals("Expected nothing but found " + storefilesAfter.toString(),
           storefilesAfter.size(), 0);
 
@@ -1370,5 +1367,17 @@ public class TestSplitTransactionOnCluster {
       }
     }
   }
+
+  private List<StoreFileInfo> getStoreFiles(TableName table) throws IOException {
+    final ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>();
+    cluster.getMaster().getMasterFileSystem().visitStoreFiles(table, new StoreFileVisitor() {
+      @Override
+      public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
+          throws IOException {
+        storeFiles.add(storeFile);
+      }
+    });
+    return storeFiles;
+  }
 }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index 958278d..7ca5d36 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -70,7 +70,6 @@ import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSVisitor;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.MD5Hash;
 import org.junit.Assert;

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java
deleted file mode 100644
index 4a870f8..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.util;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.UUID;
-import java.util.Set;
-import java.util.HashSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.testclassification.MiscTests;
-import org.junit.*;
-import org.junit.experimental.categories.Category;
-
-/**
- * Test {@link FSUtils}.
- */
-@Category({MiscTests.class, MediumTests.class})
-public class TestFSVisitor {
-  private static final Log LOG = LogFactory.getLog(TestFSVisitor.class);
-
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
-  private final String TABLE_NAME = "testtb";
-
-  private Set<String> tableFamilies;
-  private Set<String> tableRegions;
-  private Set<String> tableHFiles;
-
-  private FileSystem fs;
-  private Path tableDir;
-  private Path rootDir;
-
-  @Before
-  public void setUp() throws Exception {
-    fs = FileSystem.get(TEST_UTIL.getConfiguration());
-    rootDir = TEST_UTIL.getDataTestDir("hbase");
-
-    tableFamilies = new HashSet<String>();
-    tableRegions = new HashSet<String>();
-    tableHFiles = new HashSet<String>();
-    tableDir = createTableFiles(rootDir, TABLE_NAME, tableRegions, tableFamilies, tableHFiles);
-    FSUtils.logFileSystemState(fs, rootDir, LOG);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    fs.delete(rootDir, true);
-  }
-
-  @Test
-  public void testVisitStoreFiles() throws IOException {
-    final Set<String> regions = new HashSet<String>();
-    final Set<String> families = new HashSet<String>();
-    final Set<String> hfiles = new HashSet<String>();
-    FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
-      public void storeFile(final String region, final String family, final String hfileName)
-          throws IOException {
-        regions.add(region);
-        families.add(family);
-        hfiles.add(hfileName);
-      }
-    });
-    assertEquals(tableRegions, regions);
-    assertEquals(tableFamilies, families);
-    assertEquals(tableHFiles, hfiles);
-  }
-
-  /*
-   * |-testtb/
-   * |----f1d3ff8443297732862df21dc4e57262/
-   * |-------f1/
-   * |----------d0be84935ba84b66b1e866752ec5d663
-   * |----------9fc9d481718f4878b29aad0a597ecb94
-   * |-------f2/
-   * |----------4b0fe6068c564737946bcf4fd4ab8ae1
-   */
-  private Path createTableFiles(final Path rootDir, final String tableName,
-      final Set<String> tableRegions, final Set<String> tableFamilies,
-      final Set<String> tableHFiles) throws IOException {
-    Path tableDir = new Path(rootDir, tableName);
-    for (int r = 0; r < 10; ++r) {
-      String regionName = MD5Hash.getMD5AsHex(Bytes.toBytes(r));
-      tableRegions.add(regionName);
-      Path regionDir = new Path(tableDir, regionName);
-      for (int f = 0; f < 3; ++f) {
-        String familyName = "f" + f;
-        tableFamilies.add(familyName);
-        Path familyDir = new Path(regionDir, familyName);
-        fs.mkdirs(familyDir);
-        for (int h = 0; h < 5; ++h) {
-         String hfileName = UUID.randomUUID().toString().replaceAll("-", "");
-         tableHFiles.add(hfileName);
-         fs.createNewFile(new Path(familyDir, hfileName));
-        }
-      }
-    }
-    return tableDir;
-  }
-}


[14/18] hbase git commit: HBASE-14439 Move fs stuff out pt1 - master fs + table desc

Posted by bu...@apache.org.
HBASE-14439 Move fs stuff out pt1 - master fs + table desc


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/419800f8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/419800f8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/419800f8

Branch: refs/heads/hbase-14439
Commit: 419800f8d8f721050244398e724c1df82285b8ce
Parents: 7639671
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Tue Oct 6 16:02:20 2015 -0700
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Sep 29 11:07:29 2016 -0500

----------------------------------------------------------------------
 .../apache/hadoop/hbase/HTableDescriptor.java   |  63 +++
 .../SplitLogManagerCoordination.java            |   4 +-
 .../org/apache/hadoop/hbase/fs/FsContext.java   |  28 +
 .../hadoop/hbase/fs/MasterFileSystem.java       | 304 +++++++++++
 .../hadoop/hbase/fs/legacy/LegacyLayout.java    |  73 +++
 .../hbase/fs/legacy/LegacyMasterFileSystem.java | 385 ++++++++++++++
 .../hbase/fs/legacy/LegacyTableDescriptor.java  | 399 +++++++++++++++
 .../org/apache/hadoop/hbase/master/HMaster.java |   4 +-
 .../hadoop/hbase/master/MasterFileSystem.java   | 423 ---------------
 .../hadoop/hbase/master/MasterServices.java     |   1 +
 .../hadoop/hbase/master/MasterWalManager.java   |   2 +-
 .../procedure/CloneSnapshotProcedure.java       |   6 +-
 .../procedure/CreateNamespaceProcedure.java     |   5 +-
 .../master/procedure/CreateTableProcedure.java  |  11 +-
 .../procedure/DeleteNamespaceProcedure.java     |  20 +-
 .../master/procedure/DeleteTableProcedure.java  |   2 +-
 .../procedure/MasterDDLOperationHelper.java     |   2 +-
 .../procedure/RestoreSnapshotProcedure.java     |   2 +-
 .../hbase/master/snapshot/SnapshotManager.java  |   2 +-
 .../hbase/regionserver/CompactionTool.java      |  10 +-
 .../hadoop/hbase/snapshot/SnapshotManifest.java |   7 +-
 .../hadoop/hbase/util/FSTableDescriptors.java   | 510 ++-----------------
 .../org/apache/hadoop/hbase/util/FSUtils.java   |  33 +-
 .../org/apache/hadoop/hbase/util/HBaseFsck.java |  10 +-
 .../hadoop/hbase/util/HFileArchiveUtil.java     |   4 +-
 .../org/apache/hadoop/hbase/util/HMerge.java    |   3 +-
 .../org/apache/hadoop/hbase/util/Merge.java     |   3 +-
 .../org/apache/hadoop/hbase/util/MetaUtils.java |  16 +-
 .../TestHColumnDescriptorDefaultVersions.java   |   5 +-
 .../client/TestRestoreSnapshotFromClient.java   |   2 +-
 .../hbase/client/TestSnapshotFromClient.java    |   7 +-
 .../hadoop/hbase/master/TestCatalogJanitor.java |   1 +
 .../hbase/master/TestMasterFileSystem.java      |   1 +
 ...stTableDescriptorModificationFromClient.java |   7 +-
 .../master/snapshot/TestSnapshotManager.java    |   2 +-
 .../hbase/namespace/TestNamespaceAuditor.java   |  18 +-
 .../hbase/regionserver/TestDefaultMemStore.java |   2 +-
 .../hbase/regionserver/TestStoreFile.java       |   3 +-
 .../hbase/snapshot/SnapshotTestingUtils.java    |  10 +-
 .../hbase/util/TestFSTableDescriptors.java      |  71 +--
 .../hadoop/hbase/util/TestHBaseFsckOneRS.java   |  12 +-
 41 files changed, 1441 insertions(+), 1032 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 9abdf42..867dc0d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -34,6 +34,7 @@ import java.util.regex.Matcher;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
@@ -42,6 +43,7 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema;
+import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -1567,4 +1569,65 @@ public class HTableDescriptor implements Comparable<HTableDescriptor> {
   public void removeConfiguration(final String key) {
     configuration.remove(key);
   }
+
+  @InterfaceAudience.Private
+  public static HTableDescriptor metaTableDescriptor(final Configuration conf)
+      throws IOException {
+    HTableDescriptor metaDescriptor = new HTableDescriptor(
+        TableName.META_TABLE_NAME,
+        new HColumnDescriptor[] {
+            new HColumnDescriptor(HConstants.CATALOG_FAMILY)
+                .setMaxVersions(conf.getInt(HConstants.HBASE_META_VERSIONS,
+                    HConstants.DEFAULT_HBASE_META_VERSIONS))
+                .setInMemory(true)
+                .setBlocksize(conf.getInt(HConstants.HBASE_META_BLOCK_SIZE,
+                    HConstants.DEFAULT_HBASE_META_BLOCK_SIZE))
+                .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+                    // Disable blooms for meta.  Needs work.  Seems to mess w/ getClosestOrBefore.
+                .setBloomFilterType(BloomType.NONE)
+                    // Enable cache of data blocks in L1 if more than one caching tier deployed:
+                    // e.g. if using CombinedBlockCache (BucketCache).
+                .setCacheDataInL1(true),
+            new HColumnDescriptor(HConstants.REPLICATION_BARRIER_FAMILY)
+                .setMaxVersions(conf.getInt(HConstants.HBASE_META_VERSIONS,
+                    HConstants.DEFAULT_HBASE_META_VERSIONS))
+                .setInMemory(true)
+                .setBlocksize(conf.getInt(HConstants.HBASE_META_BLOCK_SIZE,
+                    HConstants.DEFAULT_HBASE_META_BLOCK_SIZE))
+                .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+                // Disable blooms for meta.  Needs work.  Seems to mess w/ getClosestOrBefore.
+                .setBloomFilterType(BloomType.NONE)
+                // Enable cache of data blocks in L1 if more than one caching tier deployed:
+                // e.g. if using CombinedBlockCache (BucketCache).
+                .setCacheDataInL1(true),
+            new HColumnDescriptor(HConstants.REPLICATION_POSITION_FAMILY)
+                .setMaxVersions(conf.getInt(HConstants.HBASE_META_VERSIONS,
+                    HConstants.DEFAULT_HBASE_META_VERSIONS))
+                .setInMemory(true)
+                .setBlocksize(conf.getInt(HConstants.HBASE_META_BLOCK_SIZE,
+                    HConstants.DEFAULT_HBASE_META_BLOCK_SIZE))
+                .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+                // Disable blooms for meta.  Needs work.  Seems to mess w/ getClosestOrBefore.
+                .setBloomFilterType(BloomType.NONE)
+                // Enable cache of data blocks in L1 if more than one caching tier deployed:
+                // e.g. if using CombinedBlockCache (BucketCache).
+                .setCacheDataInL1(true),
+            new HColumnDescriptor(HConstants.TABLE_FAMILY)
+                // Ten is arbitrary number.  Keep versions to help debugging.
+                .setMaxVersions(10)
+                .setInMemory(true)
+                .setBlocksize(8 * 1024)
+                .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+                    // Disable blooms for meta.  Needs work.  Seems to mess w/ getClosestOrBefore.
+                .setBloomFilterType(BloomType.NONE)
+                    // Enable cache of data blocks in L1 if more than one caching tier deployed:
+                    // e.g. if using CombinedBlockCache (BucketCache).
+                .setCacheDataInL1(true)
+        }) {
+    };
+    metaDescriptor.addCoprocessor(
+        "org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint",
+        null, Coprocessor.PRIORITY_SYSTEM, null);
+    return metaDescriptor;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
index 3b1bbb7..3afce96 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
@@ -42,11 +42,11 @@ import com.google.common.annotations.VisibleForTesting;
  * <P>
  * Methods required for task life circle: <BR>
  * {@link #markRegionsRecovering(ServerName, Set)} mark regions for log replaying. Used by
- * {@link org.apache.hadoop.hbase.master.MasterFileSystem} <BR>
+ * {@link org.apache.hadoop.hbase.fs.MasterFileSystem} <BR>
  * {@link #removeRecoveringRegions(Set, Boolean)} make regions cleanup that previous were marked as
  * recovering. Called after all tasks processed <BR>
  * {@link #removeStaleRecoveringRegions(Set)} remove stale recovering. called by
- * {@link org.apache.hadoop.hbase.master.MasterFileSystem} after Active Master is initialized <BR>
+ * {@link org.apache.hadoop.hbase.fs.MasterFileSystem} after Active Master is initialized <BR>
  * {@link #getLastRecoveryTime()} required for garbage collector and should indicate when the last
  * recovery has been made<BR>
  * {@link #checkTaskStillAvailable(String)} Check that task is still there <BR>

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java
new file mode 100644
index 0000000..ced095d
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java
@@ -0,0 +1,28 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+public enum FsContext {
+  TEMP,
+  DATA,
+  ARCHIVE,
+  SNAPSHOT,
+  SIDELINE,
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
new file mode 100644
index 0000000..68a588e
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
@@ -0,0 +1,304 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.ClusterId;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.legacy.LegacyMasterFileSystem;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+
+@InterfaceAudience.Private
+public abstract class MasterFileSystem {
+  private static Log LOG = LogFactory.getLog(MasterFileSystem.class);
+
+  // Persisted unique cluster ID
+  private ClusterId clusterId;
+
+
+  private Configuration conf;
+  private FileSystem fs;
+  private Path rootDir;
+
+  protected MasterFileSystem(Configuration conf, FileSystem fs, Path rootDir) {
+    this.rootDir = rootDir;
+    this.conf = conf;
+    this.fs = fs;
+  }
+
+  public Configuration getConfiguration() { return conf; }
+  public FileSystem getFileSystem() { return fs; }
+  public Path getRootDir() { return rootDir; }
+
+  // ==========================================================================
+  //  PUBLIC Interfaces - Visitors
+  // ==========================================================================
+  public interface NamespaceVisitor {
+    void visitNamespace(String namespace) throws IOException;
+  }
+
+  public interface TableVisitor {
+    void visitTable(TableName tableName) throws IOException;
+  }
+
+  public interface RegionVisitor {
+    void visitRegion(HRegionInfo regionInfo) throws IOException;
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - Namespace related
+  // ==========================================================================
+  public abstract void createNamespace(NamespaceDescriptor nsDescriptor) throws IOException;
+  public abstract void deleteNamespace(String namespaceName) throws IOException;
+  public abstract Collection<String> getNamespaces(FsContext ctx) throws IOException;
+
+  public Collection<String> getNamespaces() throws IOException {
+    return getNamespaces(FsContext.DATA);
+  }
+  // should return or get a NamespaceDescriptor? how is that different from HTD?
+
+  // ==========================================================================
+  //  PUBLIC Methods - Table Descriptor related
+  // ==========================================================================
+  public HTableDescriptor getTableDescriptor(TableName tableName)
+      throws IOException {
+    return getTableDescriptor(FsContext.DATA, tableName);
+  }
+
+  public boolean createTableDescriptor(HTableDescriptor tableDesc, boolean force)
+      throws IOException {
+    return createTableDescriptor(FsContext.DATA, tableDesc, force);
+  }
+
+  public void updateTableDescriptor(HTableDescriptor tableDesc) throws IOException {
+    updateTableDescriptor(FsContext.DATA, tableDesc);
+  }
+
+  public abstract HTableDescriptor getTableDescriptor(FsContext ctx, TableName tableName)
+      throws IOException;
+  public abstract boolean createTableDescriptor(FsContext ctx, HTableDescriptor tableDesc,
+      boolean force) throws IOException;
+  public abstract void updateTableDescriptor(FsContext ctx, HTableDescriptor tableDesc)
+      throws IOException;
+
+  // ==========================================================================
+  //  PUBLIC Methods - Table related
+  // ==========================================================================
+  public void deleteTable(TableName tableName) throws IOException {
+    deleteTable(FsContext.DATA, tableName);
+  }
+
+  public Collection<TableName> getTables(String namespace) throws IOException {
+    return getTables(FsContext.DATA, namespace);
+  }
+
+  public abstract void deleteTable(FsContext ctx, TableName tableName) throws IOException;
+
+  public abstract Collection<TableName> getTables(FsContext ctx, String namespace)
+    throws IOException;
+
+  public Collection<TableName> getTables() throws IOException {
+    ArrayList<TableName> tables = new ArrayList<TableName>();
+    for (String ns: getNamespaces()) {
+      tables.addAll(getTables(ns));
+    }
+    return tables;
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - bootstrap
+  // ==========================================================================
+  public abstract Path getTempDir();
+
+  public void logFileSystemState(Log log) throws IOException {
+    FSUtils.logFileSystemState(getFileSystem(), getRootDir(), LOG);
+  }
+
+  /**
+   * @return The unique identifier generated for this cluster
+   */
+  public ClusterId getClusterId() {
+    return clusterId;
+  }
+
+  protected void bootstrap() throws IOException {
+    // check if the root directory exists
+    createInitialLayout(getRootDir(), conf, this.fs);
+
+    // check if temp directory exists and clean it
+    startupCleanup();
+  }
+
+  protected abstract void bootstrapMeta() throws IOException;
+  protected abstract void startupCleanup() throws IOException;
+
+  /**
+   * Create initial layout in filesystem.
+   * <ol>
+   * <li>Check if the meta region exists and is readable, if not create it.
+   * Create hbase.version and the hbase:meta directory if not one.
+   * </li>
+   * <li>Create a log archive directory for RS to put archived logs</li>
+   * </ol>
+   * Idempotent.
+   */
+  private void createInitialLayout(final Path rd, final Configuration c, final FileSystem fs)
+      throws IOException {
+    // If FS is in safe mode wait till out of it.
+    FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
+
+    // Filesystem is good. Go ahead and check for hbase.rootdir.
+    try {
+      if (!fs.exists(rd)) {
+        fs.mkdirs(rd);
+        // DFS leaves safe mode with 0 DNs when there are 0 blocks.
+        // We used to handle this by checking the current DN count and waiting until
+        // it is nonzero. With security, the check for datanode count doesn't work --
+        // it is a privileged op. So instead we adopt the strategy of the jobtracker
+        // and simply retry file creation during bootstrap indefinitely. As soon as
+        // there is one datanode it will succeed. Permission problems should have
+        // already been caught by mkdirs above.
+        FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+          10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
+            HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
+      } else {
+        if (!fs.isDirectory(rd)) {
+          throw new IllegalArgumentException(rd.toString() + " is not a directory");
+        }
+        // as above
+        FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+          10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
+            HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
+      }
+    } catch (DeserializationException de) {
+      LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
+      IOException ioe = new IOException();
+      ioe.initCause(de);
+      throw ioe;
+    } catch (IllegalArgumentException iae) {
+      LOG.fatal("Please fix invalid configuration for "
+        + HConstants.HBASE_DIR + " " + rd.toString(), iae);
+      throw iae;
+    }
+    // Make sure cluster ID exists
+    if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
+        HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
+      FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
+    }
+    clusterId = FSUtils.getClusterId(fs, rd);
+
+    // Make sure the meta region exists!
+    bootstrapMeta();
+  }
+
+  // TODO: Move in HRegionFileSystem
+  public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName, boolean hasMob)
+      throws IOException {
+    // archive family store files
+    Path tableDir = FSUtils.getTableDir(getRootDir(), region.getTable());
+    HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
+
+    // delete the family folder
+    Path familyDir = new Path(tableDir,
+      new Path(region.getEncodedName(), Bytes.toString(familyName)));
+    if (fs.delete(familyDir, true) == false) {
+      if (fs.exists(familyDir)) {
+        throw new IOException("Could not delete family "
+            + Bytes.toString(familyName) + " from FileSystem for region "
+            + region.getRegionNameAsString() + "(" + region.getEncodedName()
+            + ")");
+      }
+    }
+
+    // archive and delete mob files
+    if (hasMob) {
+      Path mobTableDir =
+          FSUtils.getTableDir(new Path(getRootDir(), MobConstants.MOB_DIR_NAME), region.getTable());
+      HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(region.getTable());
+      Path mobFamilyDir =
+          new Path(mobTableDir,
+              new Path(mobRegionInfo.getEncodedName(), Bytes.toString(familyName)));
+      // archive mob family store files
+      MobUtils.archiveMobStoreFiles(conf, fs, mobRegionInfo, mobFamilyDir, familyName);
+
+      if (!fs.delete(mobFamilyDir, true)) {
+        throw new IOException("Could not delete mob store files for family "
+            + Bytes.toString(familyName) + " from FileSystem region "
+            + mobRegionInfo.getRegionNameAsString() + "(" + mobRegionInfo.getEncodedName() + ")");
+      }
+    }
+  }
+
+  // ==========================================================================
+  //  PUBLIC
+  // ==========================================================================
+  public static MasterFileSystem open(Configuration conf, boolean bootstrap)
+      throws IOException {
+    return open(conf, FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), bootstrap);
+  }
+
+  public static MasterFileSystem open(Configuration conf, FileSystem fs,
+      Path rootDir, boolean bootstrap) throws IOException {
+    // Cover both bases, the old way of setting default fs and the new.
+    // We're supposed to run on 0.20 and 0.21 anyways.
+    fs = rootDir.getFileSystem(conf);
+    FSUtils.setFsDefault(conf, new Path(fs.getUri()));
+    // make sure the fs has the same conf
+    fs.setConf(conf);
+
+    MasterFileSystem mfs = getInstance(conf, fs, rootDir);
+    if (bootstrap) {
+      mfs.bootstrap();
+    }
+    HFileSystem.addLocationsOrderInterceptor(conf);
+    return mfs;
+  }
+
+  private static MasterFileSystem getInstance(Configuration conf, final FileSystem fs,
+      Path rootDir) throws IOException {
+    String fsType = conf.get("hbase.fs.layout.type", "legacy").toLowerCase();
+    switch (fsType) {
+      case "legacy":
+        return new LegacyMasterFileSystem(conf, fs, rootDir);
+      default:
+        throw new IOException("Invalid filesystem type " + fsType);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
new file mode 100644
index 0000000..8f54bae
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
@@ -0,0 +1,73 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs.legacy;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+
+public final class LegacyLayout {
+  /** Name of the region info file that resides just under the region directory. */
+  public final static String REGION_INFO_FILE = ".regioninfo";
+
+  private LegacyLayout() {}
+
+  public static Path getDataDir(final Path rootDir) {
+    return new Path(rootDir, HConstants.BASE_NAMESPACE_DIR);
+  }
+
+  public static Path getSidelineDir(Path rootDir) {
+    return new Path(rootDir, HConstants.HBCK_SIDELINEDIR_NAME);
+  }
+
+  public static Path getSnapshotDir(Path rootDir) {
+    return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
+  }
+
+  public static Path getSnapshotDir(Path baseSnapshotDir, String snapshotName) {
+    return new Path(baseSnapshotDir, snapshotName);
+  }
+
+  public static Path getArchiveDir(Path rootDir) {
+    return new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+  }
+
+  public static Path getTempDir(Path rootDir) {
+    return new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY);
+  }
+
+  public static Path getNamespaceDir(Path baseDataDir, String namespace) {
+    return new Path(baseDataDir, namespace);
+  }
+
+  public static Path getTableDir(Path baseDataDir, TableName table) {
+    Path nsDir = getNamespaceDir(baseDataDir, table.getNamespaceAsString());
+    return new Path(nsDir, table.getQualifierAsString());
+  }
+
+  public static Path getRegionDir(Path baseDataDir, TableName table, HRegionInfo hri) {
+    return new Path(getTableDir(baseDataDir, table), hri.getEncodedName());
+  }
+
+  public static Path getBulkDir(Path rootDir) {
+    return new Path(rootDir, HConstants.BULKLOAD_STAGING_DIR_NAME);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
new file mode 100644
index 0000000..992dce1
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
@@ -0,0 +1,385 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs.legacy;
+
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.FsContext;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.MetaUtils;
+
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+
+@InterfaceAudience.Private
+public class LegacyMasterFileSystem extends MasterFileSystem {
+  private static final Log LOG = LogFactory.getLog(LegacyMasterFileSystem.class);
+
+  private final Path sidelineDir;
+  private final Path snapshotDir;
+  private final Path archiveDataDir;
+  private final Path archiveDir;
+  private final Path tmpDataDir;
+  private final Path dataDir;
+  private final Path tmpDir;
+  private final Path bulkDir;
+
+  /*
+   * In a secure env, the protected sub-directories and files under the HBase rootDir
+   * would be restricted. The sub-directory will have '700' except the bulk load staging dir,
+   * which will have '711'.  The default '700' can be overwritten by setting the property
+   * 'hbase.rootdir.perms'. The protected files (version file, clusterId file) will have '600'.
+   * The rootDir itself will be created with HDFS default permissions if it does not exist.
+   * We will check the rootDir permissions to make sure it has 'x' for all to ensure access
+   * to the staging dir. If it does not, we will add it.
+   */
+  // Permissions for the directories under rootDir that need protection
+  private final FsPermission secureRootSubDirPerms;
+  // Permissions for the files under rootDir that need protection
+  private final FsPermission secureRootFilePerms = new FsPermission("600");
+  // Permissions for bulk load staging directory under rootDir
+  private final FsPermission HiddenDirPerms = FsPermission.valueOf("-rwx--x--x");
+
+  private final boolean isSecurityEnabled;
+
+  public LegacyMasterFileSystem(Configuration conf, FileSystem fs, Path rootDir) {
+    super(conf, fs, rootDir);
+
+    // base directories
+    this.sidelineDir = LegacyLayout.getSidelineDir(rootDir);
+    this.snapshotDir = LegacyLayout.getSnapshotDir(rootDir);
+    this.archiveDir = LegacyLayout.getArchiveDir(rootDir);
+    this.archiveDataDir = LegacyLayout.getDataDir(this.archiveDir);
+    this.dataDir = LegacyLayout.getDataDir(rootDir);
+    this.tmpDir = LegacyLayout.getTempDir(rootDir);
+    this.tmpDataDir = LegacyLayout.getDataDir(this.tmpDir);
+    this.bulkDir = LegacyLayout.getBulkDir(rootDir);
+
+    this.secureRootSubDirPerms = new FsPermission(conf.get("hbase.rootdir.perms", "700"));
+    this.isSecurityEnabled = "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - Namespace related
+  // ==========================================================================
+  public void createNamespace(NamespaceDescriptor nsDescriptor) throws IOException {
+    getFileSystem().mkdirs(getNamespaceDir(FsContext.DATA, nsDescriptor.getName()));
+  }
+
+  public void deleteNamespace(String namespaceName) throws IOException {
+    FileSystem fs = getFileSystem();
+    Path nsDir = getNamespaceDir(FsContext.DATA, namespaceName);
+
+    try {
+      for (FileStatus status : fs.listStatus(nsDir)) {
+        if (!HConstants.HBASE_NON_TABLE_DIRS.contains(status.getPath().getName())) {
+          throw new IOException("Namespace directory contains table dir: " + status.getPath());
+        }
+      }
+      if (!fs.delete(nsDir, true)) {
+        throw new IOException("Failed to remove namespace: " + namespaceName);
+      }
+    } catch (FileNotFoundException e) {
+      // File already deleted, continue
+      LOG.debug("deleteDirectory throws exception: " + e);
+    }
+  }
+
+  public Collection<String> getNamespaces(FsContext ctx) throws IOException {
+    FileStatus[] stats = FSUtils.listStatus(getFileSystem(), getNamespaceDir(ctx));
+    if (stats == null) return Collections.emptyList();
+
+    ArrayList<String> namespaces = new ArrayList<String>(stats.length);
+    for (int i = 0; i < stats.length; ++i) {
+      namespaces.add(stats[i].getPath().getName());
+    }
+    return namespaces;
+  }
+
+  // should return or get a NamespaceDescriptor? how is that different from HTD?
+
+  // ==========================================================================
+  //  PUBLIC Methods - Table Descriptor related
+  // ==========================================================================s
+  @Override
+  public boolean createTableDescriptor(FsContext ctx, HTableDescriptor tableDesc, boolean force)
+      throws IOException {
+    return LegacyTableDescriptor.createTableDescriptor(getFileSystem(),
+      getTableDir(ctx, tableDesc.getTableName()), tableDesc, force);
+  }
+
+  @Override
+  public void updateTableDescriptor(FsContext ctx, HTableDescriptor tableDesc) throws IOException {
+    LegacyTableDescriptor.updateTableDescriptor(getFileSystem(),
+        getTableDir(ctx, tableDesc.getTableName()), tableDesc);
+  }
+
+  @Override
+  public HTableDescriptor getTableDescriptor(FsContext ctx, TableName tableName)
+      throws IOException {
+    return LegacyTableDescriptor.getTableDescriptorFromFs(
+        getFileSystem(), getTableDir(ctx, tableName));
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - Table related
+  // ==========================================================================
+  @Override
+  public void deleteTable(FsContext ctx, TableName tableName) throws IOException {
+    Path tableDir = getTableDir(ctx, tableName);
+    if (!FSUtils.deleteDirectory(getFileSystem(), tableDir)) {
+      throw new IOException("Failed delete of " + tableName);
+    }
+  }
+
+  @Override
+  public Collection<TableName> getTables(FsContext ctx, String namespace)
+      throws IOException {
+    FileStatus[] stats = FSUtils.listStatus(getFileSystem(),
+        getNamespaceDir(ctx, namespace), new FSUtils.UserTableDirFilter(getFileSystem()));
+    if (stats == null) return Collections.emptyList();
+
+    ArrayList<TableName> tables = new ArrayList<TableName>(stats.length);
+    for (int i = 0; i < stats.length; ++i) {
+      tables.add(TableName.valueOf(namespace, stats[i].getPath().getName()));
+    }
+    return tables;
+  }
+
+  // ==========================================================================
+  //  PROTECTED Methods - Bootstrap
+  // ==========================================================================
+  @Override
+  protected void bootstrapMeta() throws IOException {
+    if (!FSUtils.metaRegionExists(getFileSystem(), getRootDir())) {
+      bootstrapMeta(getRootDir(), getConfiguration());
+    }
+
+    // Create tableinfo-s for hbase:meta if not already there.
+    // assume, created table descriptor is for enabling table
+    // meta table is a system table, so descriptors are predefined,
+    // we should get them from registry.
+    createTableDescriptor(HTableDescriptor.metaTableDescriptor(getConfiguration()), false);
+  }
+
+  private static void bootstrapMeta(final Path rd, final Configuration c) throws IOException {
+    LOG.info("BOOTSTRAP: creating hbase:meta region");
+    try {
+      // Bootstrapping, make sure blockcache is off.  Else, one will be
+      // created here in bootstrap and it'll need to be cleaned up.  Better to
+      // not make it in first place.  Turn off block caching for bootstrap.
+      // Enable after.
+      HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
+      HTableDescriptor metaDescriptor = HTableDescriptor.metaTableDescriptor(c);
+      MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, false);
+      HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor, null);
+      MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, true);
+      meta.close();
+    } catch (IOException e) {
+        e = e instanceof RemoteException ?
+                ((RemoteException)e).unwrapRemoteException() : e;
+      LOG.error("bootstrap", e);
+      throw e;
+    }
+  }
+
+  @Override
+  protected void startupCleanup() throws IOException {
+    final FileSystem fs = getFileSystem();
+    // Check the directories under rootdir.
+    checkTempDir(getTempDir(), getConfiguration(), fs);
+    final String[] protectedSubDirs = new String[] {
+        HConstants.BASE_NAMESPACE_DIR,
+        HConstants.HFILE_ARCHIVE_DIRECTORY,
+        HConstants.HREGION_LOGDIR_NAME,
+        HConstants.HREGION_OLDLOGDIR_NAME,
+        MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR,
+        HConstants.CORRUPT_DIR_NAME,
+        HConstants.HBCK_SIDELINEDIR_NAME,
+        MobConstants.MOB_DIR_NAME
+    };
+    for (String subDir : protectedSubDirs) {
+      checkSubDir(new Path(getRootDir(), subDir));
+    }
+
+    checkStagingDir();
+
+    // Handle the last few special files and set the final rootDir permissions
+    // rootDir needs 'x' for all to support bulk load staging dir
+    if (isSecurityEnabled) {
+      fs.setPermission(new Path(getRootDir(), HConstants.VERSION_FILE_NAME), secureRootFilePerms);
+      fs.setPermission(new Path(getRootDir(), HConstants.CLUSTER_ID_FILE_NAME), secureRootFilePerms);
+    }
+    FsPermission currentRootPerms = fs.getFileStatus(getRootDir()).getPermission();
+    if (!currentRootPerms.getUserAction().implies(FsAction.EXECUTE)
+        || !currentRootPerms.getGroupAction().implies(FsAction.EXECUTE)
+        || !currentRootPerms.getOtherAction().implies(FsAction.EXECUTE)) {
+      LOG.warn("rootdir permissions do not contain 'excute' for user, group or other. "
+        + "Automatically adding 'excute' permission for all");
+      fs.setPermission(
+        getRootDir(),
+        new FsPermission(currentRootPerms.getUserAction().or(FsAction.EXECUTE), currentRootPerms
+            .getGroupAction().or(FsAction.EXECUTE), currentRootPerms.getOtherAction().or(
+          FsAction.EXECUTE)));
+    }
+  }
+
+  /**
+   * Make sure the hbase temp directory exists and is empty.
+   * NOTE that this method is only executed once just after the master becomes the active one.
+   */
+  private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
+      throws IOException {
+    // If the temp directory exists, clear the content (left over, from the previous run)
+    if (fs.exists(tmpdir)) {
+      // Archive table in temp, maybe left over from failed deletion,
+      // if not the cleaner will take care of them.
+      for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
+        for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
+          HFileArchiver.archiveRegion(fs, getRootDir(), tabledir, regiondir);
+        }
+      }
+      if (!fs.delete(tmpdir, true)) {
+        throw new IOException("Unable to clean the temp directory: " + tmpdir);
+      }
+    }
+
+    // Create the temp directory
+    if (isSecurityEnabled) {
+      if (!fs.mkdirs(tmpdir, secureRootSubDirPerms)) {
+        throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
+      }
+    } else {
+      if (!fs.mkdirs(tmpdir)) {
+        throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
+      }
+    }
+  }
+
+  /**
+   * Make sure the directories under rootDir have good permissions. Create if necessary.
+   * @param p
+   * @throws IOException
+   */
+  private void checkSubDir(final Path p) throws IOException {
+    final FileSystem fs = getFileSystem();
+    if (!fs.exists(p)) {
+      if (isSecurityEnabled) {
+        if (!fs.mkdirs(p, secureRootSubDirPerms)) {
+          throw new IOException("HBase directory '" + p + "' creation failure.");
+        }
+      } else {
+        if (!fs.mkdirs(p)) {
+          throw new IOException("HBase directory '" + p + "' creation failure.");
+        }
+      }
+    }
+    else {
+      if (isSecurityEnabled && !secureRootSubDirPerms.equals(fs.getFileStatus(p).getPermission())) {
+        // check whether the permission match
+        LOG.warn("Found HBase directory permissions NOT matching expected permissions for "
+            + p.toString() + " permissions=" + fs.getFileStatus(p).getPermission()
+            + ", expecting " + secureRootSubDirPerms + ". Automatically setting the permissions. "
+            + "You can change the permissions by setting \"hbase.rootdir.perms\" in hbase-site.xml "
+            + "and restarting the master");
+        fs.setPermission(p, secureRootSubDirPerms);
+      }
+    }
+  }
+
+  /**
+   * Check permissions for bulk load staging directory. This directory has special hidden
+   * permissions. Create it if necessary.
+   * @throws IOException
+   */
+  private void checkStagingDir() throws IOException {
+    final FileSystem fs = getFileSystem();
+    Path p = new Path(getRootDir(), HConstants.BULKLOAD_STAGING_DIR_NAME);
+    try {
+      if (!fs.exists(p)) {
+        if (!fs.mkdirs(p, HiddenDirPerms)) {
+          throw new IOException("Failed to create staging directory " + p.toString());
+        }
+      } else {
+        fs.setPermission(p, HiddenDirPerms);
+      }
+    } catch (IOException e) {
+      LOG.error("Failed to create or set permission on staging directory " + p.toString());
+      throw new IOException("Failed to create or set permission on staging directory "
+          + p.toString(), e);
+    }
+  }
+
+  // ==========================================================================
+  //  PROTECTED Methods - Path
+  // ==========================================================================
+  protected Path getNamespaceDir(FsContext ctx) {
+    return getBaseDirFromContext(ctx);
+  }
+
+  protected Path getNamespaceDir(FsContext ctx, String namespace) {
+    return LegacyLayout.getNamespaceDir(getBaseDirFromContext(ctx), namespace);
+  }
+
+  protected Path getTableDir(FsContext ctx, TableName table) {
+    return LegacyLayout.getTableDir(getBaseDirFromContext(ctx), table);
+  }
+
+  protected Path getRegionDir(FsContext ctx, TableName table, HRegionInfo hri) {
+    return LegacyLayout.getRegionDir(getBaseDirFromContext(ctx), table, hri);
+  }
+
+  public Path getTempDir() {
+    return tmpDir;
+  }
+
+  protected Path getBaseDirFromContext(FsContext ctx) {
+    switch (ctx) {
+      case TEMP: return tmpDataDir;
+      case DATA: return dataDir;
+      case ARCHIVE: return archiveDataDir;
+      case SNAPSHOT: return snapshotDir;
+      case SIDELINE: return sidelineDir;
+      default: throw new RuntimeException("Invalid context: " + ctx);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyTableDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyTableDescriptor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyTableDescriptor.java
new file mode 100644
index 0000000..e4d5bc1
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyTableDescriptor.java
@@ -0,0 +1,399 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.fs.legacy;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableDescriptors;
+import org.apache.hadoop.hbase.TableInfoMissingException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.util.*;
+
+/**
+ * Implementation of {@link TableDescriptors} that reads descriptors from the
+ * passed filesystem.  It expects descriptors to be in a file in the
+ * {@link #TABLEINFO_DIR} subdir of the table's directory in FS.  Can be read-only
+ *  -- i.e. does not modify the filesystem or can be read and write.
+ *
+ * <p>Also has utility for keeping up the table descriptors tableinfo file.
+ * The table schema file is kept in the {@link #TABLEINFO_DIR} subdir
+ * of the table directory in the filesystem.
+ * It has a {@link #TABLEINFO_FILE_PREFIX} and then a suffix that is the
+ * edit sequenceid: e.g. <code>.tableinfo.0000000003</code>.  This sequenceid
+ * is always increasing.  It starts at zero.  The table schema file with the
+ * highest sequenceid has the most recent schema edit. Usually there is one file
+ * only, the most recent but there may be short periods where there are more
+ * than one file. Old files are eventually cleaned.  Presumption is that there
+ * will not be lots of concurrent clients making table schema edits.  If so,
+ * the below needs a bit of a reworking and perhaps some supporting api in hdfs.
+ */
+@InterfaceAudience.Private
+public final class LegacyTableDescriptor {
+  private static final Log LOG = LogFactory.getLog(LegacyTableDescriptor.class);
+
+  /** The file name prefix used to store HTD in HDFS  */
+  public static final String TABLEINFO_FILE_PREFIX = ".tableinfo";
+  public static final String TABLEINFO_DIR = ".tabledesc";
+  public static final String TMP_DIR = ".tmp";
+
+  private LegacyTableDescriptor() {}
+
+  /**
+   * Find the most current table info file for the table located in the given table directory.
+   *
+   * Looks within the {@link #TABLEINFO_DIR} subdirectory of the given directory for any table info
+   * files and takes the 'current' one - meaning the one with the highest sequence number if present
+   * or no sequence number at all if none exist (for backward compatibility from before there
+   * were sequence numbers).
+   *
+   * @return The file status of the current table info file or null if it does not exist
+   * @throws IOException
+   */
+  public static FileStatus getTableInfoPath(FileSystem fs, Path tableDir) throws IOException {
+    return getTableInfoPath(fs, tableDir, false);
+  }
+
+  /**
+   * Find the most current table info file for the table in the given table directory.
+   *
+   * Looks within the {@link #TABLEINFO_DIR} subdirectory of the given directory for any table info
+   * files and takes the 'current' one - meaning the one with the highest sequence number if
+   * present or no sequence number at all if none exist (for backward compatibility from before
+   * there were sequence numbers).
+   * If there are multiple table info files found and removeOldFiles is true it also deletes the
+   * older files.
+   *
+   * @return The file status of the current table info file or null if none exist
+   * @throws IOException
+   */
+  private static FileStatus getTableInfoPath(FileSystem fs, Path tableDir, boolean removeOldFiles)
+      throws IOException {
+    Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
+    return getCurrentTableInfoStatus(fs, tableInfoDir, removeOldFiles);
+  }
+
+  /**
+   * Find the most current table info file in the given directory
+   *
+   * Looks within the given directory for any table info files
+   * and takes the 'current' one - meaning the one with the highest sequence number if present
+   * or no sequence number at all if none exist (for backward compatibility from before there
+   * were sequence numbers).
+   * If there are multiple possible files found
+   * and the we're not in read only mode it also deletes the older files.
+   *
+   * @return The file status of the current table info file or null if it does not exist
+   * @throws IOException
+   */
+  // only visible for FSTableDescriptorMigrationToSubdir, can be removed with that
+  public static FileStatus getCurrentTableInfoStatus(FileSystem fs, Path dir, boolean removeOldFiles)
+      throws IOException {
+    FileStatus [] status = FSUtils.listStatus(fs, dir, TABLEINFO_PATHFILTER);
+    if (status == null || status.length < 1) return null;
+    FileStatus mostCurrent = null;
+    for (FileStatus file : status) {
+      if (mostCurrent == null || TABLEINFO_FILESTATUS_COMPARATOR.compare(file, mostCurrent) < 0) {
+        mostCurrent = file;
+      }
+    }
+    if (removeOldFiles && status.length > 1) {
+      // Clean away old versions
+      for (FileStatus file : status) {
+        Path path = file.getPath();
+        if (file != mostCurrent) {
+          if (!fs.delete(path, false)) {
+            LOG.warn("Failed cleanup of " + path);
+          } else {
+            LOG.debug("Cleaned up old tableinfo file " + path);
+          }
+        }
+      }
+    }
+    return mostCurrent;
+  }
+
+  /**
+   * Update table descriptor on the file system
+   * @throws IOException Thrown if failed update.
+   * @throws NotImplementedException if in read only mode
+   */
+  public static void updateTableDescriptor(FileSystem fs, Path tableDir, HTableDescriptor td)
+      throws IOException {
+    TableName tableName = td.getTableName();
+    Path p = writeTableDescriptor(fs, td, tableDir, getTableInfoPath(fs, tableDir));
+    if (p == null) throw new IOException("Failed update");
+    LOG.info("Updated tableinfo=" + p);
+  }
+
+  /**
+   * Compare {@link FileStatus} instances by {@link Path#getName()}. Returns in
+   * reverse order.
+   */
+  @VisibleForTesting
+  public static final Comparator<FileStatus> TABLEINFO_FILESTATUS_COMPARATOR =
+  new Comparator<FileStatus>() {
+    @Override
+    public int compare(FileStatus left, FileStatus right) {
+      return right.compareTo(left);
+    }};
+
+  private static final PathFilter TABLEINFO_PATHFILTER = new PathFilter() {
+    @Override
+    public boolean accept(Path p) {
+      // Accept any file that starts with TABLEINFO_NAME
+      return p.getName().startsWith(TABLEINFO_FILE_PREFIX);
+    }};
+
+  /**
+   * Width of the sequenceid that is a suffix on a tableinfo file.
+   */
+  @VisibleForTesting public static final int WIDTH_OF_SEQUENCE_ID = 10;
+
+  /*
+   * @param number Number to use as suffix.
+   * @return Returns zero-prefixed decimal version of passed
+   * number (Does absolute in case number is negative).
+   */
+  private static String formatTableInfoSequenceId(final int number) {
+    byte [] b = new byte[WIDTH_OF_SEQUENCE_ID];
+    int d = Math.abs(number);
+    for (int i = b.length - 1; i >= 0; i--) {
+      b[i] = (byte)((d % 10) + '0');
+      d /= 10;
+    }
+    return Bytes.toString(b);
+  }
+
+  /**
+   * Regex to eat up sequenceid suffix on a .tableinfo file.
+   * Use regex because may encounter oldstyle .tableinfos where there is no
+   * sequenceid on the end.
+   */
+  private static final Pattern TABLEINFO_FILE_REGEX =
+    Pattern.compile(TABLEINFO_FILE_PREFIX + "(\\.([0-9]{" + WIDTH_OF_SEQUENCE_ID + "}))?$");
+
+  /**
+   * @param p Path to a <code>.tableinfo</code> file.
+   * @return The current editid or 0 if none found.
+   */
+  @VisibleForTesting
+  public static int getTableInfoSequenceId(final Path p) {
+    if (p == null) return 0;
+    Matcher m = TABLEINFO_FILE_REGEX.matcher(p.getName());
+    if (!m.matches()) throw new IllegalArgumentException(p.toString());
+    String suffix = m.group(2);
+    if (suffix == null || suffix.length() <= 0) return 0;
+    return Integer.parseInt(m.group(2));
+  }
+
+  /**
+   * @param sequenceid
+   * @return Name of tableinfo file.
+   */
+  @VisibleForTesting
+  public static String getTableInfoFileName(final int sequenceid) {
+    return TABLEINFO_FILE_PREFIX + "." + formatTableInfoSequenceId(sequenceid);
+  }
+
+  /**
+   * Returns the latest table descriptor for the given table directly from the file system
+   * if it exists, bypassing the local cache.
+   * Returns null if it's not found.
+   */
+  public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs,
+      Path hbaseRootDir, TableName tableName) throws IOException {
+    Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName);
+    return readTableDescriptor(fs, tableDir);
+  }
+
+  /**
+   * Returns the latest table descriptor for the table located at the given directory
+   * directly from the file system if it exists.
+   * @throws TableInfoMissingException if there is no descriptor
+   */
+  public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs, Path tableDir)
+      throws IOException {
+    final HTableDescriptor htd = readTableDescriptor(fs, tableDir);
+    if (htd == null) {
+      throw new TableInfoMissingException("No table descriptor file under " + tableDir);
+    }
+    return htd;
+  }
+
+  /**
+   * @return null if not found
+   */
+  private static HTableDescriptor readTableDescriptor(FileSystem fs, Path tableDir)
+      throws IOException {
+    FileStatus status = getTableInfoPath(fs, tableDir, false);
+    if (status == null) {
+      return null;
+    }
+    return readTableDescriptor(fs, status);
+  }
+
+  private static HTableDescriptor readTableDescriptor(FileSystem fs, FileStatus status)
+      throws IOException {
+    int len = Ints.checkedCast(status.getLen());
+    byte[] content = new byte[len];
+    FSUtils.readFully(fs, status.getPath(), content);
+    try {
+      HTableDescriptor td = HTableDescriptor.parseFrom(content);
+      return td;
+    } catch (DeserializationException exception) {
+      throw new IOException("failed to deserialize descriptor.", exception);
+    }
+  }
+
+  /**
+   * Deletes files matching the table info file pattern within the given directory
+   * whose sequenceId is at most the given max sequenceId.
+   */
+  private static void deleteTableDescriptorFiles(FileSystem fs, Path dir, int maxSequenceId)
+      throws IOException {
+    FileStatus [] status = FSUtils.listStatus(fs, dir, TABLEINFO_PATHFILTER);
+    for (FileStatus file : status) {
+      Path path = file.getPath();
+      int sequenceId = getTableInfoSequenceId(path);
+      if (sequenceId <= maxSequenceId) {
+        boolean success = FSUtils.delete(fs, path, false);
+        if (success) {
+          LOG.debug("Deleted table descriptor at " + path);
+        } else {
+          LOG.error("Failed to delete descriptor at " + path);
+        }
+      }
+    }
+  }
+
+  /**
+   * Attempts to write a new table descriptor to the given table's directory.
+   * It first writes it to the .tmp dir then uses an atomic rename to move it into place.
+   * It begins at the currentSequenceId + 1 and tries 10 times to find a new sequence number
+   * not already in use.
+   * Removes the current descriptor file if passed in.
+   *
+   * @return Descriptor file or null if we failed write.
+   */
+  private static Path writeTableDescriptor(final FileSystem fs,
+    final HTableDescriptor htd, final Path tableDir,
+    final FileStatus currentDescriptorFile) throws IOException {
+    // Get temporary dir into which we'll first write a file to avoid half-written file phenomenon.
+    // This directory is never removed to avoid removing it out from under a concurrent writer.
+    Path tmpTableDir = new Path(tableDir, TMP_DIR);
+    Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
+
+    // What is current sequenceid?  We read the current sequenceid from
+    // the current file.  After we read it, another thread could come in and
+    // compete with us writing out next version of file.  The below retries
+    // should help in this case some but its hard to do guarantees in face of
+    // concurrent schema edits.
+    int currentSequenceId = currentDescriptorFile == null ? 0 :
+      getTableInfoSequenceId(currentDescriptorFile.getPath());
+    int newSequenceId = currentSequenceId;
+
+    // Put arbitrary upperbound on how often we retry
+    int retries = 10;
+    int retrymax = currentSequenceId + retries;
+    Path tableInfoDirPath = null;
+    do {
+      newSequenceId += 1;
+      String filename = getTableInfoFileName(newSequenceId);
+      Path tempPath = new Path(tmpTableDir, filename);
+      if (fs.exists(tempPath)) {
+        LOG.debug(tempPath + " exists; retrying up to " + retries + " times");
+        continue;
+      }
+      tableInfoDirPath = new Path(tableInfoDir, filename);
+      try {
+        writeTD(fs, tempPath, htd);
+        fs.mkdirs(tableInfoDirPath.getParent());
+        if (!fs.rename(tempPath, tableInfoDirPath)) {
+          throw new IOException("Failed rename of " + tempPath + " to " + tableInfoDirPath);
+        }
+        LOG.debug("Wrote descriptor into: " + tableInfoDirPath);
+      } catch (IOException ioe) {
+        // Presume clash of names or something; go around again.
+        LOG.debug("Failed write and/or rename; retrying", ioe);
+        if (!FSUtils.deleteDirectory(fs, tempPath)) {
+          LOG.warn("Failed cleanup of " + tempPath);
+        }
+        tableInfoDirPath = null;
+        continue;
+      }
+      break;
+    } while (newSequenceId < retrymax);
+    if (tableInfoDirPath != null) {
+      // if we succeeded, remove old table info files.
+      deleteTableDescriptorFiles(fs, tableInfoDir, newSequenceId - 1);
+    }
+    return tableInfoDirPath;
+  }
+
+  private static void writeTD(final FileSystem fs, final Path p, final HTableDescriptor htd)
+      throws IOException {
+    // We used to write this file out as a serialized HTD Writable followed by two '\n's and then
+    // the toString version of HTD.  Now we just write out the pb serialization.
+    FSUtils.writeFully(fs, p, htd.toByteArray(), false);
+  }
+
+  /**
+   * Create a new HTableDescriptor in HDFS in the specified table directory. Happens when we create
+   * a new table or snapshot a table.
+   * @param tableDir table directory under which we should write the file
+   * @param htd description of the table to write
+   * @param forceCreation if <tt>true</tt>,then even if previous table descriptor is present it will
+   *          be overwritten
+   * @return <tt>true</tt> if the we successfully created the file, <tt>false</tt> if the file
+   *         already exists and we weren't forcing the descriptor creation.
+   * @throws IOException if a filesystem error occurs
+   */
+  public static boolean createTableDescriptor(FileSystem fs, Path tableDir,
+      HTableDescriptor htd, boolean forceCreation) throws IOException {
+    FileStatus status = getTableInfoPath(fs, tableDir);
+    if (status != null) {
+      LOG.debug("Current tableInfoPath = " + status.getPath());
+      if (!forceCreation) {
+        if (fs.exists(status.getPath()) && status.getLen() > 0) {
+          if (readTableDescriptor(fs, status).equals(htd)) {
+            LOG.debug("TableInfo already exists.. Skipping creation");
+            return false;
+          }
+        }
+      }
+    }
+    Path p = writeTableDescriptor(fs, htd, tableDir, status);
+    return p != null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 810f95b..c9056e3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.exceptions.MergeRegionException;
 import org.apache.hadoop.hbase.executor.ExecutorType;
 import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.ipc.RpcServer;
 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
 import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
@@ -681,7 +682,7 @@ public class HMaster extends HRegionServer implements MasterServices {
 
     this.masterActiveTime = System.currentTimeMillis();
     // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
-    this.fileSystemManager = new MasterFileSystem(this);
+    this.fileSystemManager = MasterFileSystem.open(conf, true);
     this.walManager = new MasterWalManager(this);
 
     // enable table descriptors cache
@@ -1024,7 +1025,6 @@ public class HMaster extends HRegionServer implements MasterServices {
     if (this.serverManager != null) this.serverManager.stop();
     if (this.assignmentManager != null) this.assignmentManager.stop();
     if (this.walManager != null) this.walManager.stop();
-    if (this.fileSystemManager != null) this.fileSystemManager.stop();
     if (this.mpmHost != null) this.mpmHost.stop("server shutting down.");
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
deleted file mode 100644
index a8f81ee..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hbase.ClusterId;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.ipc.RemoteException;
-
-/**
- * This class abstracts a bunch of operations the HMaster needs to interact with
- * the underlying file system like creating the initial layout, checking file
- * system status, etc.
- */
-@InterfaceAudience.Private
-public class MasterFileSystem {
-  private static final Log LOG = LogFactory.getLog(MasterFileSystem.class);
-
-  // HBase configuration
-  private final Configuration conf;
-  // Persisted unique cluster ID
-  private ClusterId clusterId;
-  // Keep around for convenience.
-  private final FileSystem fs;
-  // root hbase directory on the FS
-  private final Path rootdir;
-  // hbase temp directory used for table construction and deletion
-  private final Path tempdir;
-
-
-  /*
-   * In a secure env, the protected sub-directories and files under the HBase rootDir
-   * would be restricted. The sub-directory will have '700' except the bulk load staging dir,
-   * which will have '711'.  The default '700' can be overwritten by setting the property
-   * 'hbase.rootdir.perms'. The protected files (version file, clusterId file) will have '600'.
-   * The rootDir itself will be created with HDFS default permissions if it does not exist.
-   * We will check the rootDir permissions to make sure it has 'x' for all to ensure access
-   * to the staging dir. If it does not, we will add it.
-   */
-  // Permissions for the directories under rootDir that need protection
-  private final FsPermission secureRootSubDirPerms;
-  // Permissions for the files under rootDir that need protection
-  private final FsPermission secureRootFilePerms = new FsPermission("600");
-  // Permissions for bulk load staging directory under rootDir
-  private final FsPermission HiddenDirPerms = FsPermission.valueOf("-rwx--x--x");
-
-  private boolean isSecurityEnabled;
-
-  private final MasterServices services;
-
-  public MasterFileSystem(MasterServices services) throws IOException {
-    this.conf = services.getConfiguration();
-    this.services = services;
-    // Set filesystem to be that of this.rootdir else we get complaints about
-    // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
-    // default localfs.  Presumption is that rootdir is fully-qualified before
-    // we get to here with appropriate fs scheme.
-    this.rootdir = FSUtils.getRootDir(conf);
-    this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
-    // Cover both bases, the old way of setting default fs and the new.
-    // We're supposed to run on 0.20 and 0.21 anyways.
-    this.fs = this.rootdir.getFileSystem(conf);
-    FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
-    // make sure the fs has the same conf
-    fs.setConf(conf);
-    this.secureRootSubDirPerms = new FsPermission(conf.get("hbase.rootdir.perms", "700"));
-    this.isSecurityEnabled = "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
-    // setup the filesystem variable
-    createInitialFileSystemLayout();
-    HFileSystem.addLocationsOrderInterceptor(conf);
-  }
-
-  /**
-   * Create initial layout in filesystem.
-   * <ol>
-   * <li>Check if the meta region exists and is readable, if not create it.
-   * Create hbase.version and the hbase:meta directory if not one.
-   * </li>
-   * </ol>
-   * Idempotent.
-   */
-  private void createInitialFileSystemLayout() throws IOException {
-
-    final String[] protectedSubDirs = new String[] {
-        HConstants.BASE_NAMESPACE_DIR,
-        HConstants.HFILE_ARCHIVE_DIRECTORY,
-        HConstants.HREGION_LOGDIR_NAME,
-        HConstants.HREGION_OLDLOGDIR_NAME,
-        MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR,
-        HConstants.CORRUPT_DIR_NAME,
-        HConstants.HBCK_SIDELINEDIR_NAME,
-        MobConstants.MOB_DIR_NAME
-    };
-    // check if the root directory exists
-    checkRootDir(this.rootdir, conf, this.fs);
-
-    // Check the directories under rootdir.
-    checkTempDir(this.tempdir, conf, this.fs);
-    for (String subDir : protectedSubDirs) {
-      checkSubDir(new Path(this.rootdir, subDir));
-    }
-
-    checkStagingDir();
-
-    // Handle the last few special files and set the final rootDir permissions
-    // rootDir needs 'x' for all to support bulk load staging dir
-    if (isSecurityEnabled) {
-      fs.setPermission(new Path(rootdir, HConstants.VERSION_FILE_NAME), secureRootFilePerms);
-      fs.setPermission(new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME), secureRootFilePerms);
-    }
-    FsPermission currentRootPerms = fs.getFileStatus(this.rootdir).getPermission();
-    if (!currentRootPerms.getUserAction().implies(FsAction.EXECUTE)
-        || !currentRootPerms.getGroupAction().implies(FsAction.EXECUTE)
-        || !currentRootPerms.getOtherAction().implies(FsAction.EXECUTE)) {
-      LOG.warn("rootdir permissions do not contain 'excute' for user, group or other. "
-        + "Automatically adding 'excute' permission for all");
-      fs.setPermission(
-        this.rootdir,
-        new FsPermission(currentRootPerms.getUserAction().or(FsAction.EXECUTE), currentRootPerms
-            .getGroupAction().or(FsAction.EXECUTE), currentRootPerms.getOtherAction().or(
-          FsAction.EXECUTE)));
-    }
-  }
-
-  public FileSystem getFileSystem() {
-    return this.fs;
-  }
-
-  public Configuration getConfiguration() {
-    return this.conf;
-  }
-
-  /**
-   * @return HBase root dir.
-   */
-  public Path getRootDir() {
-    return this.rootdir;
-  }
-
-  /**
-   * @return HBase temp dir.
-   */
-  public Path getTempDir() {
-    return this.tempdir;
-  }
-
-  /**
-   * @return The unique identifier generated for this cluster
-   */
-  public ClusterId getClusterId() {
-    return clusterId;
-  }
-
-  /**
-   * Get the rootdir.  Make sure its wholesome and exists before returning.
-   * @param rd
-   * @param c
-   * @param fs
-   * @return hbase.rootdir (after checks for existence and bootstrapping if
-   * needed populating the directory with necessary bootup files).
-   * @throws IOException
-   */
-  private Path checkRootDir(final Path rd, final Configuration c, final FileSystem fs)
-      throws IOException {
-    // If FS is in safe mode wait till out of it.
-    FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
-
-    // Filesystem is good. Go ahead and check for hbase.rootdir.
-    try {
-      if (!fs.exists(rd)) {
-        fs.mkdirs(rd);
-        // DFS leaves safe mode with 0 DNs when there are 0 blocks.
-        // We used to handle this by checking the current DN count and waiting until
-        // it is nonzero. With security, the check for datanode count doesn't work --
-        // it is a privileged op. So instead we adopt the strategy of the jobtracker
-        // and simply retry file creation during bootstrap indefinitely. As soon as
-        // there is one datanode it will succeed. Permission problems should have
-        // already been caught by mkdirs above.
-        FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
-          10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
-            HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
-      } else {
-        if (!fs.isDirectory(rd)) {
-          throw new IllegalArgumentException(rd.toString() + " is not a directory");
-        }
-        // as above
-        FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
-          10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
-            HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
-      }
-    } catch (DeserializationException de) {
-      LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
-      IOException ioe = new IOException();
-      ioe.initCause(de);
-      throw ioe;
-    } catch (IllegalArgumentException iae) {
-      LOG.fatal("Please fix invalid configuration for "
-        + HConstants.HBASE_DIR + " " + rd.toString(), iae);
-      throw iae;
-    }
-    // Make sure cluster ID exists
-    if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
-        HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
-      FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
-    }
-    clusterId = FSUtils.getClusterId(fs, rd);
-
-    // Make sure the meta region directory exists!
-    if (!FSUtils.metaRegionExists(fs, rd)) {
-      bootstrap(rd, c);
-    }
-
-    // Create tableinfo-s for hbase:meta if not already there.
-    // assume, created table descriptor is for enabling table
-    // meta table is a system table, so descriptors are predefined,
-    // we should get them from registry.
-    FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd);
-    fsd.createTableDescriptor(fsd.get(TableName.META_TABLE_NAME));
-
-    return rd;
-  }
-
-  /**
-   * Make sure the hbase temp directory exists and is empty.
-   * NOTE that this method is only executed once just after the master becomes the active one.
-   */
-  private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
-      throws IOException {
-    // If the temp directory exists, clear the content (left over, from the previous run)
-    if (fs.exists(tmpdir)) {
-      // Archive table in temp, maybe left over from failed deletion,
-      // if not the cleaner will take care of them.
-      for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
-        for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
-          HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
-        }
-      }
-      if (!fs.delete(tmpdir, true)) {
-        throw new IOException("Unable to clean the temp directory: " + tmpdir);
-      }
-    }
-
-    // Create the temp directory
-    if (isSecurityEnabled) {
-      if (!fs.mkdirs(tmpdir, secureRootSubDirPerms)) {
-        throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
-      }
-    } else {
-      if (!fs.mkdirs(tmpdir)) {
-        throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
-      }
-    }
-  }
-
-  /**
-   * Make sure the directories under rootDir have good permissions. Create if necessary.
-   * @param p
-   * @throws IOException
-   */
-  private void checkSubDir(final Path p) throws IOException {
-    if (!fs.exists(p)) {
-      if (isSecurityEnabled) {
-        if (!fs.mkdirs(p, secureRootSubDirPerms)) {
-          throw new IOException("HBase directory '" + p + "' creation failure.");
-        }
-      } else {
-        if (!fs.mkdirs(p)) {
-          throw new IOException("HBase directory '" + p + "' creation failure.");
-        }
-      }
-    }
-    else {
-      if (isSecurityEnabled && !secureRootSubDirPerms.equals(fs.getFileStatus(p).getPermission())) {
-        // check whether the permission match
-        LOG.warn("Found HBase directory permissions NOT matching expected permissions for "
-            + p.toString() + " permissions=" + fs.getFileStatus(p).getPermission()
-            + ", expecting " + secureRootSubDirPerms + ". Automatically setting the permissions. "
-            + "You can change the permissions by setting \"hbase.rootdir.perms\" in hbase-site.xml "
-            + "and restarting the master");
-        fs.setPermission(p, secureRootSubDirPerms);
-      }
-    }
-  }
-
-  /**
-   * Check permissions for bulk load staging directory. This directory has special hidden
-   * permissions. Create it if necessary.
-   * @throws IOException
-   */
-  private void checkStagingDir() throws IOException {
-    Path p = new Path(this.rootdir, HConstants.BULKLOAD_STAGING_DIR_NAME);
-    try {
-      if (!this.fs.exists(p)) {
-        if (!this.fs.mkdirs(p, HiddenDirPerms)) {
-          throw new IOException("Failed to create staging directory " + p.toString());
-        }
-      } else {
-        this.fs.setPermission(p, HiddenDirPerms);
-      }
-    } catch (IOException e) {
-      LOG.error("Failed to create or set permission on staging directory " + p.toString());
-      throw new IOException("Failed to create or set permission on staging directory "
-          + p.toString(), e);
-    }
-  }
-
-  private static void bootstrap(final Path rd, final Configuration c)
-  throws IOException {
-    LOG.info("BOOTSTRAP: creating hbase:meta region");
-    try {
-      // Bootstrapping, make sure blockcache is off.  Else, one will be
-      // created here in bootstrap and it'll need to be cleaned up.  Better to
-      // not make it in first place.  Turn off block caching for bootstrap.
-      // Enable after.
-      HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
-      HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
-      setInfoFamilyCachingForMeta(metaDescriptor, false);
-      HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor, null);
-      setInfoFamilyCachingForMeta(metaDescriptor, true);
-      meta.close();
-    } catch (IOException e) {
-        e = e instanceof RemoteException ?
-                ((RemoteException)e).unwrapRemoteException() : e;
-      LOG.error("bootstrap", e);
-      throw e;
-    }
-  }
-
-  /**
-   * Enable in memory caching for hbase:meta
-   */
-  public static void setInfoFamilyCachingForMeta(HTableDescriptor metaDescriptor, final boolean b) {
-    for (HColumnDescriptor hcd: metaDescriptor.getColumnFamilies()) {
-      if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
-        hcd.setBlockCacheEnabled(b);
-        hcd.setInMemory(b);
-      }
-    }
-  }
-
-  public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName, boolean hasMob)
-      throws IOException {
-    // archive family store files
-    Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
-    HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
-
-    // delete the family folder
-    Path familyDir = new Path(tableDir,
-      new Path(region.getEncodedName(), Bytes.toString(familyName)));
-    if (fs.delete(familyDir, true) == false) {
-      if (fs.exists(familyDir)) {
-        throw new IOException("Could not delete family "
-            + Bytes.toString(familyName) + " from FileSystem for region "
-            + region.getRegionNameAsString() + "(" + region.getEncodedName()
-            + ")");
-      }
-    }
-
-    // archive and delete mob files
-    if (hasMob) {
-      Path mobTableDir =
-          FSUtils.getTableDir(new Path(getRootDir(), MobConstants.MOB_DIR_NAME), region.getTable());
-      HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(region.getTable());
-      Path mobFamilyDir =
-          new Path(mobTableDir,
-              new Path(mobRegionInfo.getEncodedName(), Bytes.toString(familyName)));
-      // archive mob family store files
-      MobUtils.archiveMobStoreFiles(conf, fs, mobRegionInfo, mobFamilyDir, familyName);
-
-      if (!fs.delete(mobFamilyDir, true)) {
-        throw new IOException("Could not delete mob store files for family "
-            + Bytes.toString(familyName) + " from FileSystem region "
-            + mobRegionInfo.getRegionNameAsString() + "(" + mobRegionInfo.getEncodedName() + ")");
-      }
-    }
-  }
-
-  public void stop() {
-  }
-
-  public void logFileSystemState(Log log) throws IOException {
-    FSUtils.logFileSystemState(fs, rootdir, log);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index 670642f..3dfe4a6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
index e287a6d..e90b744 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
@@ -342,4 +342,4 @@ public class MasterWalManager {
   public RecoveryMode getLogRecoveryMode() {
     return this.splitLogManager.getRecoveryMode();
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
index 9c21040..f52e321 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
@@ -41,8 +41,9 @@ import org.apache.hadoop.hbase.TableExistsException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.fs.FsContext;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.MetricsSnapshot;
 import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
@@ -419,8 +420,7 @@ public class CloneSnapshotProcedure
     // using a copy of descriptor, table will be created enabling first
     HTableDescriptor underConstruction = new HTableDescriptor(hTableDescriptor);
     final Path tempTableDir = FSUtils.getTableDir(tempdir, hTableDescriptor.getTableName());
-    ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors()))
-      .createTableDescriptorForTableDirectory(tempTableDir, underConstruction, false);
+    mfs.createTableDescriptor(FsContext.TEMP, underConstruction, false);
 
     // 2. Create Regions
     newRegions = hdfsRegionHandler.createHdfsRegions(

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
index c350185..7562a04 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
@@ -27,7 +27,6 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.NamespaceExistException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.TableNamespaceManager;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
@@ -203,9 +202,7 @@ public class CreateNamespaceProcedure
   protected static void createDirectory(
       final MasterProcedureEnv env,
       final NamespaceDescriptor nsDescriptor) throws IOException {
-    MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
-    mfs.getFileSystem().mkdirs(
-      FSUtils.getNamespaceDir(mfs.getRootDir(), nsDescriptor.getName()));
+    env.getMasterServices().getMasterFileSystem().createNamespace(nsDescriptor);
   }
 
   /**


[04/18] hbase git commit: HBASE-14439 break everything HRegion

Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
index 11bd280..38e0a61 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.io.HeapSize;
@@ -96,7 +97,7 @@ public class TestAtomicOperation {
   static final byte [] row = Bytes.toBytes("rowA");
   static final byte [] row2 = Bytes.toBytes("rowB");
 
-  @Before 
+  @Before
   public void setup() {
     tableName = Bytes.toBytes(name.getMethodName());
   }
@@ -114,7 +115,7 @@ public class TestAtomicOperation {
   }
   //////////////////////////////////////////////////////////////////////////////
   // New tests that doesn't spin up a mini cluster but rather just test the
-  // individual code pieces in the HRegion. 
+  // individual code pieces in the HRegion.
   //////////////////////////////////////////////////////////////////////////////
 
   /**
@@ -334,8 +335,8 @@ public class TestAtomicOperation {
 
               Get g = new Get(row);
               Result result = region.get(g);
-              assertEquals(result.getValue(fam1, qual1).length, result.getValue(fam1, qual2).length); 
-              assertEquals(result.getValue(fam1, qual1).length, result.getValue(fam2, qual3).length); 
+              assertEquals(result.getValue(fam1, qual1).length, result.getValue(fam1, qual2).length);
+              assertEquals(result.getValue(fam1, qual1).length, result.getValue(fam2, qual3).length);
             } catch (IOException e) {
               e.printStackTrace();
               failures.incrementAndGet();
@@ -567,7 +568,7 @@ public class TestAtomicOperation {
       this.failures = failures;
     }
   }
-  
+
   private static CountDownLatch latch = new CountDownLatch(1);
   private enum TestStep {
     INIT,                  // initial put of 10 to set value of the cell
@@ -579,11 +580,11 @@ public class TestAtomicOperation {
   }
   private static volatile TestStep testStep = TestStep.INIT;
   private final String family = "f1";
-     
+
   /**
    * Test written as a verifier for HBASE-7051, CheckAndPut should properly read
-   * MVCC. 
-   * 
+   * MVCC.
+   *
    * Moved into TestAtomicOperation from its original location, TestHBase7051
    */
   @Test
@@ -598,7 +599,7 @@ public class TestAtomicOperation {
     Put put = new Put(Bytes.toBytes("r1"));
     put.addColumn(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("10"));
     puts[0] = put;
-    
+
     region.batchMutate(puts, HConstants.NO_NONCE, HConstants.NO_NONCE);
     MultithreadedTestUtil.TestContext ctx =
       new MultithreadedTestUtil.TestContext(conf);
@@ -660,9 +661,9 @@ public class TestAtomicOperation {
 
   public static class MockHRegion extends HRegion {
 
-    public MockHRegion(Path tableDir, WAL log, FileSystem fs, Configuration conf,
-        final HRegionInfo regionInfo, final HTableDescriptor htd, RegionServerServices rsServices) {
-      super(tableDir, log, fs, conf, regionInfo, htd, rsServices);
+    public MockHRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+        final WAL wal, final RegionServerServices rsServices) {
+      super(rfs, htd, wal, rsServices);
     }
 
     @Override
@@ -672,7 +673,7 @@ public class TestAtomicOperation {
       }
       return new WrappedRowLock(super.getRowLockInternal(row, readLock));
     }
-    
+
     public class WrappedRowLock implements RowLock {
 
       private final RowLock rowLock;

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
index caf09ad..212a635 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
@@ -244,12 +244,8 @@ public class TestBulkLoad {
     }
 
     // TODO We need a way to do this without creating files
-    return HRegion.createHRegion(hRegionInfo,
-        new Path(testFolder.newFolder().toURI()),
-        conf,
-        hTableDescriptor,
-        log);
-
+    return HRegion.createHRegion(conf, new Path(testFolder.newFolder().toURI()),
+        hTableDescriptor, hRegionInfo, log);
   }
 
   private HRegion testRegionWithFamilies(byte[]... families) throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
index 1513cd0..ef332a4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
@@ -22,6 +22,12 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
@@ -31,8 +37,161 @@ import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-@Category(SmallTests.class)
-public class TestDefaultCompactSelection extends TestCompactionPolicy {
+import com.google.common.collect.Lists;
+
+@Category({RegionServerTests.class, SmallTests.class})
+public class TestDefaultCompactSelection extends TestCase {
+  private final static Log LOG = LogFactory.getLog(TestDefaultCompactSelection.class);
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  protected Configuration conf;
+  protected HStore store;
+  private static final String DIR=
+    TEST_UTIL.getDataTestDir(TestDefaultCompactSelection.class.getSimpleName()).toString();
+  private static Path TEST_FILE;
+
+  protected static final int minFiles = 3;
+  protected static final int maxFiles = 5;
+
+  protected static final long minSize = 10;
+  protected static final long maxSize = 2100;
+
+  private WALFactory wals;
+  private HRegion region;
+
+  @Override
+  public void setUp() throws Exception {
+    // setup config values necessary for store
+    this.conf = TEST_UTIL.getConfiguration();
+    this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
+    this.conf.setInt("hbase.hstore.compaction.min", minFiles);
+    this.conf.setInt("hbase.hstore.compaction.max", maxFiles);
+    this.conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, minSize);
+    this.conf.setLong("hbase.hstore.compaction.max.size", maxSize);
+    this.conf.setFloat("hbase.hstore.compaction.ratio", 1.0F);
+    // Test depends on this not being set to pass.  Default breaks test.  TODO: Revisit.
+    this.conf.unset("hbase.hstore.compaction.min.size");
+
+    //Setting up a Store
+    final String id = TestDefaultCompactSelection.class.getName();
+    Path basedir = new Path(DIR);
+    final Path logdir = new Path(basedir, DefaultWALProvider.getWALDirectoryName(id));
+    HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family"));
+    FileSystem fs = FileSystem.get(conf);
+
+    fs.delete(logdir, true);
+
+    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("table")));
+    htd.addFamily(hcd);
+    HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
+
+    final Configuration walConf = new Configuration(conf);
+    FSUtils.setRootDir(walConf, basedir);
+    wals = new WALFactory(walConf, null, id);
+    region = HBaseTestingUtility.createRegionAndWAL(info, basedir, conf, htd);
+    HBaseTestingUtility.closeRegionAndWAL(region);
+
+    RegionFileSystem rfs = RegionFileSystem.open(conf, fs, basedir, info, false);
+    region = new HRegion(rfs, htd,
+      wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), null);
+
+    store = new HStore(region, hcd, conf);
+
+    TEST_FILE = region.getRegionFileSystem().createTempName();
+    fs.createNewFile(TEST_FILE);
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    IOException ex = null;
+    try {
+      region.close();
+    } catch (IOException e) {
+      LOG.warn("Caught Exception", e);
+      ex = e;
+    }
+    try {
+      wals.close();
+    } catch (IOException e) {
+      LOG.warn("Caught Exception", e);
+      ex = e;
+    }
+    if (ex != null) {
+      throw ex;
+    }
+  }
+
+  ArrayList<Long> toArrayList(long... numbers) {
+    ArrayList<Long> result = new ArrayList<Long>();
+    for (long i : numbers) {
+      result.add(i);
+    }
+    return result;
+  }
+
+  List<StoreFile> sfCreate(long... sizes) throws IOException {
+    ArrayList<Long> ageInDisk = new ArrayList<Long>();
+    for (int i = 0; i < sizes.length; i++) {
+      ageInDisk.add(0L);
+    }
+    return sfCreate(toArrayList(sizes), ageInDisk);
+  }
+
+  List<StoreFile> sfCreate(ArrayList<Long> sizes, ArrayList<Long> ageInDisk)
+    throws IOException {
+    return sfCreate(false, sizes, ageInDisk);
+  }
+
+  List<StoreFile> sfCreate(boolean isReference, long... sizes) throws IOException {
+    ArrayList<Long> ageInDisk = new ArrayList<Long>(sizes.length);
+    for (int i = 0; i < sizes.length; i++) {
+      ageInDisk.add(0L);
+    }
+    return sfCreate(isReference, toArrayList(sizes), ageInDisk);
+  }
+
+  List<StoreFile> sfCreate(boolean isReference, ArrayList<Long> sizes, ArrayList<Long> ageInDisk)
+      throws IOException {
+    List<StoreFile> ret = Lists.newArrayList();
+    for (int i = 0; i < sizes.size(); i++) {
+      ret.add(new MockStoreFile(TEST_UTIL, TEST_FILE,
+          sizes.get(i), ageInDisk.get(i), isReference, i));
+    }
+    return ret;
+  }
+
+  long[] getSizes(List<StoreFile> sfList) {
+    long[] aNums = new long[sfList.size()];
+    for (int i = 0; i < sfList.size(); ++i) {
+      aNums[i] = sfList.get(i).getReader().length();
+    }
+    return aNums;
+  }
+
+  void compactEquals(List<StoreFile> candidates, long... expected)
+    throws IOException {
+    compactEquals(candidates, false, false, expected);
+  }
+
+  void compactEquals(List<StoreFile> candidates, boolean forcemajor, long... expected)
+    throws IOException {
+    compactEquals(candidates, forcemajor, false, expected);
+  }
+
+  void compactEquals(List<StoreFile> candidates, boolean forcemajor, boolean isOffPeak,
+      long ... expected)
+  throws IOException {
+    store.forceMajor = forcemajor;
+    //Test Default compactions
+    CompactionRequest result = ((RatioBasedCompactionPolicy)store.storeEngine.getCompactionPolicy())
+        .selectCompaction(candidates, new ArrayList<StoreFile>(), false, isOffPeak, forcemajor);
+    List<StoreFile> actual = new ArrayList<StoreFile>(result.getFiles());
+    if (isOffPeak && !forcemajor) {
+      assertTrue(result.isOffPeak());
+    }
+    assertEquals(Arrays.toString(expected), Arrays.toString(getSizes(actual)));
+    store.forceMajor = false;
+  }
 
   @Test
   public void testCompactionRatio() throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
index e4eff84..3faf8bb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
@@ -997,7 +997,7 @@ public class TestDefaultMemStore {
     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf("testShouldFlushMeta"));
     desc.addFamily(new HColumnDescriptor("foo".getBytes()));
     HRegion r =
-        HRegion.createHRegion(hri, testDir, conf, desc,
+        HRegion.createHRegion(conf, testDir, desc, hri,
             wFactory.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace()));
     HRegion.addRegionToMETA(meta, r);
     edge.setCurrentTimeMillis(1234 + 100);

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
index dfc97e9..a324328 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.ArrayBackedTag;
 import org.apache.hadoop.hbase.TagType;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
 import org.apache.hadoop.hbase.io.crypto.aes.AES;
 import org.apache.hadoop.hbase.io.hfile.HFile;
@@ -157,8 +158,10 @@ public class TestHMobStore {
     final Configuration walConf = new Configuration(conf);
     FSUtils.setRootDir(walConf, basedir);
     final WALFactory wals = new WALFactory(walConf, null, methodName);
-    region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes(),
-            info.getTable().getNamespace()), fs, conf, info, htd, null);
+
+    RegionFileSystem rfs = RegionFileSystem.open(conf, fs, basedir, info, false);
+    region = new HRegion(rfs, htd,
+      wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), null);
     store = new HMobStore(region, hcd, conf);
     if(testStore) {
       init(conf, hcd);

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 79d305f..df0b119 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
@@ -101,6 +102,7 @@ import org.apache.hadoop.hbase.filter.NullComparator;
 import org.apache.hadoop.hbase.filter.PrefixFilter;
 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.fs.legacy.LegacyLayout;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
@@ -4471,7 +4473,7 @@ public class TestHRegion {
       info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
           HConstants.EMPTY_BYTE_ARRAY, false);
       Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
-      region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
+      region = HRegion.newHRegion(CONF, fs, path, htd, info, null, null);
       // region initialization throws IOException and set task state to ABORTED.
       region.initialize();
       fail("Region initialization should fail due to IOException");
@@ -4509,10 +4511,10 @@ public class TestHRegion {
     FileSystem fs = region.getRegionFileSystem().getFileSystem();
     HBaseTestingUtility.closeRegionAndWAL(region);
 
-    Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
+    Path regionInfoFile = LegacyLayout.getRegionInfoFile(regionDir);
 
     // Verify that the .regioninfo file is present
-    assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
+    assertTrue(LegacyLayout.REGION_INFO_FILE + " should be present in the region dir",
         fs.exists(regionInfoFile));
 
     // Try to open the region
@@ -4521,12 +4523,12 @@ public class TestHRegion {
     HBaseTestingUtility.closeRegionAndWAL(region);
 
     // Verify that the .regioninfo file is still there
-    assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
+    assertTrue(LegacyLayout.REGION_INFO_FILE + " should be present in the region dir",
         fs.exists(regionInfoFile));
 
     // Remove the .regioninfo file and verify is recreated on region open
     fs.delete(regionInfoFile, true);
-    assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
+    assertFalse(LegacyLayout.REGION_INFO_FILE + " should be removed from the region dir",
         fs.exists(regionInfoFile));
 
     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
@@ -4535,8 +4537,8 @@ public class TestHRegion {
     HBaseTestingUtility.closeRegionAndWAL(region);
 
     // Verify that the .regioninfo file is still there
-    assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
-        fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
+    assertTrue(LegacyLayout.REGION_INFO_FILE + " should be present in the region dir",
+        fs.exists(new Path(regionDir, LegacyLayout.REGION_INFO_FILE)));
   }
 
   /**
@@ -6097,10 +6099,9 @@ public class TestHRegion {
 
   // Helper for test testOpenRegionWrittenToWALForLogReplay
   static class HRegionWithSeqId extends HRegion {
-    public HRegionWithSeqId(final Path tableDir, final WAL wal, final FileSystem fs,
-        final Configuration confParam, final HRegionInfo regionInfo,
-        final HTableDescriptor htd, final RegionServerServices rsServices) {
-      super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices);
+    public HRegionWithSeqId(final RegionFileSystem rfs, final HTableDescriptor htd,
+        final WAL wal, final RegionServerServices rsServices) {
+      super(rfs, htd, wal, rsServices);
     }
     @Override
     protected long getNextSequenceId(WAL wal) throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
index 5f792fa..48609b9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.FSUtilsWithRetries;
 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -60,25 +62,22 @@ public class TestHRegionFileSystem {
 
     // Create a Region
     HRegionInfo hri = new HRegionInfo(TableName.valueOf("TestTable"));
-    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
-        FSUtils.getTableDir(rootDir, hri.getTable()), hri);
+    RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, rootDir, hri, true);
 
     // Verify if the region is on disk
     Path regionDir = regionFs.getRegionDir();
     assertTrue("The region folder should be created", fs.exists(regionDir));
 
     // Verify the .regioninfo
-    HRegionInfo hriVerify = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
+    HRegionInfo hriVerify = RegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
     assertEquals(hri, hriVerify);
 
     // Open the region
-    regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
-        FSUtils.getTableDir(rootDir, hri.getTable()), hri, false);
+    regionFs = RegionFileSystem.open(conf, fs, rootDir, hri, false);
     assertEquals(regionDir, regionFs.getRegionDir());
 
     // Delete the region
-    HRegionFileSystem.deleteRegionFromFileSystem(conf, fs,
-        FSUtils.getTableDir(rootDir, hri.getTable()), hri);
+    RegionFileSystem.destroy(conf, fs, rootDir, hri);
     assertFalse("The region folder should be removed", fs.exists(regionDir));
 
     fs.delete(rootDir, true);
@@ -90,24 +89,15 @@ public class TestHRegionFileSystem {
     FileSystem fs = TEST_UTIL.getTestFileSystem();
     Configuration conf = TEST_UTIL.getConfiguration();
 
-    // Create a Region
-    HRegionInfo hri = new HRegionInfo(TableName.valueOf("TestTable"));
-    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, rootDir, hri);
-    assertTrue(fs.exists(regionFs.getRegionDir()));
-
-    regionFs = new HRegionFileSystem(conf, new MockFileSystemForCreate(),
-        null, null);
-    // HRegionFileSystem.createRegionOnFileSystem(conf, new MockFileSystemForCreate(), rootDir,
-    // hri);
+    FSUtilsWithRetries regionFs = new FSUtilsWithRetries(conf, new MockFileSystemForCreate());
     boolean result = regionFs.createDir(new Path("/foo/bar"));
     assertTrue("Couldn't create the directory", result);
 
-
-    regionFs = new HRegionFileSystem(conf, new MockFileSystem(), null, null);
+    regionFs = new FSUtilsWithRetries(conf, new MockFileSystem());
     result = regionFs.rename(new Path("/foo/bar"), new Path("/foo/bar2"));
     assertTrue("Couldn't rename the directory", result);
 
-    regionFs = new HRegionFileSystem(conf, new MockFileSystem(), null, null);
+    regionFs = new FSUtilsWithRetries(conf, new MockFileSystem());
     result = regionFs.deleteDir(new Path("/foo/bar"));
     assertTrue("Couldn't delete the directory", result);
     fs.delete(rootDir, true);
@@ -217,7 +207,7 @@ public class TestHRegionFileSystem {
     // Create a Region
     String familyName = "cf";
     HRegionInfo hri = new HRegionInfo(TableName.valueOf("TestTable"));
-    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, rootDir, hri);
+    RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, rootDir, hri, true);
 
     // New region, no store files
     Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName);

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
index 5fde726..7e0a6f6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
@@ -36,6 +36,8 @@ import org.apache.hadoop.hbase.testclassification.RegionServerTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.legacy.LegacyLayout;
 import org.apache.hadoop.hbase.master.RegionState;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
@@ -77,7 +79,7 @@ public class TestHRegionInfo {
     long modtime2 = getModTime(r);
     assertEquals(modtime, modtime2);
     // Now load the file.
-    HRegionInfo deserializedHri = HRegionFileSystem.loadRegionInfoFileContent(
+    HRegionInfo deserializedHri = RegionFileSystem.loadRegionInfoFileContent(
         r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir());
     assertTrue(hri.equals(deserializedHri));
     HBaseTestingUtility.closeRegionAndWAL(r);
@@ -85,7 +87,7 @@ public class TestHRegionInfo {
 
   long getModTime(final HRegion r) throws IOException {
     FileStatus[] statuses = r.getRegionFileSystem().getFileSystem().listStatus(
-      new Path(r.getRegionFileSystem().getRegionDir(), HRegionFileSystem.REGION_INFO_FILE));
+      LegacyLayout.getRegionInfoFile(r.getRegionFileSystem().getRegionDir()));
     assertTrue(statuses != null && statuses.length == 1);
     return statuses[0].getModificationTime();
   }
@@ -296,12 +298,12 @@ public class TestHRegionInfo {
     String firstPart = descriptiveNameForDisplay.substring(0,
         descriptiveNameForDisplay.indexOf(new String(HRegionInfo.HIDDEN_START_KEY)));
     String secondPart = descriptiveNameForDisplay.substring(
-        descriptiveNameForDisplay.indexOf(new String(HRegionInfo.HIDDEN_START_KEY)) + 
+        descriptiveNameForDisplay.indexOf(new String(HRegionInfo.HIDDEN_START_KEY)) +
         HRegionInfo.HIDDEN_START_KEY.length);
     String firstPartOrig = origDesc.substring(0,
         origDesc.indexOf(Bytes.toStringBinary(startKey)));
     String secondPartOrig = origDesc.substring(
-        origDesc.indexOf(Bytes.toStringBinary(startKey)) + 
+        origDesc.indexOf(Bytes.toStringBinary(startKey)) +
         Bytes.toStringBinary(startKey).length());
     assert(firstPart.equals(firstPartOrig));
     assert(secondPart.equals(secondPartOrig));

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
index a3804dd..542fa7a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
@@ -177,7 +177,8 @@ public class TestHRegionReplayEvents {
     es.startExecutorService(
       string+"-"+string, 1);
     when(rss.getExecutorService()).thenReturn(es);
-    primaryRegion = HRegion.createHRegion(primaryHri, rootDir, CONF, htd, walPrimary);
+
+    primaryRegion = HRegion.createHRegion(CONF, rootDir, htd, primaryHri, walPrimary);
     primaryRegion.close();
     List<Region> regions = new ArrayList<Region>();
     regions.add(primaryRegion);
@@ -1671,4 +1672,4 @@ public class TestHRegionReplayEvents {
     return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf,
       isReadOnly, durability, wal, families);
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java
index fc47d7e..014aba7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -100,13 +101,12 @@ public class TestRecoveredEdits {
     Path hbaseRootDir = TEST_UTIL.getDataTestDir();
     FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
     Path tableDir = FSUtils.getTableDir(hbaseRootDir, htd.getTableName());
-    HRegionFileSystem hrfs =
-        new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, tableDir, hri);
+    RegionFileSystem hrfs = RegionFileSystem.open(TEST_UTIL.getConfiguration(), fs, hbaseRootDir, hri, false);
     if (fs.exists(hrfs.getRegionDir())) {
       LOG.info("Region directory already exists. Deleting.");
       fs.delete(hrfs.getRegionDir(), true);
     }
-    HRegion region = HRegion.createHRegion(hri, hbaseRootDir, conf, htd, null);
+    HRegion region = HRegion.createHRegion(conf, hbaseRootDir, htd, hri, null);
     assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
     List<String> storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
     // There should be no store files.

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
index f88c7dd..326bbef 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.exceptions.MergeRegionException;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.HMaster;
@@ -247,18 +248,11 @@ public class TestRegionMergeTransactionOnCluster {
       assertTrue(fs.exists(regionAdir));
       assertTrue(fs.exists(regionBdir));
 
-      HColumnDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies();
-      HRegionFileSystem hrfs = new HRegionFileSystem(
-        TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
-      int count = 0;
-      for(HColumnDescriptor colFamily : columnFamilies) {
-        count += hrfs.getStoreFiles(colFamily.getName()).size();
-      }
-      ADMIN.compactRegion(mergedRegionInfo.getRegionName());
-      // clean up the merged region store files
-      // wait until merged region have reference file
+      admin.compactRegion(mergedRegionInfo.getRegionName());
+      // wait until merged region doesn't have reference file
       long timeout = System.currentTimeMillis() + waitTime;
-      int newcount = 0;
+      RegionFileSystem hrfs = RegionFileSystem.open(
+          TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo, false);
       while (System.currentTimeMillis() < timeout) {
         for(HColumnDescriptor colFamily : columnFamilies) {
           newcount += hrfs.getStoreFiles(colFamily.getName()).size();
@@ -561,4 +555,4 @@ public class TestRegionMergeTransactionOnCluster {
       return resp;
     }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
index b031413..52e8724 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.client.ScannerCallable;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
@@ -444,14 +445,9 @@ public class TestScannerHeartbeatMessages {
     private static volatile int columnFamilySleepTime = DEFAULT_CF_SLEEP_TIME;
     private static volatile boolean sleepBetweenColumnFamilies = false;
 
-    public HeartbeatHRegion(Path tableDir, WAL wal, FileSystem fs, Configuration confParam,
-        HRegionInfo regionInfo, HTableDescriptor htd, RegionServerServices rsServices) {
-      super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices);
-    }
-
-    public HeartbeatHRegion(HRegionFileSystem fs, WAL wal, Configuration confParam,
-        HTableDescriptor htd, RegionServerServices rsServices) {
-      super(fs, wal, confParam, htd, rsServices);
+    public HeartbeatHRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+        final WAL wal, final RegionServerServices rsServices) {
+      super(rfs, htd, wal, rsServices);
     }
 
     private static void columnFamilySleep() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index c318fe4..420d77f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -769,7 +769,7 @@ public class TestSplitTransactionOnCluster {
     try {
       // Precondition: we created a table with no data, no store files.
       printOutRegions(regionServer, "Initial regions: ");
-      cluster.getMaster().getMasterFileSystem().logFSTree(LOG);
+      cluster.getMaster().getMasterFileSystem().logFileSystemState(LOG);
       List<StoreFileInfo> storefiles = getStoreFiles(tableName);
       assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
 
@@ -793,7 +793,7 @@ public class TestSplitTransactionOnCluster {
       assertTrue(daughters.size() == 2);
 
       // check dirs
-      cluster.getMaster().getMasterFileSystem().logFSTree(LOG);
+      cluster.getMaster().getMasterFileSystem().logFileSystemState(LOG);
       List<StoreFileInfo> storefilesAfter = getStoreFiles(tableName);
       assertEquals("Expected nothing but found " + storefilesAfter.toString(),
           storefilesAfter.size(), 0);
@@ -949,12 +949,8 @@ public class TestSplitTransactionOnCluster {
       SplitTransactionImpl st = new SplitTransactionImpl(regions.get(0), Bytes.toBytes("r3"));
       st.prepare();
       st.stepsBeforePONR(regionServer, regionServer, false);
-      Path tableDir =
-          FSUtils.getTableDir(cluster.getMaster().getMasterFileSystem().getRootDir(),
-            desc.getTableName());
-      List<Path> regionDirs =
-          FSUtils.getRegionDirs(tableDir.getFileSystem(cluster.getConfiguration()), tableDir);
-      assertEquals(3,regionDirs.size());
+      assertEquals(3, cluster.getMaster().getMasterFileSystem().getRegions(desc.getTableName()).size());
+      cluster.startRegionServer();
       regionServer.kill();
       // Before we check deadServerInProgress, we should ensure server is dead at master side.
       while (!cluster.getMaster().getServerManager().
@@ -969,9 +965,7 @@ public class TestSplitTransactionOnCluster {
       AssignmentManager am = cluster.getMaster().getAssignmentManager();
       assertEquals(am.getRegionStates().getRegionsInTransition().toString(), 0, am
           .getRegionStates().getRegionsInTransition().size());
-      regionDirs =
-          FSUtils.getRegionDirs(tableDir.getFileSystem(cluster.getConfiguration()), tableDir);
-      assertEquals(1,regionDirs.size());
+      assertEquals(1, cluster.getMaster().getMasterFileSystem().getRegions(desc.getTableName()).size());
     } finally {
       TESTING_UTIL.deleteTable(table);
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
index e3e62fc..7f44324 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.io.compress.Compression;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -170,11 +171,9 @@ public class TestStore {
       HColumnDescriptor hcd) throws IOException {
     //Setting up a Store
     Path basedir = new Path(DIR+methodName);
-    Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
     final Path logdir = new Path(basedir, AbstractFSWALProvider.getWALDirectoryName(methodName));
 
     FileSystem fs = FileSystem.get(conf);
-
     fs.delete(logdir, true);
 
     if (htd.hasFamily(hcd.getName())) {
@@ -182,12 +181,14 @@ public class TestStore {
     } else {
       htd.addFamily(hcd);
     }
+
     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
     final Configuration walConf = new Configuration(conf);
     FSUtils.setRootDir(walConf, basedir);
     final WALFactory wals = new WALFactory(walConf, null, methodName);
-    HRegion region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes(),
-            info.getTable().getNamespace()), fs, conf, info, htd, null);
+    RegionFileSystem rfs = RegionFileSystem.open(conf, fs, basedir, info, false);
+    HRegion region = new HRegion(rfs, htd,
+      wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), null);
 
     store = new HStore(region, hcd, conf);
     return store;

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
index 09532cf..e88a948 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
@@ -105,8 +106,7 @@ public class TestStoreFile extends HBaseTestCase {
   public void testBasicHalfMapFile() throws Exception {
     final HRegionInfo hri =
         new HRegionInfo(TableName.valueOf("testBasicHalfMapFileTb"));
-    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(
-      conf, fs, new Path(testDir, hri.getTable().getNameAsString()), hri);
+    RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, testDir, hri, true);
 
     HFileContext meta = new HFileContextBuilder().withBlockSize(2*1024).build();
     StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
@@ -116,8 +116,7 @@ public class TestStoreFile extends HBaseTestCase {
     writeStoreFile(writer);
 
     Path sfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
-    StoreFile sf = new StoreFile(this.fs, sfPath, conf, cacheConf,
-      BloomType.NONE);
+    StoreFile sf = new StoreFile(this.fs, sfPath, conf, cacheConf, BloomType.NONE);
     checkHalfHFile(regionFs, sf);
   }
 
@@ -157,8 +156,7 @@ public class TestStoreFile extends HBaseTestCase {
   @Test
   public void testReference() throws IOException {
     final HRegionInfo hri = new HRegionInfo(TableName.valueOf("testReferenceTb"));
-    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(
-      conf, fs, new Path(testDir, hri.getTable().getNameAsString()), hri);
+    RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, testDir, hri, true);
 
     HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
     // Make a store file and write data to it.
@@ -223,8 +221,7 @@ public class TestStoreFile extends HBaseTestCase {
     // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/
     Configuration testConf = new Configuration(this.conf);
     FSUtils.setRootDir(testConf, testDir);
-    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(
-      testConf, fs, FSUtils.getTableDir(testDir, hri.getTable()), hri);
+    RegionFileSystem regionFs = RegionFileSystem.open(testConf, fs, testDir, hri, true);
     HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
 
     // Make a store file and write data to it.
@@ -268,8 +265,7 @@ public class TestStoreFile extends HBaseTestCase {
 
     // adding legal table name chars to verify regex handles it.
     HRegionInfo hri = new HRegionInfo(TableName.valueOf("_original-evil-name"));
-    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(
-      testConf, fs, FSUtils.getTableDir(testDir, hri.getTable()), hri);
+    RegionFileSystem regionFs = RegionFileSystem.open(testConf, fs, testDir, hri, true);
 
     HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
     // Make a store file and write data to it. <root>/<tablename>/<rgn>/<cf>/<file>
@@ -282,9 +278,7 @@ public class TestStoreFile extends HBaseTestCase {
 
     // create link to store file. <root>/clone/region/<cf>/<hfile>-<region>-<table>
     HRegionInfo hriClone = new HRegionInfo(TableName.valueOf("clone"));
-    HRegionFileSystem cloneRegionFs = HRegionFileSystem.createRegionOnFileSystem(
-      testConf, fs, FSUtils.getTableDir(testDir, hri.getTable()),
-        hriClone);
+    RegionFileSystem cloneRegionFs = RegionFileSystem.open(testConf, fs, testDir, hriClone, true);
     Path dstPath = cloneRegionFs.getStoreDir(TEST_FAMILY);
     HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
     Path linkFilePath = new Path(dstPath,
@@ -337,7 +331,7 @@ public class TestStoreFile extends HBaseTestCase {
     assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
   }
 
-  private void checkHalfHFile(final HRegionFileSystem regionFs, final StoreFile f)
+  private void checkHalfHFile(final RegionFileSystem regionFs, final StoreFile f)
       throws IOException {
     Cell midkey = f.createReader().midkey();
     KeyValue midKV = (KeyValue)midkey;
@@ -1001,7 +995,7 @@ public class TestStoreFile extends HBaseTestCase {
     assertEquals(startEvicted, cs.getEvictedCount());
   }
 
-  private Path splitStoreFile(final HRegionFileSystem regionFs, final HRegionInfo hri,
+  private Path splitStoreFile(final RegionFileSystem regionFs, final HRegionInfo hri,
       final String family, final StoreFile sf, final byte[] splitKey, boolean isTopRef)
       throws IOException {
     FileSystem fs = regionFs.getFileSystem();

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java
index 74b3df9..7435567 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java
@@ -46,6 +46,8 @@ import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.legacy.LegacyRegionFileSystem;
 import org.apache.hadoop.hbase.wal.WALFactory;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -79,7 +81,7 @@ public class TestStoreFileRefresherChore {
     return htd;
   }
 
-  static class FailingHRegionFileSystem extends HRegionFileSystem {
+  static class FailingHRegionFileSystem extends LegacyRegionFileSystem {
     boolean fail = false;
     FailingHRegionFileSystem(Configuration conf, FileSystem fs, Path tableDir, HRegionInfo regionInfo) {
       super(conf, fs, tableDir, regionInfo);
@@ -101,14 +103,14 @@ public class TestStoreFileRefresherChore {
 
     HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false, 0, replicaId);
 
-    HRegionFileSystem fs = new FailingHRegionFileSystem(conf, tableDir.getFileSystem(conf), tableDir,
+    RegionFileSystem fs = new FailingHRegionFileSystem(conf, tableDir.getFileSystem(conf), tableDir,
       info);
     final Configuration walConf = new Configuration(conf);
     FSUtils.setRootDir(walConf, tableDir);
     final WALFactory wals = new WALFactory(walConf, null, "log_" + replicaId);
-    HRegion region =
-        new HRegion(fs, wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()),
-            conf, htd, null);
+
+    HRegion region = new HRegion(fs, htd,
+      wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), null);
 
     region.initialize();
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java
index d6d4404..3065771 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java
@@ -218,7 +218,7 @@ public class TestDurability {
     assertEquals(3, Bytes.toLong(res.getValue(FAMILY, col3)));
     verifyWALCount(wals, wal, 2);
   }
-  
+
   /*
    * Test when returnResults set to false in increment it should not return the result instead it
    * resturn null.
@@ -275,7 +275,7 @@ public class TestDurability {
           throw new IOException("Failed delete of " + path);
         }
       }
-      return HRegion.createHRegion(info, path, CONF, htd, log);
+      return HRegion.createHRegion(CONF, path, htd, info, log);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index 7ca5d36..d943ccd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
 import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.master.HMaster;
@@ -66,7 +67,6 @@ import org.apache.hadoop.hbase.client.SnapshotDescription;
 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -357,6 +357,7 @@ public final class SnapshotTestingUtils {
   public static ArrayList<String> listHFileNames(final FileSystem fs, final Path tableDir)
       throws IOException {
     final ArrayList<String> hfiles = new ArrayList<String>();
+    /* TODO
     FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
       @Override
       public void storeFile(final String region, final String family, final String hfileName)
@@ -364,6 +365,7 @@ public final class SnapshotTestingUtils {
         hfiles.add(hfileName);
       }
     });
+    */
     Collections.sort(hfiles);
     return hfiles;
   }
@@ -699,7 +701,7 @@ public final class SnapshotTestingUtils {
 
         // First region, simple with one plain hfile.
         HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
-        HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
+        RegionFileSystem rfs = RegionFileSystem.open(conf, fs, tableDir, hri, true);
         regions[i] = new RegionData(tableDir, hri, 3);
         for (int j = 0; j < regions[i].files.length; ++j) {
           Path storeFile = createStoreFile(rfs.createTempName());
@@ -711,7 +713,7 @@ public final class SnapshotTestingUtils {
         startKey = Bytes.toBytes(2 + i * 2);
         endKey = Bytes.toBytes(3 + i * 2);
         hri = new HRegionInfo(htd.getTableName());
-        rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
+        rfs = RegionFileSystem.open(conf, fs, tableDir, hri, true);
         regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
         for (int j = 0; j < regions[i].files.length; ++j) {
           String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
index da51516..f6867f2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
@@ -61,12 +61,12 @@ import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.fs.legacy.LegacyLayout;
 import org.apache.hadoop.hbase.master.RegionStates;
 import org.apache.hadoop.hbase.mob.MobFileName;
 import org.apache.hadoop.hbase.mob.MobUtils;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
 import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
@@ -200,7 +200,7 @@ public class BaseTestHBaseFsck {
           FileSystem fs = rootDir.getFileSystem(conf);
           Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
               hri.getEncodedName());
-          Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
+          Path hriPath = LegacyLayout.getRegionInfoFile(p);
           fs.delete(hriPath, true);
         }
 
@@ -582,7 +582,7 @@ public class BaseTestHBaseFsck {
       FileSystem fs = rootDir.getFileSystem(conf);
       Path p = new Path(rootDir + "/" + TableName.META_TABLE_NAME.getNameAsString(),
           hri.getEncodedName());
-      Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
+      Path hriPath = LegacyLayout.getRegionInfoFile(p);
       fs.delete(hriPath, true);
     }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java
index 97b6ddb..f5c2ac4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java
@@ -48,7 +48,7 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -206,15 +206,7 @@ public class OfflineMetaRebuildTestCore {
     HRegionInfo hri = new HRegionInfo(htbl.getName(), startKey, endKey);
 
     LOG.info("manually adding regioninfo and hdfs data: " + hri.toString());
-    Path rootDir = FSUtils.getRootDir(conf);
-    FileSystem fs = rootDir.getFileSystem(conf);
-    Path p = new Path(FSUtils.getTableDir(rootDir, htbl.getName()),
-        hri.getEncodedName());
-    fs.mkdirs(p);
-    Path riPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
-    FSDataOutputStream out = fs.create(riPath);
-    out.write(hri.toDelimitedByteArray());
-    out.close();
+    RegionFileSystem rfs = RegionFileSystem.open(conf, hri, true);
 
     // add to meta.
     MetaTableAccessor.addRegionToMeta(meta, hri);

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
index c7ea5f2..2b1bc8f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
@@ -349,7 +349,7 @@ public final class WALPerformanceEvaluation extends Configured implements Tool {
         long putTime = runBenchmark(benchmarks, numThreads);
         logBenchmarkResult("Summary: threads=" + numThreads + ", iterations=" + numIterations +
           ", syncInterval=" + syncInterval, numIterations * numThreads, putTime);
-        
+
         for (int i = 0; i < numRegions; i++) {
           if (regions[i] != null) {
             closeRegion(regions[i]);
@@ -451,7 +451,7 @@ public final class WALPerformanceEvaluation extends Configured implements Tool {
   private static void logBenchmarkResult(String testName, long numTests, long totalTime) {
     float tsec = totalTime / 1000.0f;
     LOG.info(String.format("%s took %.3fs %.3fops/s", testName, tsec, numTests / tsec));
-    
+
   }
 
   private void printUsageAndExit() {
@@ -531,8 +531,8 @@ public final class WALPerformanceEvaluation extends Configured implements Tool {
         }
       });
     }
-     
-    return HRegion.createHRegion(regionInfo, dir, getConf(), htd, wal);
+
+    return HRegion.createHRegion(getConf(), dir, htd, regionInfo, wal);
   }
 
   private void closeRegion(final HRegion region) throws IOException {


[15/18] hbase git commit: HBASE-14439 API cleanup

Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 420d77f..c980ae6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -76,7 +76,7 @@ import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.MasterRpcServices;
@@ -1033,11 +1033,11 @@ public class TestSplitTransactionOnCluster {
       assertEquals(storefiles.size(), 1);
       assertFalse(region.hasReferences());
       Path referencePath =
-          region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f",
+          region.getRegionStorage().splitStoreFile(region.getRegionInfo(), "f",
             storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
       assertNull(referencePath);
       referencePath =
-          region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "i_f",
+          region.getRegionStorage().splitStoreFile(region.getRegionInfo(), "i_f",
             storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
       assertNotNull(referencePath);
     } finally {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
index 7f44324..4fc5a11 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
@@ -63,7 +63,7 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.io.compress.Compression;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -186,7 +186,7 @@ public class TestStore {
     final Configuration walConf = new Configuration(conf);
     FSUtils.setRootDir(walConf, basedir);
     final WALFactory wals = new WALFactory(walConf, null, methodName);
-    RegionFileSystem rfs = RegionFileSystem.open(conf, fs, basedir, info, false);
+    RegionStorage rfs = RegionStorage.open(conf, fs, basedir, info, false);
     HRegion region = new HRegion(rfs, htd,
       wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), null);
 
@@ -774,7 +774,7 @@ public class TestStore {
         LOG.info("Before flush, we should have no files");
 
         Collection<StoreFileInfo> files =
-          store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
+          store.getRegionStorage().getStoreFiles(store.getColumnFamilyName());
         Assert.assertEquals(0, files != null ? files.size() : 0);
 
         //flush
@@ -787,7 +787,7 @@ public class TestStore {
         }
 
         LOG.info("After failed flush, we should still have no files!");
-        files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
+        files = store.getRegionStorage().getStoreFiles(store.getColumnFamilyName());
         Assert.assertEquals(0, files != null ? files.size() : 0);
         store.getHRegion().getWAL().close();
         return null;
@@ -1028,7 +1028,7 @@ public class TestStore {
     for (int i = 0; i <= index; i++) {
       sf = it.next();
     }
-    store.getRegionFileSystem().removeStoreFiles(store.getColumnFamilyName(), Lists.newArrayList(sf));
+    store.getRegionStorage().removeStoreFiles(store.getColumnFamilyName(), Lists.newArrayList(sf));
   }
 
   private void closeCompactedFile(int index) throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
index e88a948..e5a9f00 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
@@ -106,7 +106,7 @@ public class TestStoreFile extends HBaseTestCase {
   public void testBasicHalfMapFile() throws Exception {
     final HRegionInfo hri =
         new HRegionInfo(TableName.valueOf("testBasicHalfMapFileTb"));
-    RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, testDir, hri, true);
+    RegionStorage regionFs = RegionStorage.open(conf, fs, testDir, hri, true);
 
     HFileContext meta = new HFileContextBuilder().withBlockSize(2*1024).build();
     StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
@@ -156,7 +156,7 @@ public class TestStoreFile extends HBaseTestCase {
   @Test
   public void testReference() throws IOException {
     final HRegionInfo hri = new HRegionInfo(TableName.valueOf("testReferenceTb"));
-    RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, testDir, hri, true);
+    RegionStorage regionFs = RegionStorage.open(conf, fs, testDir, hri, true);
 
     HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
     // Make a store file and write data to it.
@@ -221,7 +221,7 @@ public class TestStoreFile extends HBaseTestCase {
     // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/
     Configuration testConf = new Configuration(this.conf);
     FSUtils.setRootDir(testConf, testDir);
-    RegionFileSystem regionFs = RegionFileSystem.open(testConf, fs, testDir, hri, true);
+    RegionStorage regionFs = RegionStorage.open(testConf, fs, testDir, hri, true);
     HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
 
     // Make a store file and write data to it.
@@ -265,7 +265,7 @@ public class TestStoreFile extends HBaseTestCase {
 
     // adding legal table name chars to verify regex handles it.
     HRegionInfo hri = new HRegionInfo(TableName.valueOf("_original-evil-name"));
-    RegionFileSystem regionFs = RegionFileSystem.open(testConf, fs, testDir, hri, true);
+    RegionStorage regionFs = RegionStorage.open(testConf, fs, testDir, hri, true);
 
     HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
     // Make a store file and write data to it. <root>/<tablename>/<rgn>/<cf>/<file>
@@ -278,7 +278,7 @@ public class TestStoreFile extends HBaseTestCase {
 
     // create link to store file. <root>/clone/region/<cf>/<hfile>-<region>-<table>
     HRegionInfo hriClone = new HRegionInfo(TableName.valueOf("clone"));
-    RegionFileSystem cloneRegionFs = RegionFileSystem.open(testConf, fs, testDir, hriClone, true);
+    RegionStorage cloneRegionFs = RegionStorage.open(testConf, fs, testDir, hriClone, true);
     Path dstPath = cloneRegionFs.getStoreDir(TEST_FAMILY);
     HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
     Path linkFilePath = new Path(dstPath,
@@ -331,7 +331,7 @@ public class TestStoreFile extends HBaseTestCase {
     assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
   }
 
-  private void checkHalfHFile(final RegionFileSystem regionFs, final StoreFile f)
+  private void checkHalfHFile(final RegionStorage regionFs, final StoreFile f)
       throws IOException {
     Cell midkey = f.createReader().midkey();
     KeyValue midKV = (KeyValue)midkey;
@@ -995,7 +995,7 @@ public class TestStoreFile extends HBaseTestCase {
     assertEquals(startEvicted, cs.getEvictedCount());
   }
 
-  private Path splitStoreFile(final RegionFileSystem regionFs, final HRegionInfo hri,
+  private Path splitStoreFile(final RegionStorage regionFs, final HRegionInfo hri,
       final String family, final StoreFile sf, final byte[] splitKey, boolean isTopRef)
       throws IOException {
     FileSystem fs = regionFs.getFileSystem();

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java
index 7435567..de193cf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileRefresherChore.java
@@ -46,8 +46,8 @@ import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
-import org.apache.hadoop.hbase.fs.legacy.LegacyRegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.legacy.LegacyRegionStorage;
 import org.apache.hadoop.hbase.wal.WALFactory;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -81,9 +81,9 @@ public class TestStoreFileRefresherChore {
     return htd;
   }
 
-  static class FailingHRegionFileSystem extends LegacyRegionFileSystem {
+  static class FailingHRegionStorage extends LegacyRegionStorage {
     boolean fail = false;
-    FailingHRegionFileSystem(Configuration conf, FileSystem fs, Path tableDir, HRegionInfo regionInfo) {
+    FailingHRegionStorage(Configuration conf, FileSystem fs, Path tableDir, HRegionInfo regionInfo) {
       super(conf, fs, tableDir, regionInfo);
     }
 
@@ -103,7 +103,7 @@ public class TestStoreFileRefresherChore {
 
     HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false, 0, replicaId);
 
-    RegionFileSystem fs = new FailingHRegionFileSystem(conf, tableDir.getFileSystem(conf), tableDir,
+    RegionStorage fs = new FailingHRegionStorage(conf, tableDir.getFileSystem(conf), tableDir,
       info);
     final Configuration walConf = new Configuration(conf);
     FSUtils.setRootDir(walConf, tableDir);
@@ -194,7 +194,7 @@ public class TestStoreFileRefresherChore {
     verifyData(replica1, 0, 100, qf, families);
 
     // simulate an fs failure where we cannot refresh the store files for the replica
-    ((FailingHRegionFileSystem)((HRegion)replica1).getRegionFileSystem()).fail = true;
+    ((FailingHRegionStorage)((HRegion)replica1).getRegionStorage()).fail = true;
 
     // write some more data to primary and flush
     putData(primary, 100, 100, qf, families);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
index faa539e..73ce47c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
@@ -643,7 +643,7 @@ public abstract class AbstractTestWALReplay {
     for (HColumnDescriptor hcd: htd.getFamilies()) {
       cf_count++;
       if (cf_count == 2) {
-        region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
+        region.getRegionStorage().deleteFamily(hcd.getNameAsString());
       }
     }
 
@@ -963,7 +963,7 @@ public abstract class AbstractTestWALReplay {
     final int countPerFamily = 10;
     final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
     HRegion region1 = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
-    Path regionDir = region1.getRegionFileSystem().getRegionDir();
+    Path regionDir = region1.getRegionStorage().getRegionDir();
     HBaseTestingUtility.closeRegionAndWAL(region1);
 
     WAL wal = createWAL(this.conf, hbaseRootDir, logName);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index d943ccd..d8ceafb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -56,7 +56,7 @@ import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
 import org.apache.hadoop.hbase.fs.MasterFileSystem;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.master.HMaster;
@@ -701,7 +701,7 @@ public final class SnapshotTestingUtils {
 
         // First region, simple with one plain hfile.
         HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
-        RegionFileSystem rfs = RegionFileSystem.open(conf, fs, tableDir, hri, true);
+        RegionStorage rfs = RegionStorage.open(conf, fs, tableDir, hri, true);
         regions[i] = new RegionData(tableDir, hri, 3);
         for (int j = 0; j < regions[i].files.length; ++j) {
           Path storeFile = createStoreFile(rfs.createTempName());
@@ -713,7 +713,7 @@ public final class SnapshotTestingUtils {
         startKey = Bytes.toBytes(2 + i * 2);
         endKey = Bytes.toBytes(3 + i * 2);
         hri = new HRegionInfo(htd.getTableName());
-        rfs = RegionFileSystem.open(conf, fs, tableDir, hri, true);
+        rfs = RegionStorage.open(conf, fs, tableDir, hri, true);
         regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
         for (int j = 0; j < regions[i].files.length; ++j) {
           String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java
index 7b5aed5..ab974e76 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java
@@ -227,7 +227,7 @@ public class HFileArchiveTestingUtil {
   public static Path getStoreArchivePath(Configuration conf, HRegion region, Store store)
       throws IOException {
     return HFileArchiveUtil.getStoreArchivePath(conf, region.getRegionInfo(),
-        region.getRegionFileSystem().getTableDir(), store.getFamily().getName());
+        region.getRegionStorage().getTableDir(), store.getFamily().getName());
   }
 
   public static Path getStoreArchivePath(HBaseTestingUtility util, String tableName,

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java
index f5c2ac4..4697aa1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRebuildTestCore.java
@@ -48,7 +48,7 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -206,7 +206,7 @@ public class OfflineMetaRebuildTestCore {
     HRegionInfo hri = new HRegionInfo(htbl.getName(), startKey, endKey);
 
     LOG.info("manually adding regioninfo and hdfs data: " + hri.toString());
-    RegionFileSystem rfs = RegionFileSystem.open(conf, hri, true);
+    RegionStorage rfs = RegionStorage.open(conf, hri, true);
 
     // add to meta.
     MetaTableAccessor.addRegionToMeta(meta, hri);


[06/18] hbase git commit: HBASE-14439 break everything HRegion

Posted by bu...@apache.org.
HBASE-14439 break everything HRegion


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c347b84c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c347b84c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c347b84c

Branch: refs/heads/hbase-14439
Commit: c347b84cd4e832402b5f4cc1c332f3d8f560bdd2
Parents: 7546190
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Tue Oct 20 16:22:07 2015 -0700
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Sep 29 11:07:29 2016 -0500

----------------------------------------------------------------------
 .../hadoop/hbase/fs/RegionFileSystem.java       |  132 ++-
 .../hadoop/hbase/fs/legacy/LegacyLayout.java    |    2 +-
 .../hbase/fs/legacy/LegacyMasterFileSystem.java |    2 +-
 .../hbase/fs/legacy/LegacyRegionFileSystem.java |  101 +-
 .../hadoop/hbase/master/CatalogJanitor.java     |   18 +-
 .../hbase/regionserver/CompactionTool.java      |   18 +-
 .../hadoop/hbase/regionserver/HRegion.java      |  113 +-
 .../hbase/regionserver/HRegionFileSystem.java   | 1109 ------------------
 .../hadoop/hbase/regionserver/HStore.java       |    5 +-
 .../regionserver/RegionMergeTransaction.java    |    6 +-
 .../RegionMergeTransactionImpl.java             |   19 +-
 .../regionserver/SplitTransactionImpl.java      |    3 +-
 .../hbase/snapshot/RestoreSnapshotHelper.java   |    8 +-
 .../hadoop/hbase/snapshot/SnapshotManifest.java |    6 +-
 .../hbase/snapshot/SnapshotManifestV1.java      |   24 +-
 .../org/apache/hadoop/hbase/util/HBaseFsck.java |   27 +-
 .../hadoop/hbase/util/HBaseFsckRepair.java      |    2 +-
 .../hadoop/hbase/util/ModifyRegionUtils.java    |    2 +-
 .../hadoop/hbase/util/RegionSplitter.java       |   12 +-
 .../hadoop/hbase/HBaseTestingUtility.java       |    6 +-
 .../org/apache/hadoop/hbase/TestIOFencing.java  |   25 +-
 .../client/TestRestoreSnapshotFromClient.java   |   16 +-
 .../TestRegionObserverScannerOpenHook.java      |   11 +-
 .../TableSnapshotInputFormatTestBase.java       |   38 +-
 .../MasterProcedureTestingUtility.java          |   19 +-
 .../hbase/regionserver/TestAtomicOperation.java |   27 +-
 .../hadoop/hbase/regionserver/TestBulkLoad.java |    8 +-
 .../TestDefaultCompactSelection.java            |  163 ++-
 .../hbase/regionserver/TestDefaultMemStore.java |    2 +-
 .../hbase/regionserver/TestHMobStore.java       |    7 +-
 .../hadoop/hbase/regionserver/TestHRegion.java  |   23 +-
 .../regionserver/TestHRegionFileSystem.java     |   30 +-
 .../hbase/regionserver/TestHRegionInfo.java     |   10 +-
 .../regionserver/TestHRegionReplayEvents.java   |    5 +-
 .../hbase/regionserver/TestRecoveredEdits.java  |    6 +-
 .../TestRegionMergeTransactionOnCluster.java    |   18 +-
 .../TestScannerHeartbeatMessages.java           |   12 +-
 .../TestSplitTransactionOnCluster.java          |   16 +-
 .../hadoop/hbase/regionserver/TestStore.java    |    9 +-
 .../hbase/regionserver/TestStoreFile.java       |   24 +-
 .../TestStoreFileRefresherChore.java            |   12 +-
 .../hbase/regionserver/wal/TestDurability.java  |    4 +-
 .../hbase/snapshot/SnapshotTestingUtils.java    |    8 +-
 .../hadoop/hbase/util/BaseTestHBaseFsck.java    |    6 +-
 .../util/hbck/OfflineMetaRebuildTestCore.java   |   12 +-
 .../hbase/wal/WALPerformanceEvaluation.java     |    8 +-
 46 files changed, 643 insertions(+), 1491 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
index 3b94355..154d648 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.hbase.ClusterId;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.fs.legacy.LegacyRegionFileSystem;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -46,6 +46,61 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.mob.MobConstants;
 import org.apache.hadoop.hbase.mob.MobUtils;
 
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.FSUtilsWithRetries;
+import org.apache.hadoop.hbase.fs.FsContext;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.MetaUtils;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.regionserver.*;
+import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSHDFSUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
+
 @InterfaceAudience.Private
 public abstract class RegionFileSystem {
   private static Log LOG = LogFactory.getLog(RegionFileSystem.class);
@@ -135,6 +190,78 @@ public abstract class RegionFileSystem {
   protected abstract void destroy() throws IOException;
 
   // ==========================================================================
+  //  NOOOOO
+  // ==========================================================================
+  public abstract void checkRegionInfoOnFilesystem() throws IOException;
+  public abstract Path getRegionDir();
+  public abstract Path getTableDir();
+
+  public abstract Path getTempDir();
+
+  public HRegionInfo getRegionInfoForFS() { return hri; }
+
+  public abstract Path getStoreDir(final String familyName);
+  public abstract Path createTempName();
+  public abstract Path createStoreDir(final String familyName) throws IOException;
+  public abstract Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
+      throws IOException;
+
+  public abstract void cleanupTempDir() throws IOException;
+  public abstract void cleanupSplitsDir() throws IOException;
+  public abstract void cleanupMergesDir() throws IOException;
+  public abstract void cleanupAnySplitDetritus() throws IOException;
+
+  public abstract Path commitDaughterRegion(final HRegionInfo regionInfo)
+      throws IOException;
+  public abstract void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException;
+  public abstract StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
+      throws IOException;
+
+  public abstract Path commitStoreFile(final String familyName, final Path buildPath) throws IOException;
+  public abstract void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException;
+
+  public abstract void removeStoreFile(final String familyName, final Path filePath)
+      throws IOException;
+  public abstract void removeStoreFiles(final String familyName, final Collection<StoreFile> storeFiles)
+      throws IOException;
+
+  public abstract boolean hasReferences(final String familyName) throws IOException;
+  public abstract boolean hasReferences(final HTableDescriptor htd) throws IOException;
+
+  public abstract Path getStoreFilePath(final String familyName, final String fileName);
+
+  public abstract void logFileSystemState(final Log LOG) throws IOException;
+
+  public abstract void createSplitsDir() throws IOException;
+  public abstract Path getSplitsDir();
+  public abstract Path getSplitsDir(final HRegionInfo hri);
+
+  public abstract Path getMergesDir();
+  public abstract void createMergesDir() throws IOException;
+
+  public abstract Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
+      final StoreFile f, final Path mergedDir)
+      throws IOException;
+
+  public abstract void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException;
+
+  public abstract Path splitStoreFile(final HRegionInfo hri, final String familyName,
+      final StoreFile f, final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy)
+          throws IOException;
+
+  public abstract void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException;
+
+  public static HRegionInfo loadRegionInfoFileContent(FileSystem fs, Path regionDir)
+      throws IOException {
+    FSDataInputStream in = fs.open(new Path(regionDir, ".regioninfo"));
+    try {
+      return HRegionInfo.parseFrom(in);
+    } finally {
+      in.close();
+    }
+  }
+
+  // ==========================================================================
   //  PUBLIC
   // ==========================================================================
   public static RegionFileSystem open(Configuration conf, HRegionInfo regionInfo, boolean bootstrap)
@@ -155,8 +282,9 @@ public abstract class RegionFileSystem {
     RegionFileSystem rfs = getInstance(conf, fs, rootDir, regionInfo);
     if (bootstrap) {
       // TODO: are bootstrap and create two different things?
-      // should switch to bootstrap & read-only 
+      // should switch to bootstrap & read-only
       // legacy region wants to recover the .regioninfo :(
+      rfs.bootstrap();
     }
     return rfs;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
index 8c26cdf..53f4fb0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.hbase.mob.MobConstants;
 
 public final class LegacyLayout {
   /** Name of the region info file that resides just under the region directory. */
-  private final static String REGION_INFO_FILE = ".regioninfo";
+  public final static String REGION_INFO_FILE = ".regioninfo";
 
   /** Temporary subdirectory of the region directory used for merges. */
   public static final String REGION_MERGES_DIR = ".merges";

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
index b2675a0..f6201a1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
@@ -238,7 +238,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
       HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
       HTableDescriptor metaDescriptor = HTableDescriptor.metaTableDescriptor(c);
       MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, false);
-      HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor, null);
+      HRegion meta = HRegion.createHRegion(c, rd, metaDescriptor, metaHRI, null);
       MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, true);
       meta.close();
     } catch (IOException e) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java
index d07a1f1..ce914a0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java
@@ -24,6 +24,7 @@ import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -45,7 +46,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableDescriptor;
 import org.apache.hadoop.hbase.fs.FSUtilsWithRetries;
 import org.apache.hadoop.hbase.fs.FsContext;
 import org.apache.hadoop.hbase.fs.RegionFileSystem;
@@ -61,12 +61,10 @@ import org.apache.hadoop.hbase.mob.MobConstants;
 import org.apache.hadoop.hbase.mob.MobUtils;
 
 import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.regionserver.*;
@@ -101,10 +99,14 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
     this.regionInfoForFs = ServerRegionReplicaUtil.getRegionInfoForFs(hri);
   }
 
-  private Path getRegionDir() {
+  public Path getRegionDir() {
     return regionDir;
   }
 
+  public Path getTableDir() {
+    return tableDir;
+  }
+
   // ==========================================================================
   //  PUBLIC Methods - Families Related
   // ==========================================================================
@@ -112,7 +114,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
   public Collection<String> getFamilies() throws IOException {
     FileSystem fs = getFileSystem();
     FileStatus[] fds = FSUtils.listStatus(fs, regionDir, new FSUtils.FamilyDirFilter(fs));
-    if (fds == null) return null;
+    if (fds == null) return Collections.emptyList();
 
     ArrayList<String> families = new ArrayList<String>(fds.length);
     for (FileStatus status: fds) {
@@ -161,14 +163,14 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
   //  Temp Helpers
   // ===========================================================================
   /** @return {@link Path} to the region's temp directory, used for file creations */
-  Path getTempDir() {
+  public Path getTempDir() {
     return LegacyLayout.getRegionTempDir(regionDir);
   }
 
   /**
    * Clean up any temp detritus that may have been left around from previous operation attempts.
    */
-  void cleanupTempDir() throws IOException {
+  public void cleanupTempDir() throws IOException {
     fsWithRetries.deleteDir(getTempDir());
   }
 
@@ -190,7 +192,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @return {@link Path} to the directory of the specified family
    * @throws IOException if the directory creation fails.
    */
-  Path createStoreDir(final String familyName) throws IOException {
+  public Path createStoreDir(final String familyName) throws IOException {
     Path storeDir = getStoreDir(familyName);
     if (!fsWithRetries.createDir(storeDir))
       throw new IOException("Failed creating "+storeDir);
@@ -207,7 +209,9 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
     Path familyDir = getStoreDir(familyName);
     FileStatus[] files = FSUtils.listStatus(getFileSystem(), familyDir);
     if (files == null) {
-      LOG.debug("No StoreFiles for: " + familyDir);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("No StoreFiles for: " + familyDir);
+      }
       return null;
     }
 
@@ -233,7 +237,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @param fileName File Name
    * @return The qualified Path for the specified family/file
    */
-  Path getStoreFilePath(final String familyName, final String fileName) {
+  public Path getStoreFilePath(final String familyName, final String fileName) {
     Path familyDir = getStoreDir(familyName);
     return LegacyLayout.getStoreFile(familyDir, fileName).makeQualified(getFileSystem());
   }
@@ -245,7 +249,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @param fileName File Name
    * @return The {@link StoreFileInfo} for the specified family/file
    */
-  StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
+  public StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
       throws IOException {
     Path familyDir = getStoreDir(familyName);
     return ServerRegionReplicaUtil.getStoreFileInfo(getConfiguration(),
@@ -260,9 +264,18 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @throws IOException
    */
   public boolean hasReferences(final String familyName) throws IOException {
-    FileStatus[] files = FSUtils.listStatus(getFileSystem(),
-        getStoreDir(familyName), new FSUtils.ReferenceFileFilter(getFileSystem()));
-    return files != null && files.length > 0;
+    FileStatus[] files = FSUtils.listStatus(getFileSystem(), getStoreDir(familyName));
+    if (files != null) {
+      for(FileStatus stat: files) {
+        if(stat.isDirectory()) {
+          continue;
+        }
+        if(StoreFileInfo.isReference(stat.getPath())) {
+          return true;
+        }
+      }
+    }
+    return false;
   }
 
   /**
@@ -357,7 +370,9 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
     if (!fsWithRetries.exists(buildPath)) {
       throw new FileNotFoundException(buildPath.toString());
     }
-    LOG.debug("Committing store file " + buildPath + " as " + dstPath);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Committing store file " + buildPath + " as " + dstPath);
+    }
     // buildPath exists, therefore not doing an exists() check.
     if (!fsWithRetries.rename(buildPath, dstPath)) {
       throw new IOException("Failed rename of " + buildPath + " to " + dstPath);
@@ -371,7 +386,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @param storeFiles list of store files divided by family
    * @throws IOException
    */
-  void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException {
+  public void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException {
     for (Map.Entry<byte[], List<StoreFile>> es: storeFiles.entrySet()) {
       String familyName = Bytes.toString(es.getKey());
       for (StoreFile sf: es.getValue()) {
@@ -415,7 +430,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @return The destination {@link Path} of the bulk loaded file
    * @throws IOException
    */
-  Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
+  public Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
       throws IOException {
     // Copy the file if it's on another filesystem
     FileSystem fs = getFileSystem();
@@ -441,18 +456,18 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
   //  Splits Helpers
   // ===========================================================================
   /** @return {@link Path} to the temp directory used during split operations */
-  Path getSplitsDir() {
+  public Path getSplitsDir() {
     return LegacyLayout.getRegionSplitsDir(getRegionDir());
   }
 
-  Path getSplitsDir(final HRegionInfo hri) {
+  public Path getSplitsDir(final HRegionInfo hri) {
     return LegacyLayout.getRegionSplitsDir(getSplitsDir(), hri);
   }
 
   /**
    * Clean up any split detritus that may have been left around from previous split attempts.
    */
-  void cleanupSplitsDir() throws IOException {
+  public void cleanupSplitsDir() throws IOException {
     fsWithRetries.deleteDir(getSplitsDir());
   }
 
@@ -462,7 +477,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * Call this method on initial region deploy.
    * @throws IOException
    */
-  void cleanupAnySplitDetritus() throws IOException {
+  public void cleanupAnySplitDetritus() throws IOException {
     Path splitdir = this.getSplitsDir();
     if (!fsWithRetries.exists(splitdir)) return;
     // Look at the splitdir.  It could have the encoded names of the daughter
@@ -491,7 +506,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @param regionInfo daughter {@link HRegionInfo}
    * @throws IOException
    */
-  void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException {
+  public void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException {
     Path regionDir = LegacyLayout.getRegionDir(tableDir, regionInfo);
     if (!fsWithRetries.deleteDir(regionDir)) {
       throw new IOException("Failed delete of " + regionDir);
@@ -505,7 +520,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @param regionInfo                 daughter {@link org.apache.hadoop.hbase.HRegionInfo}
    * @throws IOException
    */
-  Path commitDaughterRegion(final HRegionInfo regionInfo)
+  public Path commitDaughterRegion(final HRegionInfo regionInfo)
       throws IOException {
     Path regionDir = LegacyLayout.getRegionDir(tableDir, regionInfo);
     Path daughterTmpDir = this.getSplitsDir(regionInfo);
@@ -529,7 +544,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
   /**
    * Create the region splits directory.
    */
-  void createSplitsDir() throws IOException {
+  public void createSplitsDir() throws IOException {
     createTempDir(getSplitsDir());
   }
 
@@ -558,7 +573,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @return Path to created reference.
    * @throws IOException
    */
-  Path splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f,
+  public Path splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f,
       final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy)
           throws IOException {
 
@@ -568,24 +583,24 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
       try {
         if (top) {
           //check if larger than last key.
-          KeyValue splitKey = KeyValueUtil.createFirstOnRow(splitRow);
-          Cell lastKey = f.createReader().getLastKey();
+          Cell splitKey = CellUtil.createFirstOnRow(splitRow);
+          Cell lastKey = f.getLastKey();
           // If lastKey is null means storefile is empty.
           if (lastKey == null) {
             return null;
           }
-          if (f.getReader().getComparator().compare(splitKey, lastKey) > 0) {
+          if (f.getComparator().compare(splitKey, lastKey) > 0) {
             return null;
           }
         } else {
           //check if smaller than first key
-          KeyValue splitKey = KeyValueUtil.createLastOnRow(splitRow);
-          Cell firstKey = f.createReader().getFirstKey();
+          Cell splitKey = CellUtil.createLastOnRow(splitRow);
+          Cell firstKey = f.getFirstKey();
           // If firstKey is null means storefile is empty.
           if (firstKey == null) {
             return null;
           }
-          if (f.getReader().getComparator().compare(splitKey, firstKey) < 0) {
+          if (f.getComparator().compare(splitKey, firstKey) < 0) {
             return null;
           }
         }
@@ -613,7 +628,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
   //  Merge Helpers
   // ===========================================================================
   /** @return {@link Path} to the temp directory used during merge operations */
-  Path getMergesDir() {
+  public Path getMergesDir() {
     return LegacyLayout.getRegionMergesDir(getRegionDir());
   }
 
@@ -624,7 +639,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
   /**
    * Clean up any merge detritus that may have been left around from previous merge attempts.
    */
-  void cleanupMergesDir() throws IOException {
+  public void cleanupMergesDir() throws IOException {
     fsWithRetries.deleteDir(getMergesDir());
   }
 
@@ -633,7 +648,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @param mergedRegion {@link HRegionInfo}
    * @throws IOException
    */
-  void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException {
+  public void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException {
     Path regionDir = LegacyLayout.getRegionDir(tableDir, mergedRegion);
     if (fsWithRetries.deleteDir(regionDir)) {
       throw new IOException("Failed delete of " + regionDir);
@@ -645,7 +660,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @throws IOException If merges dir already exists or we fail to create it.
    * @see HRegionFileSystem#cleanupMergesDir()
    */
-  void createMergesDir() throws IOException {
+  public void createMergesDir() throws IOException {
     createTempDir(getMergesDir());
   }
 
@@ -659,7 +674,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @return Path to created reference.
    * @throws IOException
    */
-  Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
+  public Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
       final StoreFile f, final Path mergedDir)
       throws IOException {
     Path referenceDir = new Path(new Path(mergedDir,
@@ -684,7 +699,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @param mergedRegionInfo merged region {@link HRegionInfo}
    * @throws IOException
    */
-  void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException {
+  public void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException {
     Path regionDir = new Path(this.tableDir, mergedRegionInfo.getEncodedName());
     Path mergedRegionTmpDir = this.getMergesDir(mergedRegionInfo);
     // Move the tmp dir in the expected location
@@ -704,7 +719,7 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
    * @param LOG log to output information
    * @throws IOException if an unexpected exception occurs
    */
-  void logFileSystemState(final Log LOG) throws IOException {
+  public void logFileSystemState(final Log LOG) throws IOException {
     FSUtils.logFileSystemState(getFileSystem(), this.getRegionDir(), LOG);
   }
 
@@ -738,6 +753,8 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
   // ==========================================================================
   @Override
   protected void bootstrap() throws IOException {
+    fsWithRetries.createDir(getRegionDir());
+
     // Cleanup temporary directories
     cleanupTempDir();
     cleanupSplitsDir();
@@ -747,8 +764,10 @@ public class LegacyRegionFileSystem extends RegionFileSystem {
     checkRegionInfoOnFilesystem();
   }
 
-  private void checkRegionInfoOnFilesystem() throws IOException {
-    // TODO
+  public void checkRegionInfoOnFilesystem() throws IOException {
+    writeRegionInfoFileContent(getConfiguration(), getFileSystem(),
+      LegacyLayout.getRegionInfoFile(getRegionDir()),
+      getRegionInfoFileContent(getRegionInfo()));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index e748c3b..1005449 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
@@ -199,10 +199,10 @@ public class CatalogJanitor extends ScheduledChore {
     Path rootdir = this.services.getMasterFileSystem().getRootDir();
     Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());
     HTableDescriptor htd = getTableDescriptor(mergedRegion.getTable());
-    HRegionFileSystem regionFs = null;
+    RegionFileSystem regionFs = null;
     try {
-      regionFs = HRegionFileSystem.openRegionFromFileSystem(
-          this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
+      regionFs = RegionFileSystem.open(
+          this.services.getConfiguration(), fs, tabledir, mergedRegion, false);
     } catch (IOException e) {
       LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
     }
@@ -397,8 +397,6 @@ public class CatalogJanitor extends ScheduledChore {
 
     Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());
 
-    HRegionFileSystem regionFs = null;
-
     try {
       if (!FSUtils.isExists(fs, daughterRegionDir)) {
         return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
@@ -410,10 +408,10 @@ public class CatalogJanitor extends ScheduledChore {
     }
 
     boolean references = false;
-    HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());
     try {
-      regionFs = HRegionFileSystem.openRegionFromFileSystem(
-          this.services.getConfiguration(), fs, tabledir, daughter, true);
+      final RegionFileSystem regionFs = RegionFileSystem.open(this.services.getConfiguration(),
+          fs, tabledir, daughter, false);
+      final HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());
 
       for (HColumnDescriptor family: parentDescriptor.getFamilies()) {
         if ((references = regionFs.hasReferences(family.getNameAsString()))) {
@@ -461,4 +459,4 @@ public class CatalogJanitor extends ScheduledChore {
     return cleanMergeRegion(region, mergeRegions.getFirst(),
         mergeRegions.getSecond());
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
index 65b32a4..73ed9e9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
@@ -52,9 +52,9 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
 import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
 import org.apache.hadoop.hbase.mapreduce.JobUtil;
@@ -112,7 +112,7 @@ public class CompactionTool extends Configured implements Tool {
         Path regionDir = path.getParent();
         Path tableDir = regionDir.getParent();
         HTableDescriptor htd = LegacyTableDescriptor.getTableDescriptorFromFs(fs, tableDir);
-        HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
+        HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
         compactStoreFiles(tableDir, htd, hri,
             path.getName(), compactOnce, major);
       } else if (isRegionDir(fs, path)) {
@@ -138,7 +138,7 @@ public class CompactionTool extends Configured implements Tool {
     private void compactRegion(final Path tableDir, final HTableDescriptor htd,
         final Path regionDir, final boolean compactOnce, final boolean major)
         throws IOException {
-      HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
+      HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
       for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
         compactStoreFiles(tableDir, htd, hri, familyDir.getName(), compactOnce, major);
       }
@@ -181,20 +181,14 @@ public class CompactionTool extends Configured implements Tool {
     private static HStore getStore(final Configuration conf, final FileSystem fs,
         final Path tableDir, final HTableDescriptor htd, final HRegionInfo hri,
         final String familyName, final Path tempDir) throws IOException {
-      HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, hri) {
-        @Override
-        public Path getTempDir() {
-          return tempDir;
-        }
-      };
-      HRegion region = new HRegion(regionFs, null, conf, htd, null);
+      RegionFileSystem regionFs = null;
+      HRegion region = new HRegion(regionFs, htd, null, null);
       return new HStore(region, htd.getFamily(Bytes.toBytes(familyName)), conf);
     }
   }
 
   private static boolean isRegionDir(final FileSystem fs, final Path path) throws IOException {
-    Path regionInfo = new Path(path, HRegionFileSystem.REGION_INFO_FILE);
-    return fs.exists(regionInfo);
+    return fs.exists(null);
   }
 
   private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/c347b84c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index f6d2e36..c284082 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -133,6 +133,7 @@ import org.apache.hadoop.hbase.filter.ByteArrayComparable;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.FilterWrapper;
 import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.io.TimeRange;
 import org.apache.hadoop.hbase.io.hfile.HFile;
@@ -287,7 +288,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
   final AtomicLong compactionNumBytesCompacted = new AtomicLong(0L);
 
   private final WAL wal;
-  private final HRegionFileSystem fs;
+  private final RegionFileSystem fs;
   protected final Configuration conf;
   private final Configuration baseConf;
   private final int rowLockWaitDuration;
@@ -616,11 +617,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
    */
   @Deprecated
   @VisibleForTesting
-  public HRegion(final Path tableDir, final WAL wal, final FileSystem fs,
+  private HRegion(final Path tableDir, final WAL wal, final FileSystem fs,
       final Configuration confParam, final HRegionInfo regionInfo,
-      final HTableDescriptor htd, final RegionServerServices rsServices) {
-    this(new HRegionFileSystem(confParam, fs, tableDir, regionInfo),
-      wal, confParam, htd, rsServices);
+      final HTableDescriptor htd, final RegionServerServices rsServices)
+      throws IOException {
+    this(RegionFileSystem.open(confParam, fs, tableDir, regionInfo, false), htd, wal, rsServices);
   }
 
   /**
@@ -639,7 +640,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
    * @param htd the table descriptor
    * @param rsServices reference to {@link RegionServerServices} or null
    */
-  public HRegion(final HRegionFileSystem fs, final WAL wal, final Configuration confParam,
+  public HRegion(final RegionFileSystem rfs, final HTableDescriptor htd, final WAL wal,
+      final RegionServerServices rsServices) {
+    this(rfs, wal, rfs.getConfiguration(), htd, rsServices);
+  }
+
+  private HRegion(final RegionFileSystem fs, final WAL wal, final Configuration confParam,
       final HTableDescriptor htd, final RegionServerServices rsServices) {
     if (htd == null) {
       throw new IllegalArgumentException("Need table descriptor");
@@ -842,9 +848,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
       // Remove temporary data left over from old regions
       status.setStatus("Cleaning up temporary data from old regions");
       fs.cleanupTempDir();
-    }
 
-    if (this.writestate.writesEnabled) {
       status.setStatus("Cleaning up detritus from prior splits");
       // Get rid of any splits or merges that were lost in-progress.  Clean out
       // these directories here on open.  We may be opening a region that was
@@ -1097,7 +1101,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
     FileSystem fs = tablePath.getFileSystem(conf);
 
-    HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tablePath, regionInfo);
+    RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, tablePath, regionInfo, false);
     for (HColumnDescriptor family: tableDescriptor.getFamilies()) {
       Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family.getNameAsString());
       if (storeFiles == null) continue;
@@ -1704,7 +1708,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
   }
 
   /** @return the {@link HRegionFileSystem} used by this region */
-  public HRegionFileSystem getRegionFileSystem() {
+  public RegionFileSystem getRegionFileSystem() {
     return this.fs;
   }
 
@@ -6296,20 +6300,27 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
    * @param htd the table descriptor
    * @return the new instance
    */
-  static HRegion newHRegion(Path tableDir, WAL wal, FileSystem fs,
-      Configuration conf, HRegionInfo regionInfo, final HTableDescriptor htd,
-      RegionServerServices rsServices) {
+  static HRegion newHRegion(Configuration conf, FileSystem fs, Path rootDir,
+      HTableDescriptor htd, HRegionInfo regionInfo, WAL wal, RegionServerServices rsServices)
+      throws IOException {
+    RegionFileSystem rfs = RegionFileSystem.open(conf, fs, rootDir, regionInfo, false);
+    return newHRegion(rfs, htd, wal, rsServices);
+  }
+
+  private static HRegion newHRegion(RegionFileSystem rfs, HTableDescriptor htd, WAL wal,
+      RegionServerServices rsServices) throws IOException {
     try {
+      Configuration conf = rfs.getConfiguration();
+
       @SuppressWarnings("unchecked")
       Class<? extends HRegion> regionClass =
           (Class<? extends HRegion>) conf.getClass(HConstants.REGION_IMPL, HRegion.class);
 
       Constructor<? extends HRegion> c =
-          regionClass.getConstructor(Path.class, WAL.class, FileSystem.class,
-              Configuration.class, HRegionInfo.class, HTableDescriptor.class,
-              RegionServerServices.class);
+          regionClass.getConstructor(RegionFileSystem.class, HTableDescriptor.class,
+              WAL.class, RegionServerServices.class);
 
-      return c.newInstance(tableDir, wal, fs, conf, regionInfo, htd, rsServices);
+      return c.newInstance(rfs, htd, wal, rsServices);
     } catch (Throwable e) {
       // todo: what should I throw here?
       throw new IllegalStateException("Could not instantiate a region instance.", e);
@@ -6326,27 +6337,23 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
    * @return new HRegion
    * @throws IOException
    */
-  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
-        final Configuration conf, final HTableDescriptor hTableDescriptor,
-        final WAL wal, final boolean initialize)
-  throws IOException {
+  public static HRegion createHRegion(final Configuration conf, final Path rootDir,
+        final HTableDescriptor hTableDescriptor, final HRegionInfo info,
+        final WAL wal, final boolean initialize) throws IOException {
     LOG.info("creating HRegion " + info.getTable().getNameAsString()
         + " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
         " Table name == " + info.getTable().getNameAsString());
-    FileSystem fs = FileSystem.get(conf);
-    Path tableDir = FSUtils.getTableDir(rootDir, info.getTable());
-    HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, info);
-    HRegion region = HRegion.newHRegion(tableDir, wal, fs, conf, info, hTableDescriptor, null);
+    FileSystem fs = rootDir.getFileSystem(conf);
+    RegionFileSystem rfs = RegionFileSystem.open(conf, fs, rootDir, info, true);
+    HRegion region = HRegion.newHRegion(rfs, hTableDescriptor, wal, null);
     if (initialize) region.initialize(null);
     return region;
   }
 
-  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
-                                      final Configuration conf,
-                                      final HTableDescriptor hTableDescriptor,
-                                      final WAL wal)
-    throws IOException {
-    return createHRegion(info, rootDir, conf, hTableDescriptor, wal, true);
+  public static HRegion createHRegion(final Configuration conf, final Path rootDir,
+      final HTableDescriptor hTableDescriptor, final HRegionInfo info, final WAL wal)
+      throws IOException {
+    return createHRegion(conf, rootDir, hTableDescriptor, info, wal, true);
   }
 
 
@@ -6480,36 +6487,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
       final Path rootDir, final HRegionInfo info, final HTableDescriptor htd, final WAL wal,
       final RegionServerServices rsServices, final CancelableProgressable reporter)
       throws IOException {
-    Path tableDir = FSUtils.getTableDir(rootDir, info.getTable());
-    return openHRegion(conf, fs, rootDir, tableDir, info, htd, wal, rsServices, reporter);
-  }
-
-  /**
-   * Open a Region.
-   * @param conf The Configuration object to use.
-   * @param fs Filesystem to use
-   * @param rootDir Root directory for HBase instance
-   * @param info Info for region to be opened.
-   * @param htd the table descriptor
-   * @param wal WAL for region to use. This method will call
-   * WAL#setSequenceNumber(long) passing the result of the call to
-   * HRegion#getMinSequenceId() to ensure the wal id is properly kept
-   * up.  HRegionStore does this every time it opens a new region.
-   * @param rsServices An interface we can request flushes against.
-   * @param reporter An interface we can report progress against.
-   * @return new HRegion
-   * @throws IOException
-   */
-  public static HRegion openHRegion(final Configuration conf, final FileSystem fs,
-      final Path rootDir, final Path tableDir, final HRegionInfo info, final HTableDescriptor htd,
-      final WAL wal, final RegionServerServices rsServices,
-      final CancelableProgressable reporter)
-      throws IOException {
     if (info == null) throw new NullPointerException("Passed region info is null");
     if (LOG.isDebugEnabled()) {
       LOG.debug("Opening region: " + info);
     }
-    HRegion r = HRegion.newHRegion(tableDir, wal, fs, conf, info, htd, rsServices);
+    HRegion r = HRegion.newHRegion(conf, fs, rootDir, htd, info, wal, rsServices);
     return r.openHRegion(reporter);
   }
 
@@ -6527,9 +6509,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
    */
   public static HRegion openHRegion(final HRegion other, final CancelableProgressable reporter)
       throws IOException {
-    HRegionFileSystem regionFs = other.getRegionFileSystem();
-    HRegion r = newHRegion(regionFs.getTableDir(), other.getWAL(), regionFs.getFileSystem(),
-        other.baseConf, other.getRegionInfo(), other.getTableDesc(), null);
+    RegionFileSystem regionFs = other.getRegionFileSystem();
+    HRegion r = newHRegion(other.baseConf, regionFs.getFileSystem(),
+      regionFs.getRootDir(), other.getTableDesc(), other.getRegionInfo(),
+      other.getWAL(), null);
     return r.openHRegion(reporter);
   }
 
@@ -6578,7 +6561,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     }
 
     Path rootDir = FSUtils.getRootDir(conf);
-    Path tableDir = FSUtils.getTableDir(rootDir, info.getTable());
 
     FileSystem fs = null;
     if (rsServices != null) {
@@ -6621,8 +6603,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
     fs.commitDaughterRegion(hri);
 
     // Create the daughter HRegion instance
-    HRegion r = HRegion.newHRegion(this.fs.getTableDir(), this.getWAL(), fs.getFileSystem(),
-        this.getBaseConf(), hri, this.getTableDesc(), rsServices);
+    HRegion r = HRegion.newHRegion(this.getBaseConf(), fs.getFileSystem(), this.fs.getRootDir(),
+        this.getTableDesc(), hri, this.getWAL(), rsServices);
     r.readRequestsCount.add(this.getReadRequestsCount() / 2);
     r.filteredReadRequestsCount.add(this.getFilteredReadRequestsCount() / 2);
     r.writeRequestsCount.add(this.getWriteRequestsCount() / 2);
@@ -6637,9 +6619,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
    */
   HRegion createMergedRegionFromMerges(final HRegionInfo mergedRegionInfo,
       final HRegion region_b) throws IOException {
-    HRegion r = HRegion.newHRegion(this.fs.getTableDir(), this.getWAL(),
-        fs.getFileSystem(), this.getBaseConf(), mergedRegionInfo,
-        this.getTableDesc(), this.rsServices);
+    HRegion r = HRegion.newHRegion(this.getBaseConf(), fs.getFileSystem(), this.fs.getRootDir(),
+        this.getTableDesc(), mergedRegionInfo, this.getWAL(), this.rsServices);
     r.readRequestsCount.add(this.getReadRequestsCount()
         + region_b.getReadRequestsCount());
     r.filteredReadRequestsCount.add(this.getFilteredReadRequestsCount()


[03/18] hbase git commit: HBASE-15984 Handle premature EOF treatment of WALs in replication.

Posted by bu...@apache.org.
HBASE-15984 Handle premature EOF treatment of WALs in replication.

In some particular deployments, the Replication code believes it has
reached EOF for a WAL prior to succesfully parsing all bytes known to
exist in a cleanly closed file.

Consistently this failure happens due to an InvalidProtobufException
after some number of seeks during our attempts to tail the in-progress
RegionServer WAL. As a work-around, this patch treats cleanly closed
files differently than other execution paths. If an EOF is detected due
to parsing or other errors while there are still unparsed bytes before
the end-of-file trailer, we now reset the WAL to the very beginning and
attempt a clean read-through.

In current testing, a single such reset is sufficient to work around
observed dataloss. However, the above change will retry a given WAL file
indefinitely. On each such attempt, a log message like the below will
be emitted at the WARN level:

  Processing end of WAL file '{}'. At position {}, which is too far away
  from reported file length {}. Restarting WAL reading (see HBASE-15983
  for details).

Additionally, this patch adds some additional log detail at the TRACE
level about file offsets seen while handling recoverable errors. It also
add metrics that measure the use of this recovery mechanism.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/76396714
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/76396714
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/76396714

Branch: refs/heads/hbase-14439
Commit: 76396714e1dbb3d7c65064c807f4131d0b2f11b8
Parents: 63808a2
Author: Sean Busbey <bu...@apache.org>
Authored: Tue Jun 7 16:00:46 2016 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Sep 29 10:07:14 2016 -0500

----------------------------------------------------------------------
 .../MetricsReplicationSourceSource.java         | 17 +++++
 .../MetricsReplicationGlobalSourceSource.java   | 43 +++++++++++
 .../MetricsReplicationSourceSourceImpl.java     | 79 ++++++++++++++++++++
 .../regionserver/wal/ProtobufLogReader.java     | 45 +++++++++--
 .../replication/regionserver/MetricsSource.java | 35 +++++++++
 .../regionserver/ReplicationSource.java         | 39 ++++++++--
 .../ReplicationWALReaderManager.java            | 10 +++
 src/main/asciidoc/_chapters/ops_mgt.adoc        | 24 +++++-
 8 files changed, 276 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/76396714/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
index c877608..9075a68 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
@@ -40,6 +40,16 @@ public interface MetricsReplicationSourceSource extends BaseSource {
   public static final String SOURCE_SHIPPED_HFILES = "source.shippedHFiles";
   public static final String SOURCE_SIZE_OF_HFILE_REFS_QUEUE = "source.sizeOfHFileRefsQueue";
 
+  public static final String SOURCE_CLOSED_LOGS_WITH_UNKNOWN_LENGTH =
+      "source.closedLogsWithUnknownFileLength";
+  public static final String SOURCE_UNCLEANLY_CLOSED_LOGS = "source.uncleanlyClosedLogs";
+  public static final String SOURCE_UNCLEANLY_CLOSED_IGNORED_IN_BYTES =
+      "source.ignoredUncleanlyClosedLogContentsInBytes";
+  public static final String SOURCE_RESTARTED_LOG_READING = "source.restartedLogReading";
+  public static final String SOURCE_REPEATED_LOG_FILE_BYTES = "source.repeatedLogFileBytes";
+  public static final String SOURCE_COMPLETED_LOGS = "source.completedLogs";
+  public static final String SOURCE_COMPLETED_RECOVERY_QUEUES = "source.completedRecoverQueues";
+
   void setLastShippedAge(long age);
   void incrSizeOfLogQueue(int size);
   void decrSizeOfLogQueue(int size);
@@ -55,4 +65,11 @@ public interface MetricsReplicationSourceSource extends BaseSource {
   void incrHFilesShipped(long hfiles);
   void incrSizeOfHFileRefsQueue(long size);
   void decrSizeOfHFileRefsQueue(long size);
+  void incrUnknownFileLengthForClosedWAL();
+  void incrUncleanlyClosedWALs();
+  void incrBytesSkippedInUncleanlyClosedWALs(final long bytes);
+  void incrRestartedWALReading();
+  void incrRepeatedFileBytes(final long bytes);
+  void incrCompletedWAL();
+  void incrCompletedRecoveryQueue();
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/76396714/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
index d595ca9..0a67663 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
@@ -36,6 +36,13 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
   private final MutableFastCounter logReadInBytesCounter;
   private final MutableFastCounter shippedHFilesCounter;
   private final MutableGaugeLong sizeOfHFileRefsQueueGauge;
+  private final MutableFastCounter unknownFileLengthForClosedWAL;
+  private final MutableFastCounter uncleanlyClosedWAL;
+  private final MutableFastCounter uncleanlyClosedSkippedBytes;
+  private final MutableFastCounter restartWALReading;
+  private final MutableFastCounter repeatedFileBytes;
+  private final MutableFastCounter completedWAL;
+  private final MutableFastCounter completedRecoveryQueue;
 
   public MetricsReplicationGlobalSourceSource(MetricsReplicationSourceImpl rms) {
     this.rms = rms;
@@ -62,6 +69,14 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
 
     sizeOfHFileRefsQueueGauge =
         rms.getMetricsRegistry().getGauge(SOURCE_SIZE_OF_HFILE_REFS_QUEUE, 0L);
+
+    unknownFileLengthForClosedWAL = rms.getMetricsRegistry().getCounter(SOURCE_CLOSED_LOGS_WITH_UNKNOWN_LENGTH, 0L);
+    uncleanlyClosedWAL = rms.getMetricsRegistry().getCounter(SOURCE_UNCLEANLY_CLOSED_LOGS, 0L);
+    uncleanlyClosedSkippedBytes = rms.getMetricsRegistry().getCounter(SOURCE_UNCLEANLY_CLOSED_IGNORED_IN_BYTES, 0L);
+    restartWALReading = rms.getMetricsRegistry().getCounter(SOURCE_RESTARTED_LOG_READING, 0L);
+    repeatedFileBytes = rms.getMetricsRegistry().getCounter(SOURCE_REPEATED_LOG_FILE_BYTES, 0L);
+    completedWAL = rms.getMetricsRegistry().getCounter(SOURCE_COMPLETED_LOGS, 0L);
+    completedRecoveryQueue = rms.getMetricsRegistry().getCounter(SOURCE_COMPLETED_RECOVERY_QUEUES, 0L);
   }
 
   @Override public void setLastShippedAge(long age) {
@@ -144,6 +159,34 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
     return (int)sizeOfLogQueueGauge.value();
   }
 
+  @Override
+  public void incrUnknownFileLengthForClosedWAL() {
+    unknownFileLengthForClosedWAL.incr(1L);
+  }
+  @Override
+  public void incrUncleanlyClosedWALs() {
+    uncleanlyClosedWAL.incr(1L);
+  }
+  @Override
+  public void incrBytesSkippedInUncleanlyClosedWALs(final long bytes) {
+    uncleanlyClosedSkippedBytes.incr(bytes);
+  }
+  @Override
+  public void incrRestartedWALReading() {
+    restartWALReading.incr(1L);
+  }
+  @Override
+  public void incrRepeatedFileBytes(final long bytes) {
+    repeatedFileBytes.incr(bytes);
+  }
+  @Override
+  public void incrCompletedWAL() {
+    completedWAL.incr(1L);
+  }
+  @Override
+  public void incrCompletedRecoveryQueue() {
+    completedRecoveryQueue.incr(1L);
+  }
 
   @Override
   public void init() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/76396714/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
index 5a6a103..5443981 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
@@ -51,6 +51,21 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
   private final MutableFastCounter shippedHFilesCounter;
   private final MutableGaugeLong sizeOfHFileRefsQueueGauge;
 
+  private final String unknownFileLengthKey;
+  private final String uncleanlyClosedKey;
+  private final String uncleanlySkippedBytesKey;
+  private final String restartedKey;
+  private final String repeatedBytesKey;
+  private final String completedLogsKey;
+  private final String completedRecoveryKey;
+  private final MutableFastCounter unknownFileLengthForClosedWAL;
+  private final MutableFastCounter uncleanlyClosedWAL;
+  private final MutableFastCounter uncleanlyClosedSkippedBytes;
+  private final MutableFastCounter restartWALReading;
+  private final MutableFastCounter repeatedFileBytes;
+  private final MutableFastCounter completedWAL;
+  private final MutableFastCounter completedRecoveryQueue;
+
   public MetricsReplicationSourceSourceImpl(MetricsReplicationSourceImpl rms, String id) {
     this.rms = rms;
     this.id = id;
@@ -88,6 +103,27 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
 
     sizeOfHFileRefsQueueKey = this.keyPrefix + "sizeOfHFileRefsQueue";
     sizeOfHFileRefsQueueGauge = rms.getMetricsRegistry().getGauge(sizeOfHFileRefsQueueKey, 0L);
+
+    unknownFileLengthKey = this.keyPrefix + "closedLogsWithUnknownFileLength";
+    unknownFileLengthForClosedWAL = rms.getMetricsRegistry().getCounter(unknownFileLengthKey, 0L);
+
+    uncleanlyClosedKey = this.keyPrefix + "uncleanlyClosedLogs";
+    uncleanlyClosedWAL = rms.getMetricsRegistry().getCounter(uncleanlyClosedKey, 0L);
+
+    uncleanlySkippedBytesKey = this.keyPrefix + "ignoredUncleanlyClosedLogContentsInBytes";
+    uncleanlyClosedSkippedBytes = rms.getMetricsRegistry().getCounter(uncleanlySkippedBytesKey, 0L);
+
+    restartedKey = this.keyPrefix + "restartedLogReading";
+    restartWALReading = rms.getMetricsRegistry().getCounter(restartedKey, 0L);
+
+    repeatedBytesKey = this.keyPrefix + "repeatedLogFileBytes";
+    repeatedFileBytes = rms.getMetricsRegistry().getCounter(repeatedBytesKey, 0L);
+
+    completedLogsKey = this.keyPrefix + "completedLogs";
+    completedWAL = rms.getMetricsRegistry().getCounter(completedLogsKey, 0L);
+
+    completedRecoveryKey = this.keyPrefix + "completedRecoverQueues";
+    completedRecoveryQueue = rms.getMetricsRegistry().getCounter(completedRecoveryKey, 0L);
   }
 
   @Override public void setLastShippedAge(long age) {
@@ -145,6 +181,14 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
 
     rms.removeMetric(shippedHFilesKey);
     rms.removeMetric(sizeOfHFileRefsQueueKey);
+
+    rms.removeMetric(unknownFileLengthKey);
+    rms.removeMetric(uncleanlyClosedKey);
+    rms.removeMetric(uncleanlySkippedBytesKey);
+    rms.removeMetric(restartedKey);
+    rms.removeMetric(repeatedBytesKey);
+    rms.removeMetric(completedLogsKey);
+    rms.removeMetric(completedRecoveryKey);
   }
 
   @Override
@@ -173,6 +217,41 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
   }
 
   @Override
+  public void incrUnknownFileLengthForClosedWAL() {
+    unknownFileLengthForClosedWAL.incr(1L);
+  }
+
+  @Override
+  public void incrUncleanlyClosedWALs() {
+    uncleanlyClosedWAL.incr(1L);
+  }
+
+  @Override
+  public void incrBytesSkippedInUncleanlyClosedWALs(final long bytes) {
+    uncleanlyClosedSkippedBytes.incr(bytes);
+  }
+
+  @Override
+  public void incrRestartedWALReading() {
+    restartWALReading.incr(1L);
+  }
+
+  @Override
+  public void incrRepeatedFileBytes(final long bytes) {
+    repeatedFileBytes.incr(bytes);
+  }
+
+  @Override
+  public void incrCompletedWAL() {
+    completedWAL.incr(1L);
+  }
+
+  @Override
+  public void incrCompletedRecoveryQueue() {
+    completedRecoveryQueue.incr(1L);
+  }
+
+  @Override
   public void init() {
     rms.init();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/76396714/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
index 0755358..15bff8b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
@@ -97,6 +97,21 @@ public class ProtobufLogReader extends ReaderBase {
   // cell codec classname
   private String codecClsName = null;
 
+  @InterfaceAudience.Private
+  public long trailerSize() {
+    if (trailerPresent) {
+      // sizeof PB_WAL_COMPLETE_MAGIC + sizof trailerSize + trailer
+      final long calculatedSize = PB_WAL_COMPLETE_MAGIC.length + Bytes.SIZEOF_INT + trailer.getSerializedSize();
+      final long expectedSize = fileLength - walEditsStopOffset;
+      if (expectedSize != calculatedSize) {
+        LOG.warn("After parsing the trailer, we expect the total footer to be "+ expectedSize +" bytes, but we calculate it as being " + calculatedSize);
+      }
+      return expectedSize;
+    } else {
+      return -1L;
+    }
+  }
+
   enum WALHdrResult {
     EOF,                   // stream is at EOF when method starts
     SUCCESS,
@@ -217,7 +232,7 @@ public class ProtobufLogReader extends ReaderBase {
     this.seekOnFs(currentPosition);
     if (LOG.isTraceEnabled()) {
       LOG.trace("After reading the trailer: walEditsStopOffset: " + this.walEditsStopOffset
-          + ", fileLength: " + this.fileLength + ", " + "trailerPresent: " + trailerPresent);
+          + ", fileLength: " + this.fileLength + ", " + "trailerPresent: " + (trailerPresent ? "true, size: " + trailer.getSerializedSize() : "false") + ", currentPosition: " + currentPosition);
     }
     
     codecClsName = hdrCtxt.getCellCodecClsName();
@@ -313,6 +328,9 @@ public class ProtobufLogReader extends ReaderBase {
       // OriginalPosition might be < 0 on local fs; if so, it is useless to us.
       long originalPosition = this.inputStream.getPos();
       if (trailerPresent && originalPosition > 0 && originalPosition == this.walEditsStopOffset) {
+        if (LOG.isTraceEnabled()) {
+          LOG.trace("Reached end of expected edits area at offset " + originalPosition);
+        }
         return false;
       }
       WALKey.Builder builder = WALKey.newBuilder();
@@ -322,7 +340,7 @@ public class ProtobufLogReader extends ReaderBase {
         try {
           int firstByte = this.inputStream.read();
           if (firstByte == -1) {
-            throw new EOFException("First byte is negative");
+            throw new EOFException("First byte is negative at offset " + originalPosition);
           }
           size = CodedInputStream.readRawVarint32(firstByte, this.inputStream);
           // available may be < 0 on local fs for instance.  If so, can't depend on it.
@@ -330,7 +348,7 @@ public class ProtobufLogReader extends ReaderBase {
           if (available > 0 && available < size) {
             throw new EOFException("Available stream not enough for edit, " +
                 "inputStream.available()= " + this.inputStream.available() + ", " +
-                "entry size= " + size);
+                "entry size= " + size + " at offset = " + this.inputStream.getPos());
           }
           ProtobufUtil.mergeFrom(builder, new LimitInputStream(this.inputStream, size),
             (int)size);
@@ -343,12 +361,14 @@ public class ProtobufLogReader extends ReaderBase {
           // TODO: not clear if we should try to recover from corrupt PB that looks semi-legit.
           //       If we can get the KV count, we could, theoretically, try to get next record.
           throw new EOFException("Partial PB while reading WAL, " +
-              "probably an unexpected EOF, ignoring");
+              "probably an unexpected EOF, ignoring. current offset=" + this.inputStream.getPos());
         }
         WALKey walKey = builder.build();
         entry.getKey().readFieldsFromPb(walKey, this.byteStringUncompressor);
         if (!walKey.hasFollowingKvCount() || 0 == walKey.getFollowingKvCount()) {
-          LOG.trace("WALKey has no KVs that follow it; trying the next one");
+          if (LOG.isTraceEnabled()) {
+            LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" + this.inputStream.getPos());
+          }
           continue;
         }
         int expectedCells = walKey.getFollowingKvCount();
@@ -363,7 +383,9 @@ public class ProtobufLogReader extends ReaderBase {
           try {
             posAfterStr = this.inputStream.getPos() + "";
           } catch (Throwable t) {
-            LOG.trace("Error getting pos for error message - ignoring", t);
+            if (LOG.isTraceEnabled()) {
+              LOG.trace("Error getting pos for error message - ignoring", t);
+            }
           }
           String message = " while reading " + expectedCells + " WAL KVs; started reading at "
               + posBefore + " and read up to " + posAfterStr;
@@ -378,11 +400,18 @@ public class ProtobufLogReader extends ReaderBase {
           throw new EOFException("Read WALTrailer while reading WALEdits");
         }
       } catch (EOFException eof) {
-        LOG.trace("Encountered a malformed edit, seeking back to last good position in file", eof);
         // If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)
-        if (originalPosition < 0) throw eof;
+        if (originalPosition < 0) {
+          if (LOG.isTraceEnabled()) {
+            LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);
+          }
+          throw eof;
+        }
         // Else restore our position to original location in hope that next time through we will
         // read successfully.
+        if (LOG.isTraceEnabled()) {
+          LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);
+        }
         seekOnFs(originalPosition);
         return false;
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/76396714/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
index 7dfeff6..a647d03 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
@@ -242,6 +242,41 @@ public class MetricsSource implements BaseSource {
     }
   }
 
+  public void incrUnknownFileLengthForClosedWAL() {
+    singleSourceSource.incrUnknownFileLengthForClosedWAL();
+    globalSourceSource.incrUnknownFileLengthForClosedWAL();
+  }
+
+  public void incrUncleanlyClosedWALs() {
+    singleSourceSource.incrUncleanlyClosedWALs();
+    globalSourceSource.incrUncleanlyClosedWALs();
+  }
+
+  public void incrBytesSkippedInUncleanlyClosedWALs(final long bytes) {
+    singleSourceSource.incrBytesSkippedInUncleanlyClosedWALs(bytes);
+    globalSourceSource.incrBytesSkippedInUncleanlyClosedWALs(bytes);
+  }
+
+  public void incrRestartedWALReading() {
+    singleSourceSource.incrRestartedWALReading();
+    globalSourceSource.incrRestartedWALReading();
+  }
+
+  public void incrRepeatedFileBytes(final long bytes) {
+    singleSourceSource.incrRepeatedFileBytes(bytes);
+    globalSourceSource.incrRepeatedFileBytes(bytes);
+  }
+
+  public void incrCompletedWAL() {
+    singleSourceSource.incrCompletedWAL();
+    globalSourceSource.incrCompletedWAL();
+  }
+
+  public void incrCompletedRecoveryQueue() {
+    singleSourceSource.incrCompletedRecoveryQueue();
+    globalSourceSource.incrCompletedRecoveryQueue();
+  }
+
   @Override
   public void init() {
     singleSourceSource.init();

http://git-wip-us.apache.org/repos/asf/hbase/blob/76396714/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index ce0fb06..3e17292 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -1197,24 +1197,49 @@ public class ReplicationSource extends Thread
     @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DE_MIGHT_IGNORE",
         justification = "Yeah, this is how it works")
     protected boolean processEndOfFile() {
+      // We presume this means the file we're reading is closed.
       if (this.queue.size() != 0) {
-        if (LOG.isTraceEnabled()) {
-          String filesize = "N/A";
-          try {
-            FileStatus stat = fs.getFileStatus(this.currentPath);
-            filesize = stat.getLen() + "";
-          } catch (IOException ex) {
+        // -1 means the wal wasn't closed cleanly.
+        final long trailerSize = this.repLogReader.currentTrailerSize();
+        final long currentPosition = this.repLogReader.getPosition();
+        FileStatus stat = null;
+        try {
+          stat = fs.getFileStatus(this.currentPath);
+        } catch (IOException exception) {
+          LOG.warn("Couldn't get file length information about log " + this.currentPath + ", it " + (trailerSize < 0 ? "was not" : "was") + " closed cleanly"
+              + ", stats: " + getStats());
+          metrics.incrUnknownFileLengthForClosedWAL();
+        }
+        if (stat != null) {
+          if (trailerSize < 0) {
+            if (currentPosition < stat.getLen()) {
+              final long skippedBytes = stat.getLen() - currentPosition;
+              LOG.info("Reached the end of WAL file '" + currentPath + "'. It was not closed cleanly, so we did not parse " + skippedBytes + " bytes of data.");
+              metrics.incrUncleanlyClosedWALs();
+              metrics.incrBytesSkippedInUncleanlyClosedWALs(skippedBytes);
+            }
+          } else if (currentPosition + trailerSize < stat.getLen()){
+            LOG.warn("Processing end of WAL file '" + currentPath + "'. At position " + currentPosition + ", which is too far away from reported file length " + stat.getLen() +
+                ". Restarting WAL reading (see HBASE-15983 for details). stats: " + getStats());
+            repLogReader.setPosition(0);
+            metrics.incrRestartedWALReading();
+            metrics.incrRepeatedFileBytes(currentPosition);
+            return false;
           }
+        }
+        if (LOG.isTraceEnabled()) {
           LOG.trace("Reached the end of log " + this.currentPath + ", stats: " + getStats()
-              + ", and the length of the file is " + filesize);
+              + ", and the length of the file is " + (stat == null ? "N/A" : stat.getLen()));
         }
         this.currentPath = null;
         this.repLogReader.finishCurrentFile();
         this.reader = null;
+        metrics.incrCompletedWAL();
         return true;
       } else if (this.replicationQueueInfo.isQueueRecovered()) {
         LOG.debug("Finished recovering queue for group " + walGroupId + " of peer "
             + peerClusterZnode);
+        metrics.incrCompletedRecoveryQueue();
         workerRunning = false;
         return true;
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/76396714/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationWALReaderManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationWALReaderManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationWALReaderManager.java
index b63f66b..3558d08 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationWALReaderManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationWALReaderManager.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader;
 import org.apache.hadoop.hbase.wal.WAL.Reader;
 import org.apache.hadoop.hbase.wal.WAL.Entry;
 import org.apache.hadoop.hbase.wal.WALFactory;
@@ -119,6 +120,15 @@ public class ReplicationWALReaderManager {
     this.position = pos;
   }
 
+  public long currentTrailerSize() {
+    long size = -1L;
+    if (reader instanceof ProtobufLogReader) {
+      final ProtobufLogReader pblr = (ProtobufLogReader)reader;
+      size = pblr.trailerSize();
+    }
+    return size;
+  }
+
   /**
    * Close the current reader
    * @throws IOException

http://git-wip-us.apache.org/repos/asf/hbase/blob/76396714/src/main/asciidoc/_chapters/ops_mgt.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc b/src/main/asciidoc/_chapters/ops_mgt.adoc
index 550d5f7..a580bf8 100644
--- a/src/main/asciidoc/_chapters/ops_mgt.adoc
+++ b/src/main/asciidoc/_chapters/ops_mgt.adoc
@@ -1614,7 +1614,7 @@ The new layout will be:
 
 === Replication Metrics
 
-The following metrics are exposed at the global region server level and (since HBase 0.95) at the peer level:
+The following metrics are exposed at the global region server level and at the peer level:
 
 `source.sizeOfLogQueue`::
   number of WALs to process (excludes the one which is being processed) at the Replication source
@@ -1628,6 +1628,28 @@ The following metrics are exposed at the global region server level and (since H
 `source.ageOfLastShippedOp`::
   age of last batch that was shipped by the replication source
 
+`source.completedLogs`::
+  The number of write-ahead-log files that have completed their acknowledged sending to the peer associated with this source. Increments to this metric are a part of normal operation of HBase replication.
+
+`source.completedRecoverQueues`::
+  The number of recovery queues this source has completed sending to the associated peer. Increments to this metric are a part of normal recovery of HBase replication in the face of failed Region Servers.
+
+`source.uncleanlyClosedLogs`::
+  The number of write-ahead-log files the replication system considered completed after reaching the end of readable entries in the face of an uncleanly closed file.
+
+`source.ignoredUncleanlyClosedLogContentsInBytes`::
+  When a write-ahead-log file is not closed cleanly, there will likely be some entry that has been partially serialized. This metric contains the number of bytes of such entries the HBase replication system believes were remaining at the end of files skipped in the face of an uncleanly closed file. Those bytes should either be in different file or represent a client write that was not acknowledged.
+
+`source.restartedLogReading`::
+  The number of times the HBase replication system detected that it failed to correctly parse a cleanly closed write-ahead-log file. In this circumstance, the system replays the entire log from the beginning, ensuring that no edits fail to be acknowledged by the associated peer. Increments to this metric indicate that the HBase replication system is having difficulty correctly handling failures in the underlying distributed storage system. No dataloss should occur, but you should check Region Server log files for details of the failures.
+
+`source.repeatedLogFileBytes`::
+  When the HBase replication system determines that it needs to replay a given write-ahead-log file, this metric is incremented by the number of bytes the replication system believes had already been acknowledged by the associated peer prior to starting over.
+
+`source.closedLogsWithUnknownFileLength`::
+  Incremented when the HBase replication system believes it is at the end of a write-ahead-log file but it can not determine the length of that file in the underlying distributed storage system. Could indicate dataloss since the replication system is unable to determine if the end of readable entries lines up with the expected end of the file. You should check Region Server log files for details of the failures.
+
+
 === Replication Configuration Options
 
 [cols="1,1,1", options="header"]


[07/18] hbase git commit: HBASE-14439 Moved from MasterFileSystem to MasterStorage abstract class for storage abstraction layer.

Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index e332c94..dc6e36b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -271,7 +271,7 @@ public class TestExportSnapshot {
       conf.setInt("mapreduce.map.maxattempts", 3);
     }
     // Export Snapshot
-    Path sourceDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    Path sourceDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
     int res = ExportSnapshot.innerMain(conf, new String[] {
       "-snapshot", Bytes.toString(snapshotName),
       "-copy-from", sourceDir.toString(),
@@ -355,7 +355,7 @@ public class TestExportSnapshot {
   }
 
   private Path getHdfsDestinationDir() {
-    Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
     Path path = new Path(new Path(rootDir, "export-test"), "export-" + System.currentTimeMillis());
     LOG.info("HDFS export destination path: " + path);
     return path;

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index a852547..d51b62a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -145,7 +145,7 @@ public class TestFlushSnapshotFromClient {
     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
 
     LOG.debug("FS state before snapshot:");
-    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+    UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
 
     // take a snapshot of the enabled table
     String snapshotString = "offlineTableSnapshot";
@@ -160,7 +160,7 @@ public class TestFlushSnapshotFromClient {
 
     // make sure its a valid snapshot
     LOG.debug("FS state after snapshot:");
-    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+    UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
 
     SnapshotTestingUtils.confirmSnapshotValid(UTIL,
       ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);
@@ -181,7 +181,7 @@ public class TestFlushSnapshotFromClient {
     UTIL.flush(TABLE_NAME);
 
     LOG.debug("FS state before snapshot:");
-    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+    UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
 
     // take a snapshot of the enabled table
     String snapshotString = "skipFlushTableSnapshot";
@@ -196,7 +196,7 @@ public class TestFlushSnapshotFromClient {
 
     // make sure its a valid snapshot
     LOG.debug("FS state after snapshot:");
-    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+    UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
 
     SnapshotTestingUtils.confirmSnapshotValid(UTIL,
       ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);
@@ -220,7 +220,7 @@ public class TestFlushSnapshotFromClient {
     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
 
     LOG.debug("FS state before snapshot:");
-    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+    UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
 
     // take a snapshot of the enabled table
     String snapshotString = "offlineTableSnapshot";
@@ -239,7 +239,7 @@ public class TestFlushSnapshotFromClient {
 
     // make sure its a valid snapshot
     LOG.debug("FS state after snapshot:");
-    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+    UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
 
     SnapshotTestingUtils.confirmSnapshotValid(UTIL,
       ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);
@@ -289,7 +289,7 @@ public class TestFlushSnapshotFromClient {
     HMaster master = UTIL.getMiniHBaseCluster().getMaster();
     SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
     LOG.info(" === Async Snapshot Completed ===");
-    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+    UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
 
     // make sure we get the snapshot
     SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
@@ -401,8 +401,8 @@ public class TestFlushSnapshotFromClient {
     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
 
     String snapshotName = "flushSnapshotCreateListDestroy";
-    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
-    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
+    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
     SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM),
       snapshotName, rootDir, fs, true);
   }
@@ -492,7 +492,7 @@ public class TestFlushSnapshotFromClient {
     }
 
     // dump for debugging
-    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+    UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
 
     List<SnapshotDescription> taken = admin.listSnapshots();
     int takenSize = taken.size();

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
index bf26c69..a32c747 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
@@ -207,7 +207,7 @@ public class TestRestoreFlushSnapshotFromClient {
   //  Helpers
   // ==========================================================================
   private void logFSTree() throws IOException {
-    UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+    UTIL.getMiniHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
   }
 
   protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
index e15609a..dd7f18e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
@@ -422,7 +422,7 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
       htdDisabled.addFamily(new HColumnDescriptor(FAM));
 
       // Write the .tableinfo
-      cluster.getMaster().getMasterFileSystem().createTableDescriptor(
+      cluster.getMaster().getMasterStorage().createTableDescriptor(
         new TableDescriptor(htdDisabled), true);
       List<HRegionInfo> disabledRegions =
           TEST_UTIL.createMultiRegionsInMeta(conf, htdDisabled, SPLIT_KEYS);


[02/18] hbase git commit: HBASE-16725 Don't let flushThread hang in TestHRegion

Posted by bu...@apache.org.
HBASE-16725 Don't let flushThread hang in TestHRegion


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/63808a22
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/63808a22
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/63808a22

Branch: refs/heads/hbase-14439
Commit: 63808a224c8689d07e55f90efd25f9597b0d04dd
Parents: f3270d4
Author: tedyu <yu...@gmail.com>
Authored: Thu Sep 29 06:56:19 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu Sep 29 06:56:19 2016 -0700

----------------------------------------------------------------------
 .../hadoop/hbase/regionserver/TestHRegion.java  | 40 ++++++++++++--------
 1 file changed, 25 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/63808a22/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 80d220d..79d305f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -3767,8 +3767,8 @@ public class TestHRegion {
 
     String method = "testFlushCacheWhileScanning";
     this.region = initHRegion(tableName, method, CONF, family);
+    FlushThread flushThread = new FlushThread();
     try {
-      FlushThread flushThread = new FlushThread();
       flushThread.start();
 
       Scan scan = new Scan();
@@ -3787,7 +3787,7 @@ public class TestHRegion {
         region.put(put);
 
         if (i != 0 && i % compactInterval == 0) {
-          // System.out.println("iteration = " + i);
+          LOG.debug("iteration = " + i+ " ts="+System.currentTimeMillis());
           region.compact(true);
         }
 
@@ -3806,15 +3806,20 @@ public class TestHRegion {
           if (!toggle) {
             flushThread.flush();
           }
-          assertEquals("i=" + i, expectedCount, res.size());
+          assertEquals("toggle="+toggle+"i=" + i + " ts="+System.currentTimeMillis(),
+              expectedCount, res.size());
           toggle = !toggle;
         }
       }
 
-      flushThread.done();
-      flushThread.join();
-      flushThread.checkNoError();
     } finally {
+      try {
+        flushThread.done();
+        flushThread.join();
+        flushThread.checkNoError();
+      } catch (InterruptedException ie) {
+        LOG.warn("Caught exception when joining with flushThread", ie);
+      }
       HBaseTestingUtility.closeRegionAndWAL(this.region);
       this.region = null;
     }
@@ -3904,12 +3909,12 @@ public class TestHRegion {
 
     String method = "testWritesWhileScanning";
     this.region = initHRegion(tableName, method, CONF, families);
+    FlushThread flushThread = new FlushThread();
+    PutThread putThread = new PutThread(numRows, families, qualifiers);
     try {
-      PutThread putThread = new PutThread(numRows, families, qualifiers);
       putThread.start();
       putThread.waitForFirstPut();
 
-      FlushThread flushThread = new FlushThread();
       flushThread.start();
 
       Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
@@ -3949,15 +3954,20 @@ public class TestHRegion {
 
       region.flush(true);
 
-      putThread.join();
-      putThread.checkNoError();
-
-      flushThread.done();
-      flushThread.join();
-      flushThread.checkNoError();
     } finally {
       try {
-        HBaseTestingUtility.closeRegionAndWAL(this.region);
+        flushThread.done();
+        flushThread.join();
+        flushThread.checkNoError();
+
+        putThread.join();
+        putThread.checkNoError();
+      } catch (InterruptedException ie) {
+        LOG.warn("Caught exception when joining with flushThread", ie);
+      }
+
+      try {
+          HBaseTestingUtility.closeRegionAndWAL(this.region);
       } catch (DroppedSnapshotException dse) {
         // We could get this on way out because we interrupt the background flusher and it could
         // fail anywhere causing a DSE over in the background flusher... only it is not properly


[16/18] hbase git commit: HBASE-14439 API cleanup

Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java
index 61c3eda..7530a79 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java
@@ -39,7 +39,8 @@ import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
 import org.apache.hadoop.hbase.regionserver.SplitTransactionImpl.LoggingProgressable;
 import org.apache.hadoop.hbase.security.User;
@@ -59,7 +60,7 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction {
   private final HRegion region_a;
   private final HRegion region_b;
   // merges dir is under region_a
-  private final Path mergesdir;
+  private final StorageIdentifier mergesdir;
   // We only merge adjacent regions if forcible is false
   private final boolean forcible;
   private final long masterSystemTime;
@@ -148,7 +149,7 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction {
     }
     this.forcible = forcible;
     this.masterSystemTime = masterSystemTime;
-    this.mergesdir = region_a.getRegionFileSystem().getMergesDir();
+    this.mergesdir = region_a.getRegionStorage().getMergesContainer();
   }
 
   private void transition(RegionMergeTransactionPhase nextPhase) throws IOException {
@@ -383,7 +384,7 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction {
 
     transition(RegionMergeTransactionPhase.SET_MERGING);
 
-    this.region_a.getRegionFileSystem().createMergesDir();
+    this.region_a.getRegionStorage().createMergesContainer();
 
     transition(RegionMergeTransactionPhase.CREATED_MERGE_DIR);
 
@@ -557,7 +558,7 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction {
       Map<byte[], List<StoreFile>> hstoreFilesOfRegionB)
       throws IOException {
     // Create reference file(s) of region A in mergdir
-    RegionFileSystem fs_a = this.region_a.getRegionFileSystem();
+    RegionStorage fs_a = this.region_a.getRegionStorage();
     for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionA.entrySet()) {
       String familyName = Bytes.toString(entry.getKey());
       for (StoreFile storeFile : entry.getValue()) {
@@ -565,7 +566,7 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction {
       }
     }
     // Create reference file(s) of region B in mergedir
-    RegionFileSystem fs_b = this.region_b.getRegionFileSystem();
+    RegionStorage fs_b = this.region_b.getRegionStorage();
     for (Map.Entry<byte[], List<StoreFile>> entry : hstoreFilesOfRegionB.entrySet()) {
       String familyName = Bytes.toString(entry.getKey());
       for (StoreFile storeFile : entry.getValue()) {
@@ -616,7 +617,7 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction {
         case CREATED_MERGE_DIR:
           this.region_a.writestate.writesEnabled = true;
           this.region_b.writestate.writesEnabled = true;
-          this.region_a.getRegionFileSystem().cleanupMergesDir();
+          this.region_a.getRegionStorage().cleanupMergesContainer();
           break;
 
         case CLOSED_REGION_A:
@@ -655,7 +656,7 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction {
           break;
 
         case STARTED_MERGED_REGION_CREATION:
-          this.region_a.getRegionFileSystem().cleanupMergedRegion(
+          this.region_a.getRegionStorage().cleanupMergedRegion(
               this.mergedRegionInfo);
           break;
 
@@ -688,7 +689,7 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction {
   }
 
   @VisibleForTesting
-  Path getMergesDir() {
+  StorageIdentifier getMergesDir() {
     return this.mergesdir;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
index e20b3e2..b966d92 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.java
@@ -128,7 +128,7 @@ public abstract class RegionSplitPolicy extends Configured {
   }
 
   /**
-   * In {@link HRegionFileSystem#splitStoreFile(org.apache.hadoop.hbase.HRegionInfo, String,
+   * In {@link RegionStorage#splitStoreFile(org.apache.hadoop.hbase.HRegionInfo, String,
    * StoreFile, byte[], boolean, RegionSplitPolicy)} we are not creating the split reference
    * if split row not lies in the StoreFile range. But in some use cases we may need to create
    * the split reference even when the split row not lies in the range. This method can be used
@@ -137,7 +137,7 @@ public abstract class RegionSplitPolicy extends Configured {
    * @param familyName
    * @return whether to skip the StoreFile range check or not
    */
-  protected boolean skipStoreFileRangeCheck(String familyName) {
+  public boolean skipStoreFileRangeCheck(String familyName) {
     return false;
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
index e4aee69..d03365f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
@@ -43,7 +43,9 @@ import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -317,7 +319,7 @@ public class SplitTransactionImpl implements SplitTransaction {
 
     transition(SplitTransactionPhase.SET_SPLITTING);
 
-    this.parent.getRegionFileSystem().createSplitsDir();
+    this.parent.getRegionStorage().createSplitsContainer();
 
     transition(SplitTransactionPhase.CREATE_SPLIT_DIR);
 
@@ -364,25 +366,29 @@ public class SplitTransactionImpl implements SplitTransaction {
 
     transition(SplitTransactionPhase.STARTED_REGION_A_CREATION);
 
+    // TODO with referenceFileCount in RegionStorage this should clean up
     assertReferenceFileCount(expectedReferences.getFirst(),
-        this.parent.getRegionFileSystem().getSplitsDir(this.hri_a));
+        ((LegacyPathIdentifier)this.parent.getRegionStorage().getSplitsContainer(this.hri_a)).path);
     HRegion a = this.parent.createDaughterRegionFromSplits(this.hri_a);
     assertReferenceFileCount(expectedReferences.getFirst(),
-        new Path(this.parent.getRegionFileSystem().getTableDir(), this.hri_a.getEncodedName()));
+        new Path(((LegacyPathIdentifier)this.parent.getRegionStorage().getTableContainer()).path,
+            this.hri_a.getEncodedName()));
 
     // Ditto
 
     transition(SplitTransactionPhase.STARTED_REGION_B_CREATION);
 
     assertReferenceFileCount(expectedReferences.getSecond(),
-        this.parent.getRegionFileSystem().getSplitsDir(this.hri_b));
+        ((LegacyPathIdentifier)this.parent.getRegionStorage().getSplitsContainer(this.hri_b)).path);
     HRegion b = this.parent.createDaughterRegionFromSplits(this.hri_b);
     assertReferenceFileCount(expectedReferences.getSecond(),
-        new Path(this.parent.getRegionFileSystem().getTableDir(), this.hri_b.getEncodedName()));
+        new Path(((LegacyPathIdentifier)this.parent.getRegionStorage().getTableContainer()).path,
+            this.hri_b.getEncodedName()));
 
     return new PairOfSameType<Region>(a, b);
   }
 
+  // TODO file count should be in RegionStorage
   @VisibleForTesting
   void assertReferenceFileCount(int expectedReferenceFileCount, Path dir)
       throws IOException {
@@ -606,7 +612,7 @@ public class SplitTransactionImpl implements SplitTransaction {
     ThreadFactory factory = builder.build();
     ThreadPoolExecutor threadPool =
       (ThreadPoolExecutor) Executors.newFixedThreadPool(maxThreads, factory);
-    List<Future<Pair<Path,Path>>> futures = new ArrayList<Future<Pair<Path,Path>>> (nbFiles);
+    List<Future<Pair<StorageIdentifier,StorageIdentifier>>> futures = new ArrayList<> (nbFiles);
 
     // Split each store file.
     for (Map.Entry<byte[], List<StoreFile>> entry: hstoreFilesToSplit.entrySet()) {
@@ -638,9 +644,9 @@ public class SplitTransactionImpl implements SplitTransaction {
     int created_a = 0;
     int created_b = 0;
     // Look for any exception
-    for (Future<Pair<Path, Path>> future : futures) {
+    for (Future<Pair<StorageIdentifier, StorageIdentifier>> future : futures) {
       try {
-        Pair<Path, Path> p = future.get();
+        Pair<StorageIdentifier, StorageIdentifier> p = future.get();
         created_a += p.getFirst() != null ? 1 : 0;
         created_b += p.getSecond() != null ? 1 : 0;
       } catch (InterruptedException e) {
@@ -657,32 +663,32 @@ public class SplitTransactionImpl implements SplitTransaction {
     return new Pair<Integer, Integer>(created_a, created_b);
   }
 
-  private Pair<Path, Path> splitStoreFile(final byte[] family, final StoreFile sf)
+  private Pair<StorageIdentifier, StorageIdentifier> splitStoreFile(final byte[] family, final StoreFile sf)
       throws IOException {
     if (LOG.isDebugEnabled()) {
         LOG.debug("Splitting started for store file: " + sf.getPath() + " for region: " +
                   this.parent);
     }
-    RegionFileSystem fs = this.parent.getRegionFileSystem();
+    RegionStorage fs = this.parent.getRegionStorage();
     String familyName = Bytes.toString(family);
-    Path path_a =
+    StorageIdentifier path_a =
         fs.splitStoreFile(this.hri_a, familyName, sf, this.splitrow, false,
           this.parent.getSplitPolicy());
-    Path path_b =
+    StorageIdentifier path_b =
         fs.splitStoreFile(this.hri_b, familyName, sf, this.splitrow, true,
           this.parent.getSplitPolicy());
     if (LOG.isDebugEnabled()) {
         LOG.debug("Splitting complete for store file: " + sf.getPath() + " for region: " +
                   this.parent);
     }
-    return new Pair<Path,Path>(path_a, path_b);
+    return new Pair<StorageIdentifier, StorageIdentifier>(path_a, path_b);
   }
 
   /**
    * Utility class used to do the file splitting / reference writing
    * in parallel instead of sequentially.
    */
-  private class StoreFileSplitter implements Callable<Pair<Path,Path>> {
+  private class StoreFileSplitter implements Callable<Pair<StorageIdentifier,StorageIdentifier>> {
     private final byte[] family;
     private final StoreFile sf;
 
@@ -696,7 +702,7 @@ public class SplitTransactionImpl implements SplitTransaction {
       this.family = family;
     }
 
-    public Pair<Path,Path> call() throws IOException {
+    public Pair<StorageIdentifier,StorageIdentifier> call() throws IOException {
       return splitStoreFile(family, sf);
     }
   }
@@ -741,7 +747,7 @@ public class SplitTransactionImpl implements SplitTransaction {
 
       case CREATE_SPLIT_DIR:
         this.parent.writestate.writesEnabled = true;
-        this.parent.getRegionFileSystem().cleanupSplitsDir();
+        this.parent.getRegionStorage().cleanupSplitsContainer();
         break;
 
       case CLOSED_PARENT_REGION:
@@ -760,11 +766,11 @@ public class SplitTransactionImpl implements SplitTransaction {
         break;
 
       case STARTED_REGION_A_CREATION:
-        this.parent.getRegionFileSystem().cleanupDaughterRegion(this.hri_a);
+        this.parent.getRegionStorage().cleanupDaughterRegion(this.hri_a);
         break;
 
       case STARTED_REGION_B_CREATION:
-        this.parent.getRegionFileSystem().cleanupDaughterRegion(this.hri_b);
+        this.parent.getRegionStorage().cleanupDaughterRegion(this.hri_b);
         break;
 
       case OFFLINED_PARENT:

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 853a4cf..62f1d58 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -157,7 +157,7 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
 
   FileSystem getFileSystem();
 
-
+  // TODO move createWriterInTmp calls to RegionStorage as somethign like "createTempStoreFileWriter"
   /**
    * @param maxKeyCount
    * @param compression Compression algorithm to use
@@ -298,7 +298,7 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
    * @param srcPathStr
    * @param sequenceId sequence Id associated with the HFile
    */
-  Path bulkLoadHFile(String srcPathStr, long sequenceId) throws IOException;
+  StoreFile bulkLoadHFile(String srcPathStr, long sequenceId) throws IOException;
 
   // General accessors into the state of the store
   // TODO abstract some of this out into a metrics class

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 589d844..071498b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -255,6 +255,7 @@ public class StoreFile {
     return this.fileInfo;
   }
 
+  // TODO replace with StorageIdentifier
   /**
    * @return Path or null if this StoreFile was made with a Stream.
    */
@@ -612,6 +613,7 @@ public class StoreFile {
     return false;
   }
 
+  // TODO move this into RegionStorage
   /**
    * @param fs
    * @param dir Directory to create file in.

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
index cb5d12c..17abd34 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
@@ -120,6 +120,7 @@ public class StoreFileWriter implements Compactor.CellSink {
     // it no longer writable.
     this.timeRangeTrackerSet = trt != null;
     this.timeRangeTracker = this.timeRangeTrackerSet? trt: new TimeRangeTracker();
+    // TODO move creation of the writer for the StoreFileWriter into RegionStorage
     writer = HFile.getWriterFactory(conf, cacheConf)
         .withPath(fs, path)
         .withComparator(comparator)
@@ -372,6 +373,7 @@ public class StoreFileWriter implements Compactor.CellSink {
       return this;
     }
 
+    // TODO replace with RegionStorage
     /**
      * Use either this method or {@link #withFilePath}, but not both.
      * @param dir Path to column family directory. The directory is created if
@@ -385,6 +387,7 @@ public class StoreFileWriter implements Compactor.CellSink {
       return this;
     }
 
+    // TODO replace with RegionStorage
     /**
      * Use either this method or {@link #withOutputDir}, but not both.
      * @param filePath the StoreFile path to write

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
index 4cde73d..81baa82 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -47,7 +47,8 @@ import org.apache.hadoop.hbase.backup.HFileArchiver;
 import org.apache.hadoop.hbase.MetaTableAccessor;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.mob.MobUtils;
@@ -103,6 +104,8 @@ import org.apache.hadoop.io.IOUtils;
  *    </ul>
  *  <li>restore the logs, if any
  * </ul>
+ *
+ * TODO update for MasterStorage / RegionStorage
  */
 @InterfaceAudience.Private
 public class RestoreSnapshotHelper {
@@ -780,8 +783,8 @@ public class RestoreSnapshotHelper {
 
     List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
     for (FileStatus regionDir: regionDirs) {
-      HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath());
-      regions.add(hri);
+      final RegionStorage rs = RegionStorage.open(conf, new LegacyPathIdentifier(regionDir.getPath()), false);
+      regions.add(rs.getRegionInfo());
     }
     LOG.debug("found " + regions.size() + " regions for table=" +
         tableDesc.getTableName().getNameAsString());

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
index dfb706b..3fa715a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
 import org.apache.hadoop.hbase.mob.MobUtils;
@@ -250,8 +250,7 @@ public final class SnapshotManifest {
     boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
     try {
       // Open the RegionFS
-      Path rootDir = null;
-      RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, rootDir, regionInfo, false);
+      RegionStorage regionFs = RegionStorage.open(conf, regionInfo, false);
       monitor.rethrowException();
 
       // 1. dump region meta info into the snapshot directory

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
index 2c5d76a..3ca48fe 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
@@ -36,7 +36,9 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
@@ -62,35 +64,36 @@ public final class SnapshotManifestV1 {
   private SnapshotManifestV1() {
   }
 
-  static class ManifestBuilder implements SnapshotManifest.RegionVisitor<RegionFileSystem, Path> {
+  // TODO update for RegionStorage
+  static class ManifestBuilder implements SnapshotManifest.RegionVisitor<RegionStorage, Path> {
     private final Configuration conf;
-    private final Path snapshotDir;
+    private final StorageIdentifier snapshotDir;
     private final FileSystem fs;
 
     public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
-      this.snapshotDir = snapshotDir;
+      this.snapshotDir = new LegacyPathIdentifier(snapshotDir);
       this.conf = conf;
       this.fs = fs;
     }
 
-    public RegionFileSystem regionOpen(final HRegionInfo regionInfo) throws IOException {
-      RegionFileSystem snapshotRegionFs = RegionFileSystem.open(conf, fs,
+    public RegionStorage regionOpen(final HRegionInfo regionInfo) throws IOException {
+      RegionStorage snapshotRegionFs = RegionStorage.open(conf, fs,
           snapshotDir, regionInfo, true);
       return snapshotRegionFs;
     }
 
-    public void regionClose(final RegionFileSystem region) {
+    public void regionClose(final RegionStorage region) {
     }
 
-    public Path familyOpen(final RegionFileSystem snapshotRegionFs, final byte[] familyName) {
-      Path familyDir = snapshotRegionFs.getStoreDir(Bytes.toString(familyName));
+    public Path familyOpen(final RegionStorage snapshotRegionFs, final byte[] familyName) {
+      Path familyDir = ((LegacyPathIdentifier)snapshotRegionFs.getStoreContainer(Bytes.toString(familyName))).path;
       return familyDir;
     }
 
-    public void familyClose(final RegionFileSystem region, final Path family) {
+    public void familyClose(final RegionStorage region, final Path family) {
     }
 
-    public void storeFile(final RegionFileSystem region, final Path familyDir,
+    public void storeFile(final RegionStorage region, final Path familyDir,
         final StoreFileInfo storeFile) throws IOException {
       Path referenceFile = new Path(familyDir, storeFile.getPath().getName());
       boolean success = true;
@@ -125,8 +128,8 @@ public final class SnapshotManifestV1 {
       completionService.submit(new Callable<SnapshotRegionManifest>() {
         @Override
         public SnapshotRegionManifest call() throws IOException {
-          HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, region.getPath());
-          return buildManifestFromDisk(conf, fs, snapshotDir, hri);
+          final RegionStorage rs = RegionStorage.open(conf, new LegacyPathIdentifier(region.getPath()), true);
+          return buildManifestFromDisk(conf, fs, snapshotDir, rs);
         }
       });
     }
@@ -154,13 +157,12 @@ public final class SnapshotManifestV1 {
   }
 
   static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf,
-      final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
-    RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, tableDir, regionInfo, true);
+      final FileSystem fs, final Path tableDir, final RegionStorage regionFs) throws IOException {
     SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
 
     // 1. dump region meta info into the snapshot directory
     LOG.debug("Storing region-info for snapshot.");
-    manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
+    manifest.setRegionInfo(HRegionInfo.convert(regionFs.getRegionInfo()));
 
     // 2. iterate through all the stores in the region
     LOG.debug("Creating references for hfiles");

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
index d38d5c1..89124ae 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
@@ -29,8 +29,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
-import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FileStatus;
@@ -47,7 +47,7 @@ import org.apache.hadoop.hbase.util.FSUtils;
 class FSRegionScanner implements Runnable {
   static private final Log LOG = LogFactory.getLog(FSRegionScanner.class);
 
-  private final RegionFileSystem rfs;
+  private final RegionStorage rfs;
 
   /**
    * Maps region encoded names to maps of hostnames to fractional locality of
@@ -58,7 +58,7 @@ class FSRegionScanner implements Runnable {
   FSRegionScanner(Configuration conf, HRegionInfo hri,
                   Map<String, Map<String, Float>> regionDegreeLocalityMapping)
       throws IOException {
-    this.rfs = RegionFileSystem.open(conf, hri, true);
+    this.rfs = RegionStorage.open(conf, hri, true);
     this.regionDegreeLocalityMapping = regionDegreeLocalityMapping;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 80fbb76..662f20d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -74,8 +74,8 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.fs.MasterFileSystem;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
-import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
@@ -1093,8 +1093,9 @@ public abstract class FSUtils {
       int cfCount = 0;
       int cfFrag = 0;
       for (HRegionInfo hri: mfs.getRegions(table)) {
-        RegionFileSystem rfs = mfs.getRegionFileSystem(hri);
-        for (String family: rfs.getFamilies()) {
+        RegionStorage rfs = mfs.getRegionStorage(hri);
+        final Collection<String> families = rfs.getFamilies();
+        for (String family: families) {
           cfCount++;
           cfCountTotal++;
           if (rfs.getStoreFiles(family).size() > 1) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 2f34291..8a14cca 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -111,9 +111,12 @@ import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableState;
 import org.apache.hadoop.hbase.fs.MasterFileSystem;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
-import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
 import org.apache.hadoop.hbase.fs.legacy.LegacyLayout;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyRegionStorage;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -1022,11 +1025,15 @@ public class HBaseFsck extends Configured implements Closeable {
         Bytes.add(orphanRegionRange.getSecond(), new byte[1]));
     LOG.info("Creating new region : " + hri);
     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
-    Path target = region.getRegionFileSystem().getRegionDir();
+    final StorageIdentifier target = region.getRegionStorage().getRegionContainer();
 
-    // rename all the data to new region
-    mergeRegionDirs(target, hi);
-    fixes++;
+    if (target instanceof LegacyPathIdentifier) {
+      // rename all the data to new region
+      mergeRegionDirs(((LegacyPathIdentifier)target).path, hi);
+      fixes++;
+    } else {
+      LOG.info("Skipped ");
+    }
   }
 
   /**
@@ -1191,7 +1198,7 @@ public class HBaseFsck extends Configured implements Closeable {
     }
 
     FileSystem fs = FileSystem.get(getConf());
-    HRegionInfo hri = RegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
+    HRegionInfo hri = LegacyRegionStorage.loadRegionInfoFileContent(fs, regionDir);
     LOG.debug("HRegionInfo read: " + hri.toString());
     hbi.hdfsEntry.hri = hri;
   }
@@ -1424,7 +1431,7 @@ public class HBaseFsck extends Configured implements Closeable {
         Collections.<WALActionsListener>singletonList(new MetricsWAL()),
         "hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8))).
         getWAL(metaHRI.getEncodedNameAsBytes(), metaHRI.getTable().getNamespace());
-    HRegion meta = HRegion.createHRegion(c, rootdir, metaDescriptor, metaHRI, wal);
+    HRegion meta = HRegion.createHRegion(c, metaDescriptor, metaHRI, wal);
     MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, true);
     return meta;
   }
@@ -2361,7 +2368,8 @@ public class HBaseFsck extends Configured implements Closeable {
                   LOG.warn(hri + " start and stop keys are in the range of " + region
                       + ". The region might not be cleaned up from hdfs when region " + region
                       + " split failed. Hence deleting from hdfs.");
-                  RegionFileSystem.destroy(getConf(), fs, regionDir.getParent(), hri);
+                  // TODO directly reference LegacyRegionStorage?
+                  RegionStorage.destroy(getConf(), fs, new LegacyPathIdentifier(regionDir.getParent()), hri);
                   return;
                 }
               }
@@ -3004,17 +3012,21 @@ public class HBaseFsck extends Configured implements Closeable {
         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
         LOG.info("[" + thread + "] Created new empty container region: " +
             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
-        debugLsr(region.getRegionFileSystem().getRegionDir());
+        debugLsr(region.getRegionStorage().getRegionContainer());
 
         // all target regions are closed, should be able to safely cleanup.
         boolean didFix= false;
-        Path target = region.getRegionFileSystem().getRegionDir();
-        for (HbckInfo contained : overlap) {
-          LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
-          int merges = mergeRegionDirs(target, contained);
-          if (merges > 0) {
-            didFix = true;
+        StorageIdentifier target = region.getRegionStorage().getRegionContainer();
+        if (target instanceof LegacyPathIdentifier) {
+          for (HbckInfo contained : overlap) {
+            LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
+            int merges = mergeRegionDirs(((LegacyPathIdentifier)target).path, contained);
+            if (merges > 0) {
+              didFix = true;
+            }
           }
+        } else {
+          LOG.info("skipping merge into " + target + " because it is not a Path.");
         }
         if (didFix) {
           fixes++;
@@ -4146,7 +4158,7 @@ public class HBaseFsck extends Configured implements Closeable {
               try {
                 LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
 
-                Path regioninfoFile = new Path(he.hdfsRegionDir, LegacyLayout.REGION_INFO_FILE);
+                Path regioninfoFile = new Path(regionDir.getPath(), LegacyLayout.REGION_INFO_FILE);
                 boolean regioninfoFileExists = fs.exists(regioninfoFile);
 
                 if (!regioninfoFileExists) {
@@ -4866,6 +4878,14 @@ public class HBaseFsck extends Configured implements Closeable {
     debugLsr(conf, p, new PrintingErrorReporter());
   }
 
+  void debugLsr(StorageIdentifier id) throws IOException {
+    if (id instanceof LegacyPathIdentifier) {
+      debugLsr(((LegacyPathIdentifier)id).path);
+    } else {
+      LOG.debug("identifier '" + id + "' is not a Path; skipping long output.");
+    }
+  }
+
   /**
    * ls -r for debugging purposes
    */

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
index a07fed9..cf5c27c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
@@ -190,8 +190,7 @@ public class HBaseFsckRepair {
   public static HRegion createHDFSRegionDir(Configuration conf,
       HRegionInfo hri, HTableDescriptor htd) throws IOException {
     // Create HRegion
-    Path root = FSUtils.getRootDir(conf);
-    HRegion region = HRegion.createHRegion(conf, root, htd, hri, null);
+    HRegion region = HRegion.createHRegion(conf, htd, hri, null);
 
     // Close the new region to flush to disk. Close log file too.
     region.close();

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
index 1959c73..e4d830e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
@@ -144,7 +144,7 @@ class HMerge {
     protected final WALFactory walFactory;
     private final long maxFilesize;
 
-
+    // TODO update for RegionStorage
     protected Merger(Configuration conf, FileSystem fs, final TableName tableName)
     throws IOException {
       this.conf = conf;
@@ -192,14 +192,14 @@ class HMerge {
       long nextSize = 0;
       for (int i = 0; i < info.length - 1; i++) {
         if (currentRegion == null) {
-          currentRegion = HRegion.openHRegion(conf, fs, this.rootDir, info[i], this.htd,
+          currentRegion = HRegion.openHRegion(info[i], this.htd,
               walFactory.getWAL(info[i].getEncodedNameAsBytes(),
-                info[i].getTable().getNamespace()));
+                info[i].getTable().getNamespace()), conf);
           currentSize = currentRegion.getLargestHStoreSize();
         }
-        nextRegion = HRegion.openHRegion(conf, fs, this.rootDir, info[i + 1], this.htd,
+        nextRegion = HRegion.openHRegion(info[i + 1], this.htd,
             walFactory.getWAL(info[i + 1].getEncodedNameAsBytes(),
-              info[i + 1].getTable().getNamespace()));
+              info[i + 1].getTable().getNamespace()), conf);
         nextSize = nextRegion.getLargestHStoreSize();
 
         if ((currentSize + nextSize) <= (maxFilesize / 2)) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
index 4fe39ff..b670a26 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
@@ -170,7 +170,7 @@ public abstract class ModifyRegionUtils {
     // unless I pass along via the conf.
     Configuration confForWAL = new Configuration(conf);
     confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
-    HRegion region = HRegion.createHRegion(conf, rootDir, hTableDescriptor, newRegion, null, false);
+    HRegion region = HRegion.createHRegion(conf, rootDir, hTableDescriptor, newRegion);
     try {
       // 2. Custom user code to interact with the created region
       if (task != null) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
index cbc57dc..3793cc7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
@@ -61,7 +61,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.NoServerForRegionException;
 import org.apache.hadoop.hbase.client.RegionLocator;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
@@ -727,8 +727,7 @@ public class RegionSplitter {
             if (sk.length == 0)
               sk = splitAlgo.firstRow();
 
-            RegionFileSystem regionFs = RegionFileSystem.open(
-              connection.getConfiguration(), fs, tableDir, hri, true);
+            RegionStorage regionFs = RegionStorage.open(connection.getConfiguration(), hri, true);
 
             // Check every Column Family for that region -- check does not have references.
             boolean refFound = false;

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
index 6c2c882..370f03b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
@@ -30,8 +30,8 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.protobuf.ServerProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -94,7 +94,7 @@ public class TestIOFencing {
     volatile CountDownLatch compactionsBlocked = new CountDownLatch(0);
     volatile CountDownLatch compactionsWaiting = new CountDownLatch(0);
 
-    public CompactionBlockerRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+    public CompactionBlockerRegion(final RegionStorage rfs, final HTableDescriptor htd,
         final WAL wal, final RegionServerServices rsServices) {
       super(rfs, htd, wal, rsServices);
     }
@@ -153,7 +153,7 @@ public class TestIOFencing {
    */
   public static class BlockCompactionsInPrepRegion extends CompactionBlockerRegion {
 
-    public BlockCompactionsInPrepRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+    public BlockCompactionsInPrepRegion(final RegionStorage rfs, final HTableDescriptor htd,
         final WAL wal, final RegionServerServices rsServices) {
       super(rfs, htd, wal, rsServices);
     }
@@ -176,7 +176,7 @@ public class TestIOFencing {
    * entry to go the WAL before blocking, but blocks afterwards
    */
   public static class BlockCompactionsInCompletionRegion extends CompactionBlockerRegion {
-    public BlockCompactionsInCompletionRegion(final RegionFileSystem rfs,
+    public BlockCompactionsInCompletionRegion(final RegionStorage rfs,
         final HTableDescriptor htd, final WAL wal, final RegionServerServices rsServices) {
       super(rfs, htd, wal, rsServices);
     }
@@ -281,7 +281,7 @@ public class TestIOFencing {
       // those entries
       HRegionInfo oldHri = new HRegionInfo(table.getName(),
         HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
-      CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(oldHri,
+      CompactionDescriptor compactionDescriptor = ServerProtobufUtil.toCompactionDescriptor(oldHri,
         FAMILY, Lists.newArrayList(new Path("/a")), Lists.newArrayList(new Path("/b")),
         new Path("store_dir"));
       WALUtil.writeCompactionMarker(compactingRegion.getWAL(),

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
index e30d719..a371000 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
@@ -136,7 +136,7 @@ public class TestHFileArchiving {
     FileSystem fs = UTIL.getTestFileSystem();
 
     // now attempt to depose the region
-    Path rootDir = region.getRegionFileSystem().getTableDir().getParent();
+    Path rootDir = region.getRegionStorage().getTableDir().getParent();
     Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
 
     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
@@ -185,7 +185,7 @@ public class TestHFileArchiving {
     assertEquals(1, servingRegions.size());
     HRegion region = servingRegions.get(0);
 
-    FileSystem fs = region.getRegionFileSystem().getFileSystem();
+    FileSystem fs = region.getRegionStorage().getFileSystem();
 
     // make sure there are some files in the regiondir
     Path rootDir = FSUtils.getRootDir(fs.getConf());

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index a5b0002..f5f380f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.fs.MasterFileSystem;
-import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
index fa4497f..bcc15cf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
@@ -47,7 +47,7 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.filter.FilterBase;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -221,7 +221,7 @@ public class TestRegionObserverScannerOpenHook {
   public static class CompactionCompletionNotifyingRegion extends HRegion {
     private static volatile CountDownLatch compactionStateChangeLatch = null;
 
-    public CompactionCompletionNotifyingRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+    public CompactionCompletionNotifyingRegion(final RegionStorage rfs, final HTableDescriptor htd,
         final WAL wal, final RegionServerServices rsServices) {
       super(rfs, htd, wal, rsServices);
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
index 8aa4909..70c879a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index a6fe4ad..a6a882d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.TableStateManager;
 import org.apache.hadoop.hbase.procedure2.Procedure;

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
index 38e0a61..acd1d72 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
@@ -59,7 +59,7 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.io.HeapSize;
@@ -661,7 +661,7 @@ public class TestAtomicOperation {
 
   public static class MockHRegion extends HRegion {
 
-    public MockHRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+    public MockHRegion(final RegionStorage rfs, final HTableDescriptor htd,
         final WAL wal, final RegionServerServices rsServices) {
       super(rfs, htd, wal, rsServices);
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index fa630a2..86c3968 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -174,7 +174,7 @@ public class TestCompaction {
       assertEquals(compactionThreshold, s.getStorefilesCount());
       assertTrue(s.getStorefilesSize() > 15*1000);
       // and no new store files persisted past compactStores()
-      FileStatus[] ls = r.getFilesystem().listStatus(r.getRegionFileSystem().getTempDir());
+      FileStatus[] ls = r.getFilesystem().listStatus(r.getRegionStorage().getTempDir());
       assertEquals(0, ls.length);
 
     } finally {
@@ -249,11 +249,11 @@ public class TestCompaction {
     // Now lets corrupt the compacted file.
     FileSystem fs = store.getFileSystem();
     // default compaction policy created one and only one new compacted file
-    Path dstPath = store.getRegionFileSystem().createTempName();
+    Path dstPath = store.getRegionStorage().createTempName();
     FSDataOutputStream stream = fs.create(dstPath, null, true, 512, (short)3, (long)1024, null);
     stream.writeChars("CORRUPT FILE!!!!");
     stream.close();
-    Path origPath = store.getRegionFileSystem().commitStoreFile(
+    Path origPath = store.getRegionStorage().commitStoreFile(
       Bytes.toString(COLUMN_FAMILY), dstPath);
 
     try {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
index 24b3667..4e39664 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
@@ -111,7 +111,7 @@ public class TestCompactionPolicy {
 
     store = new HStore(region, hcd, conf);
 
-    TEST_FILE = region.getRegionFileSystem().createTempName();
+    TEST_FILE = region.getRegionStorage().createTempName();
     fs.createNewFile(TEST_FILE);
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
index 1325c85..140b038 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.fs.FsContext;
-import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.testclassification.MasterTests;

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
index ef332a4..95e19f2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
@@ -91,13 +91,13 @@ public class TestDefaultCompactSelection extends TestCase {
     region = HBaseTestingUtility.createRegionAndWAL(info, basedir, conf, htd);
     HBaseTestingUtility.closeRegionAndWAL(region);
 
-    RegionFileSystem rfs = RegionFileSystem.open(conf, fs, basedir, info, false);
+    RegionStorage rfs = RegionStorage.open(conf, fs, basedir, info, false);
     region = new HRegion(rfs, htd,
       wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), null);
 
     store = new HStore(region, hcd, conf);
 
-    TEST_FILE = region.getRegionFileSystem().createTempName();
+    TEST_FILE = region.getRegionStorage().createTempName();
     fs.createNewFile(TEST_FILE);
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
index a324328..73e10f7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.hbase.ArrayBackedTag;
 import org.apache.hadoop.hbase.TagType;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
 import org.apache.hadoop.hbase.io.crypto.aes.AES;
 import org.apache.hadoop.hbase.io.hfile.HFile;
@@ -159,7 +159,7 @@ public class TestHMobStore {
     FSUtils.setRootDir(walConf, basedir);
     final WALFactory wals = new WALFactory(walConf, null, methodName);
 
-    RegionFileSystem rfs = RegionFileSystem.open(conf, fs, basedir, info, false);
+    RegionStorage rfs = RegionStorage.open(conf, fs, basedir, info, false);
     region = new HRegion(rfs, htd,
       wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()), null);
     store = new HMobStore(region, hcd, conf);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index df0b119..08f0470 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -90,7 +90,7 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
@@ -108,6 +108,7 @@ import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.ServerProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
@@ -683,8 +684,8 @@ public class TestHRegion {
     this.region = initHRegion(tableName, method, CONF, family);
     final WALFactory wals = new WALFactory(CONF, null, method);
     try {
-      Path regiondir = region.getRegionFileSystem().getRegionDir();
-      FileSystem fs = region.getRegionFileSystem().getFileSystem();
+      Path regiondir = region.getRegionStorage().getRegionDir();
+      FileSystem fs = region.getRegionStorage().getFileSystem();
       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
 
       Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
@@ -736,8 +737,8 @@ public class TestHRegion {
     this.region = initHRegion(tableName, method, CONF, family);
     final WALFactory wals = new WALFactory(CONF, null, method);
     try {
-      Path regiondir = region.getRegionFileSystem().getRegionDir();
-      FileSystem fs = region.getRegionFileSystem().getFileSystem();
+      Path regiondir = region.getRegionStorage().getRegionDir();
+      FileSystem fs = region.getRegionStorage().getFileSystem();
       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
 
       Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
@@ -791,8 +792,8 @@ public class TestHRegion {
     byte[] family = Bytes.toBytes("family");
     this.region = initHRegion(tableName, method, CONF, family);
     try {
-      Path regiondir = region.getRegionFileSystem().getRegionDir();
-      FileSystem fs = region.getRegionFileSystem().getFileSystem();
+      Path regiondir = region.getRegionStorage().getRegionDir();
+      FileSystem fs = region.getRegionStorage().getFileSystem();
 
       Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
       for (int i = 1000; i < 1050; i += 10) {
@@ -826,8 +827,8 @@ public class TestHRegion {
     this.region = initHRegion(tableName, method, CONF, family);
     final WALFactory wals = new WALFactory(CONF, null, method);
     try {
-      Path regiondir = region.getRegionFileSystem().getRegionDir();
-      FileSystem fs = region.getRegionFileSystem().getFileSystem();
+      Path regiondir = region.getRegionStorage().getRegionDir();
+      FileSystem fs = region.getRegionStorage().getFileSystem();
       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
       byte[][] columns = region.getTableDesc().getFamiliesKeys().toArray(new byte[0][]);
 
@@ -895,8 +896,8 @@ public class TestHRegion {
     this.region = initHRegion(tableName, method, CONF, family);
     final WALFactory wals = new WALFactory(CONF, null, method);
     try {
-      Path regiondir = region.getRegionFileSystem().getRegionDir();
-      FileSystem fs = region.getRegionFileSystem().getFileSystem();
+      Path regiondir = region.getRegionStorage().getRegionDir();
+      FileSystem fs = region.getRegionStorage().getFileSystem();
       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
 
       long maxSeqId = 3;
@@ -924,14 +925,14 @@ public class TestHRegion {
       assertEquals(3, region.getStore(family).getStorefilesCount());
 
       // now find the compacted file, and manually add it to the recovered edits
-      Path tmpDir = region.getRegionFileSystem().getTempDir();
+      Path tmpDir = region.getRegionStorage().getTempDir();
       FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
       String errorMsg = "Expected to find 1 file in the region temp directory "
           + "from the compaction, could not find any";
       assertNotNull(errorMsg, files);
       assertEquals(errorMsg, 1, files.length);
       // move the file inside region dir
-      Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
+      Path newFile = region.getRegionStorage().commitStoreFile(Bytes.toString(family),
           files[0].getPath());
 
       byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes();
@@ -941,10 +942,10 @@ public class TestHRegion {
         fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1);
       }
 
-      CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
+      CompactionDescriptor compactionDescriptor = ServerProtobufUtil.toCompactionDescriptor(this.region
         .getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family,
             storeFiles, Lists.newArrayList(newFile),
-            region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
+            region.getRegionStorage().getStoreDir(Bytes.toString(family)));
 
       WALUtil.writeCompactionMarker(region.getWAL(), this.region.getReplicationScope(),
           this.region.getRegionInfo(), compactionDescriptor, region.getMVCC());
@@ -1011,8 +1012,8 @@ public class TestHRegion {
     this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW,
       HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
     try {
-      Path regiondir = region.getRegionFileSystem().getRegionDir();
-      FileSystem fs = region.getRegionFileSystem().getFileSystem();
+      Path regiondir = region.getRegionStorage().getRegionDir();
+      FileSystem fs = region.getRegionStorage().getFileSystem();
       byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
 
       long maxSeqId = 3;
@@ -2638,9 +2639,9 @@ public class TestHRegion {
           HRegion.openHRegion(subregions[i], null);
           subregions[i].compactStores();
         }
-        Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
-        Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
-        Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
+        Path oldRegionPath = region.getRegionStorage().getRegionDir();
+        Path oldRegion1 = subregions[0].getRegionStorage().getRegionDir();
+        Path oldRegion2 = subregions[1].getRegionStorage().getRegionDir();
         long startTime = System.currentTimeMillis();
         region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
         LOG.info("Merge regions elapsed time: "
@@ -4507,8 +4508,8 @@ public class TestHRegion {
 
     // Create a region and skip the initialization (like CreateTableHandler)
     HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, rootDir, CONF, htd, false);
-    Path regionDir = region.getRegionFileSystem().getRegionDir();
-    FileSystem fs = region.getRegionFileSystem().getFileSystem();
+    Path regionDir = region.getRegionStorage().getRegionDir();
+    FileSystem fs = region.getRegionStorage().getFileSystem();
     HBaseTestingUtility.closeRegionAndWAL(region);
 
     Path regionInfoFile = LegacyLayout.getRegionInfoFile(regionDir);
@@ -4519,7 +4520,7 @@ public class TestHRegion {
 
     // Try to open the region
     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
-    assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
+    assertEquals(regionDir, region.getRegionStorage().getRegionDir());
     HBaseTestingUtility.closeRegionAndWAL(region);
 
     // Verify that the .regioninfo file is still there
@@ -4533,7 +4534,7 @@ public class TestHRegion {
 
     region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
 //    region = TEST_UTIL.openHRegion(hri, htd);
-    assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
+    assertEquals(regionDir, region.getRegionStorage().getRegionDir());
     HBaseTestingUtility.closeRegionAndWAL(region);
 
     // Verify that the .regioninfo file is still there
@@ -5039,8 +5040,8 @@ public class TestHRegion {
 
       // move the file of the primary region to the archive, simulating a compaction
       Collection<StoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles();
-      primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles);
-      Collection<StoreFileInfo> storeFileInfos = primaryRegion.getRegionFileSystem()
+      primaryRegion.getRegionStorage().removeStoreFiles(Bytes.toString(families[0]), storeFiles);
+      Collection<StoreFileInfo> storeFileInfos = primaryRegion.getRegionStorage()
           .getStoreFiles(families[0]);
       Assert.assertTrue(storeFileInfos == null || storeFileInfos.size() == 0);
 
@@ -6099,7 +6100,7 @@ public class TestHRegion {
 
   // Helper for test testOpenRegionWrittenToWALForLogReplay
   static class HRegionWithSeqId extends HRegion {
-    public HRegionWithSeqId(final RegionFileSystem rfs, final HTableDescriptor htd,
+    public HRegionWithSeqId(final RegionStorage rfs, final HTableDescriptor htd,
         final WAL wal, final RegionServerServices rsServices) {
       super(rfs, htd, wal, rsServices);
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
index 48609b9..808029c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.fs.FSUtilsWithRetries;
 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
@@ -50,9 +50,9 @@ import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 @Category({RegionServerTests.class, SmallTests.class})
-public class TestHRegionFileSystem {
+public class TestHRegionStorage {
   private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private static final Log LOG = LogFactory.getLog(TestHRegionFileSystem.class);
+  private static final Log LOG = LogFactory.getLog(TestHRegionStorage.class);
 
   @Test
   public void testOnDiskRegionCreation() throws IOException {
@@ -62,22 +62,22 @@ public class TestHRegionFileSystem {
 
     // Create a Region
     HRegionInfo hri = new HRegionInfo(TableName.valueOf("TestTable"));
-    RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, rootDir, hri, true);
+    RegionStorage regionFs = RegionStorage.open(conf, fs, rootDir, hri, true);
 
     // Verify if the region is on disk
     Path regionDir = regionFs.getRegionDir();
     assertTrue("The region folder should be created", fs.exists(regionDir));
 
     // Verify the .regioninfo
-    HRegionInfo hriVerify = RegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
+    HRegionInfo hriVerify = RegionStorage.open(conf, regionDir, false).getRegionInfo();
     assertEquals(hri, hriVerify);
 
     // Open the region
-    regionFs = RegionFileSystem.open(conf, fs, rootDir, hri, false);
+    regionFs = RegionStorage.open(conf, fs, rootDir, hri, false);
     assertEquals(regionDir, regionFs.getRegionDir());
 
     // Delete the region
-    RegionFileSystem.destroy(conf, fs, rootDir, hri);
+    RegionStorage.destroy(conf, fs, rootDir, hri);
     assertFalse("The region folder should be removed", fs.exists(regionDir));
 
     fs.delete(rootDir, true);
@@ -207,7 +207,7 @@ public class TestHRegionFileSystem {
     // Create a Region
     String familyName = "cf";
     HRegionInfo hri = new HRegionInfo(TableName.valueOf("TestTable"));
-    RegionFileSystem regionFs = RegionFileSystem.open(conf, fs, rootDir, hri, true);
+    RegionStorage regionFs = RegionStorage.open(conf, fs, rootDir, hri, true);
 
     // New region, no store files
     Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
index 7e0a6f6..a790116 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.testclassification.RegionServerTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.fs.legacy.LegacyLayout;
 import org.apache.hadoop.hbase.master.RegionState;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
@@ -79,15 +79,15 @@ public class TestHRegionInfo {
     long modtime2 = getModTime(r);
     assertEquals(modtime, modtime2);
     // Now load the file.
-    HRegionInfo deserializedHri = RegionFileSystem.loadRegionInfoFileContent(
-        r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir());
+    HRegionInfo deserializedHri = RegionStorage.open(r.getRegionStorage().getConfiguration(),
+        r.getRegionStorage.getRegionContainer(), false).getRegionInfo();
     assertTrue(hri.equals(deserializedHri));
     HBaseTestingUtility.closeRegionAndWAL(r);
   }
 
   long getModTime(final HRegion r) throws IOException {
-    FileStatus[] statuses = r.getRegionFileSystem().getFileSystem().listStatus(
-      LegacyLayout.getRegionInfoFile(r.getRegionFileSystem().getRegionDir()));
+    FileStatus[] statuses = r.getRegionStorage().getFileSystem().listStatus(
+      LegacyLayout.getRegionInfoFile(r.getRegionStorage().getRegionDir()));
     assertTrue(statuses != null && statuses.length == 1);
     return statuses[0].getModificationTime();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java
index 014aba7..8400883 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -101,7 +101,7 @@ public class TestRecoveredEdits {
     Path hbaseRootDir = TEST_UTIL.getDataTestDir();
     FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
     Path tableDir = FSUtils.getTableDir(hbaseRootDir, htd.getTableName());
-    RegionFileSystem hrfs = RegionFileSystem.open(TEST_UTIL.getConfiguration(), fs, hbaseRootDir, hri, false);
+    RegionStorage hrfs = RegionStorage.open(TEST_UTIL.getConfiguration(), fs, hbaseRootDir, hri, false);
     if (fs.exists(hrfs.getRegionDir())) {
       LOG.info("Region directory already exists. Deleting.");
       fs.delete(hrfs.getRegionDir(), true);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java
index 4630522..535d449 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -96,10 +97,10 @@ public class TestRegionMergeTransaction {
   public void teardown() throws IOException {
     for (HRegion region : new HRegion[] { region_a, region_b, region_c }) {
       if (region != null && !region.isClosed()) region.close();
-      if (this.fs.exists(region.getRegionFileSystem().getRegionDir())
-          && !this.fs.delete(region.getRegionFileSystem().getRegionDir(), true)) {
+      if (this.fs.exists(region.getRegionStorage().getRegionDir())
+          && !this.fs.delete(region.getRegionStorage().getRegionDir(), true)) {
         throw new IOException("Failed deleting of "
-            + region.getRegionFileSystem().getRegionDir());
+            + region.getRegionStorage().getRegionDir());
       }
     }
     if (this.wals != null) {
@@ -257,14 +258,16 @@ public class TestRegionMergeTransaction {
     Server mockServer = new HRegionServer(TEST_UTIL.getConfiguration(), cp);
     HRegion mergedRegion = (HRegion)mt.execute(mockServer, null);
     // Do some assertions about execution.
-    assertTrue(this.fs.exists(mt.getMergesDir()));
+    // TODO move tests to rely on RegionStorage
+    final Path mergesdir = ((LegacyPathIdentifier)mt.getMergesDir()).path;
+    assertTrue(this.fs.exists(mergesdir));
     // Assert region_a and region_b is closed.
     assertTrue(region_a.isClosed());
     assertTrue(region_b.isClosed());
 
     // Assert mergedir is empty -- because its content will have been moved out
     // to be under the merged region dirs.
-    assertEquals(0, this.fs.listStatus(mt.getMergesDir()).length);
+    assertEquals(0, this.fs.listStatus(mergesdir).length);
     // Check merged region have correct key span.
     assertTrue(Bytes.equals(this.region_a.getRegionInfo().getStartKey(),
         mergedRegion.getRegionInfo().getStartKey()));
@@ -376,7 +379,7 @@ public class TestRegionMergeTransaction {
     // Make sure that merged region is still in the filesystem, that
     // they have not been removed; this is supposed to be the case if we go
     // past point of no return.
-    Path tableDir = this.region_a.getRegionFileSystem().getRegionDir()
+    Path tableDir = this.region_a.getRegionStorage().getRegionDir()
         .getParent();
     Path mergedRegionDir = new Path(tableDir, mt.getMergedRegionInfo()
         .getEncodedName());

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
index 326bbef..ee3d7d6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
@@ -54,7 +54,7 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.exceptions.MergeRegionException;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.HMaster;
@@ -251,7 +251,7 @@ public class TestRegionMergeTransactionOnCluster {
       admin.compactRegion(mergedRegionInfo.getRegionName());
       // wait until merged region doesn't have reference file
       long timeout = System.currentTimeMillis() + waitTime;
-      RegionFileSystem hrfs = RegionFileSystem.open(
+      RegionStorage hrfs = RegionStorage.open(
           TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo, false);
       while (System.currentTimeMillis() < timeout) {
         for(HColumnDescriptor colFamily : columnFamilies) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
index 52e8724..14903ad 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.hbase.client.ScannerCallable;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
@@ -445,7 +445,7 @@ public class TestScannerHeartbeatMessages {
     private static volatile int columnFamilySleepTime = DEFAULT_CF_SLEEP_TIME;
     private static volatile boolean sleepBetweenColumnFamilies = false;
 
-    public HeartbeatHRegion(final RegionFileSystem rfs, final HTableDescriptor htd,
+    public HeartbeatHRegion(final RegionStorage rfs, final HTableDescriptor htd,
         final WAL wal, final RegionServerServices rsServices) {
       super(rfs, htd, wal, rsServices);
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
index 2ade27a..0ae0f86 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java
@@ -100,7 +100,7 @@ public class TestSplitTransaction {
 
   @After public void teardown() throws IOException {
     if (this.parent != null && !this.parent.isClosed()) this.parent.close();
-    Path regionDir = this.parent.getRegionFileSystem().getRegionDir();
+    Path regionDir = this.parent.getRegionStorage().getRegionDir();
     if (this.fs.exists(regionDir) && !this.fs.delete(regionDir, true)) {
       throw new IOException("Failed delete of " + regionDir);
     }
@@ -143,7 +143,7 @@ public class TestSplitTransaction {
     // Make sure that region a and region b are still in the filesystem, that
     // they have not been removed; this is supposed to be the case if we go
     // past point of no return.
-    Path tableDir =  this.parent.getRegionFileSystem().getTableDir();
+    Path tableDir =  this.parent.getRegionStorage().getTableDir();
     Path daughterADir = new Path(tableDir, spiedUponSt.getFirstDaughter().getEncodedName());
     Path daughterBDir = new Path(tableDir, spiedUponSt.getSecondDaughter().getEncodedName());
     assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterADir));
@@ -245,13 +245,13 @@ public class TestSplitTransaction {
     when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
     PairOfSameType<Region> daughters = st.execute(mockServer, null);
     // Do some assertions about execution.
-    assertTrue(this.fs.exists(this.parent.getRegionFileSystem().getSplitsDir()));
+    assertTrue(this.fs.exists(this.parent.getRegionStorage().getSplitsDir()));
     // Assert the parent region is closed.
     assertTrue(this.parent.isClosed());
 
     // Assert splitdir is empty -- because its content will have been moved out
     // to be under the daughter region dirs.
-    assertEquals(0, this.fs.listStatus(this.parent.getRegionFileSystem().getSplitsDir()).length);
+    assertEquals(0, this.fs.listStatus(this.parent.getRegionStorage().getSplitsDir()).length);
     // Check daughters have correct key span.
     assertTrue(Bytes.equals(parent.getRegionInfo().getStartKey(),
       daughters.getFirst().getRegionInfo().getStartKey()));
@@ -288,7 +288,7 @@ public class TestSplitTransaction {
     SplitTransactionImpl spiedUponSt = spy(st);
     doThrow(new IOException("Failing split. Expected reference file count isn't equal."))
         .when(spiedUponSt).assertReferenceFileCount(anyInt(),
-        eq(new Path(this.parent.getRegionFileSystem().getTableDir(),
+        eq(new Path(this.parent.getRegionStorage().getTableDir(),
             st.getSecondDaughter().getEncodedName())));
 
     // Run the execute.  Look at what it returns.
@@ -315,7 +315,7 @@ public class TestSplitTransaction {
     SplitTransactionImpl st = prepareGOOD_SPLIT_ROW(spiedRegion);
     SplitTransactionImpl spiedUponSt = spy(st);
     doNothing().when(spiedUponSt).assertReferenceFileCount(anyInt(),
-        eq(parent.getRegionFileSystem().getSplitsDir(st.getFirstDaughter())));
+        eq(parent.getRegionStorage().getSplitsDir(st.getFirstDaughter())));
     when(spiedRegion.createDaughterRegionFromSplits(spiedUponSt.getSecondDaughter())).
         thenThrow(new MockedFailedDaughterCreation());
     // Run the execute.  Look at what it returns.


[18/18] hbase git commit: HBASE-14439 API cleanup

Posted by bu...@apache.org.
HBASE-14439 API cleanup

* passes `mvn -DskipTests compile`, does not pass `mvn -DskipTests test-compile`.
* some fixup from rebasing
* rename RegionFileSystem to RegionStorage
* switch from Path in that API to opaque identifiers
* move a few methods around based on propogating those changes
* mark a bunch of TODOs on current API gaps


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9fddf3cb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9fddf3cb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9fddf3cb

Branch: refs/heads/hbase-14439
Commit: 9fddf3cb288495e7b71e75d0718cc4845c245906
Parents: c347b84
Author: Sean Busbey <bu...@apache.org>
Authored: Fri Sep 16 12:08:12 2016 -0700
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Sep 29 11:07:29 2016 -0500

----------------------------------------------------------------------
 .../hadoop/hbase/protobuf/ProtobufUtil.java     |  29 +-
 .../hbase/client/ClientSideRegionScanner.java   |   2 +-
 .../hadoop/hbase/fs/MasterFileSystem.java       |  14 +-
 .../hadoop/hbase/fs/RegionFileSystem.java       | 311 -------
 .../apache/hadoop/hbase/fs/RegionStorage.java   | 342 ++++++++
 .../hadoop/hbase/fs/StorageIdentifier.java      |  26 +
 .../hbase/fs/legacy/LegacyMasterFileSystem.java |   7 +-
 .../hbase/fs/legacy/LegacyPathIdentifier.java   |  51 ++
 .../hbase/fs/legacy/LegacyRegionFileSystem.java | 777 ------------------
 .../hbase/fs/legacy/LegacyRegionStorage.java    | 807 +++++++++++++++++++
 .../hadoop/hbase/master/CatalogJanitor.java     |  15 +-
 .../master/procedure/CreateTableProcedure.java  |   1 +
 .../hbase/protobuf/ServerProtobufUtil.java      | 187 +++++
 .../hbase/regionserver/CompactionTool.java      |  16 +-
 .../hadoop/hbase/regionserver/HRegion.java      | 264 +++---
 .../hadoop/hbase/regionserver/HStore.java       |  51 +-
 .../hadoop/hbase/regionserver/Region.java       |   1 +
 .../RegionMergeTransactionImpl.java             |  19 +-
 .../hbase/regionserver/RegionSplitPolicy.java   |   4 +-
 .../regionserver/SplitTransactionImpl.java      |  44 +-
 .../apache/hadoop/hbase/regionserver/Store.java |   4 +-
 .../hadoop/hbase/regionserver/StoreFile.java    |   2 +
 .../hbase/regionserver/StoreFileWriter.java     |   3 +
 .../hbase/snapshot/RestoreSnapshotHelper.java   |   9 +-
 .../hadoop/hbase/snapshot/SnapshotManifest.java |   5 +-
 .../hbase/snapshot/SnapshotManifestV1.java      |  34 +-
 .../hadoop/hbase/util/FSRegionScanner.java      |   8 +-
 .../org/apache/hadoop/hbase/util/FSUtils.java   |   9 +-
 .../org/apache/hadoop/hbase/util/HBaseFsck.java |  54 +-
 .../hadoop/hbase/util/HBaseFsckRepair.java      |   3 +-
 .../org/apache/hadoop/hbase/util/HMerge.java    |  10 +-
 .../hadoop/hbase/util/ModifyRegionUtils.java    |   2 +-
 .../hadoop/hbase/util/RegionSplitter.java       |   5 +-
 .../org/apache/hadoop/hbase/TestIOFencing.java  |  12 +-
 .../hadoop/hbase/backup/TestHFileArchiving.java |   4 +-
 .../client/TestRestoreSnapshotFromClient.java   |   2 +-
 .../TestRegionObserverScannerOpenHook.java      |   4 +-
 .../TableSnapshotInputFormatTestBase.java       |   2 +-
 .../MasterProcedureTestingUtility.java          |   2 +-
 .../hbase/regionserver/TestAtomicOperation.java |   4 +-
 .../hbase/regionserver/TestCompaction.java      |   6 +-
 .../regionserver/TestCompactionPolicy.java      |   2 +-
 .../TestCorruptedRegionStoreFile.java           |   2 +-
 .../TestDefaultCompactSelection.java            |   6 +-
 .../hbase/regionserver/TestHMobStore.java       |   4 +-
 .../hadoop/hbase/regionserver/TestHRegion.java  |  55 +-
 .../regionserver/TestHRegionFileSystem.java     |  16 +-
 .../hbase/regionserver/TestHRegionInfo.java     |  10 +-
 .../hbase/regionserver/TestRecoveredEdits.java  |   4 +-
 .../TestRegionMergeTransaction.java             |  15 +-
 .../TestRegionMergeTransactionOnCluster.java    |   4 +-
 .../TestScannerHeartbeatMessages.java           |   4 +-
 .../regionserver/TestSplitTransaction.java      |  12 +-
 .../TestSplitTransactionOnCluster.java          |   6 +-
 .../hadoop/hbase/regionserver/TestStore.java    |  10 +-
 .../hbase/regionserver/TestStoreFile.java       |  16 +-
 .../TestStoreFileRefresherChore.java            |  12 +-
 .../regionserver/wal/AbstractTestWALReplay.java |   4 +-
 .../hbase/snapshot/SnapshotTestingUtils.java    |   6 +-
 .../hbase/util/HFileArchiveTestingUtil.java     |   2 +-
 .../util/hbck/OfflineMetaRebuildTestCore.java   |   4 +-
 61 files changed, 1824 insertions(+), 1522 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 623acd5..256071a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -148,7 +148,6 @@ import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.Regio
 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
-import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
@@ -2671,32 +2670,6 @@ public final class ProtobufUtil {
     }
   }
 
-  public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family,
-      List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
-    return toCompactionDescriptor(info, null, family, inputPaths, outputPaths, storeDir);
-  }
-
-  public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] regionName,
-      byte[] family, List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
-    // compaction descriptor contains relative paths.
-    // input / output paths are relative to the store dir
-    // store dir is relative to region dir
-    CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder()
-        .setTableName(ByteStringer.wrap(info.getTable().toBytes()))
-        .setEncodedRegionName(ByteStringer.wrap(
-          regionName == null ? info.getEncodedNameAsBytes() : regionName))
-        .setFamilyName(ByteStringer.wrap(family))
-        .setStoreHomeDir(storeDir.getName()); //make relative
-    for (Path inputPath : inputPaths) {
-      builder.addCompactionInput(inputPath.getName()); //relative path
-    }
-    for (Path outputPath : outputPaths) {
-      builder.addCompactionOutput(outputPath.getName());
-    }
-    builder.setRegionName(ByteStringer.wrap(info.getRegionName()));
-    return builder.build();
-  }
-
   public static FlushDescriptor toFlushDescriptor(FlushAction action, HRegionInfo hri,
       long flushSeqId, Map<byte[], List<Path>> committedFiles) {
     FlushDescriptor.Builder desc = FlushDescriptor.newBuilder()
@@ -3698,4 +3671,4 @@ public final class ProtobufUtil {
       throw ProtobufUtil.handleRemoteException(e);
     }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java
index dde2f10..a643428 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java
@@ -57,7 +57,7 @@ public class ClientSideRegionScanner extends AbstractClientScanner {
     scan.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED);
 
     // open region from the snapshot directory
-    this.region = HRegion.openHRegion(conf, fs, rootDir, hri, htd, null, null, null);
+    this.region = HRegion.openHRegion(fs, rootDir, hri, htd, conf);
 
     // create an internal region scanner
     this.scanner = region.getScanner(scan);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
index d4323ac..3bc89d4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.fs.legacy.LegacyMasterFileSystem;
-import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 
@@ -147,7 +147,7 @@ public abstract class MasterFileSystem {
   //  PUBLIC Methods - Table Region related
   // ==========================================================================
   public void deleteRegion(HRegionInfo regionInfo) throws IOException {
-    RegionFileSystem.destroy(conf, fs, rootDir, regionInfo);
+    RegionStorage.destroy(conf, regionInfo);
   }
 
   public Collection<HRegionInfo> getRegions(TableName tableName) throws IOException {
@@ -157,14 +157,14 @@ public abstract class MasterFileSystem {
   public abstract Collection<HRegionInfo> getRegions(FsContext ctx, TableName tableName)
     throws IOException;
 
-  // TODO: Move in HRegionFileSystem
+  // TODO: Move in HRegionStorage
   public void deleteFamilyFromFS(HRegionInfo regionInfo, byte[] familyName, boolean hasMob)
       throws IOException {
-    getRegionFileSystem(regionInfo).deleteFamily(Bytes.toString(familyName), hasMob);
+    getRegionStorage(regionInfo).deleteFamily(Bytes.toString(familyName), hasMob);
   }
 
-  public RegionFileSystem getRegionFileSystem(HRegionInfo regionInfo) throws IOException {
-    return RegionFileSystem.open(conf, fs, rootDir, regionInfo, false);
+  public RegionStorage getRegionStorage(HRegionInfo regionInfo) throws IOException {
+    return RegionStorage.open(conf, regionInfo, false);
   }
 
   // ==========================================================================
@@ -202,7 +202,7 @@ public abstract class MasterFileSystem {
   public void visitStoreFiles(FsContext ctx, TableName table, StoreFileVisitor visitor)
       throws IOException {
     for (HRegionInfo hri: getRegions(ctx, table)) {
-      RegionFileSystem.open(conf, fs, rootDir, hri, false).visitStoreFiles(visitor);
+      RegionStorage.open(conf, hri, false).visitStoreFiles(visitor);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
deleted file mode 100644
index 154d648..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionFileSystem.java
+++ /dev/null
@@ -1,311 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.fs;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.ClusterId;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.legacy.LegacyRegionFileSystem;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.FSUtilsWithRetries;
-import org.apache.hadoop.hbase.fs.FsContext;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.MetaUtils;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.regionserver.*;
-import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSHDFSUtils;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
-
-@InterfaceAudience.Private
-public abstract class RegionFileSystem {
-  private static Log LOG = LogFactory.getLog(RegionFileSystem.class);
-
-  private final Configuration conf;
-  private final HRegionInfo hri;
-  private final FileSystem fs;
-  private final Path rootDir;
-
-  protected RegionFileSystem(Configuration conf, FileSystem fs, Path rootDir, HRegionInfo hri) {
-    this.conf = conf;
-    this.rootDir = rootDir;
-    this.hri = hri;
-    this.fs = fs;
-  }
-
-  public Configuration getConfiguration() { return conf; }
-  public FileSystem getFileSystem() { return fs; }
-  public Path getRootDir() { return rootDir; }
-
-  public HRegionInfo getRegionInfo() { return hri; }
-  public TableName getTable() { return getRegionInfo().getTable(); }
-
-  // ==========================================================================
-  //  PUBLIC Interfaces - Visitors
-  // ==========================================================================
-  public interface StoreFileVisitor {
-    void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
-       throws IOException;
-  }
-
-  public void visitStoreFiles(final StoreFileVisitor visitor) throws IOException {
-    for (String familyName: getFamilies()) {
-      for (StoreFileInfo storeFile: getStoreFiles(familyName)) {
-        visitor.storeFile(getRegionInfo(), familyName, storeFile);
-      }
-    }
-  }
-
-  // ==========================================================================
-  //  PUBLIC Methods - Families Related
-  // ==========================================================================
-
-  /**
-   * @return the set of families present on disk
-   * @throws IOException
-   */
-  public abstract Collection<String> getFamilies() throws IOException;
-
-  public void deleteFamily(byte[] familyName, boolean hasMob) throws IOException {
-    deleteFamily(Bytes.toString(familyName), hasMob);
-  }
-
-  public abstract void deleteFamily(String familyName, boolean hasMob) throws IOException;
-
-  // ==========================================================================
-  //  PUBLIC Methods - Store Files related
-  // ==========================================================================
-
-  /**
-   * Returns the store files available for the family.
-   * This methods performs the filtering based on the valid store files.
-   * @param familyName Column Family Name
-   * @return a set of {@link StoreFileInfo} for the specified family.
-   */
-  public Collection<StoreFileInfo> getStoreFiles(final byte[] familyName) throws IOException {
-    return getStoreFiles(Bytes.toString(familyName));
-  }
-
-  public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
-    return getStoreFiles(familyName, true);
-  }
-
-  /**
-   * Returns the store files available for the family.
-   * This methods performs the filtering based on the valid store files.
-   * @param familyName Column Family Name
-   * @return a set of {@link StoreFileInfo} for the specified family.
-   */
-  public abstract Collection<StoreFileInfo> getStoreFiles(String familyName, boolean validate)
-      throws IOException;
-
-  // ==========================================================================
-  //  PUBLIC bootstrap
-  // ==========================================================================
-  protected abstract void bootstrap() throws IOException;
-  protected abstract void destroy() throws IOException;
-
-  // ==========================================================================
-  //  NOOOOO
-  // ==========================================================================
-  public abstract void checkRegionInfoOnFilesystem() throws IOException;
-  public abstract Path getRegionDir();
-  public abstract Path getTableDir();
-
-  public abstract Path getTempDir();
-
-  public HRegionInfo getRegionInfoForFS() { return hri; }
-
-  public abstract Path getStoreDir(final String familyName);
-  public abstract Path createTempName();
-  public abstract Path createStoreDir(final String familyName) throws IOException;
-  public abstract Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
-      throws IOException;
-
-  public abstract void cleanupTempDir() throws IOException;
-  public abstract void cleanupSplitsDir() throws IOException;
-  public abstract void cleanupMergesDir() throws IOException;
-  public abstract void cleanupAnySplitDetritus() throws IOException;
-
-  public abstract Path commitDaughterRegion(final HRegionInfo regionInfo)
-      throws IOException;
-  public abstract void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException;
-  public abstract StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
-      throws IOException;
-
-  public abstract Path commitStoreFile(final String familyName, final Path buildPath) throws IOException;
-  public abstract void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException;
-
-  public abstract void removeStoreFile(final String familyName, final Path filePath)
-      throws IOException;
-  public abstract void removeStoreFiles(final String familyName, final Collection<StoreFile> storeFiles)
-      throws IOException;
-
-  public abstract boolean hasReferences(final String familyName) throws IOException;
-  public abstract boolean hasReferences(final HTableDescriptor htd) throws IOException;
-
-  public abstract Path getStoreFilePath(final String familyName, final String fileName);
-
-  public abstract void logFileSystemState(final Log LOG) throws IOException;
-
-  public abstract void createSplitsDir() throws IOException;
-  public abstract Path getSplitsDir();
-  public abstract Path getSplitsDir(final HRegionInfo hri);
-
-  public abstract Path getMergesDir();
-  public abstract void createMergesDir() throws IOException;
-
-  public abstract Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
-      final StoreFile f, final Path mergedDir)
-      throws IOException;
-
-  public abstract void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException;
-
-  public abstract Path splitStoreFile(final HRegionInfo hri, final String familyName,
-      final StoreFile f, final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy)
-          throws IOException;
-
-  public abstract void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException;
-
-  public static HRegionInfo loadRegionInfoFileContent(FileSystem fs, Path regionDir)
-      throws IOException {
-    FSDataInputStream in = fs.open(new Path(regionDir, ".regioninfo"));
-    try {
-      return HRegionInfo.parseFrom(in);
-    } finally {
-      in.close();
-    }
-  }
-
-  // ==========================================================================
-  //  PUBLIC
-  // ==========================================================================
-  public static RegionFileSystem open(Configuration conf, HRegionInfo regionInfo, boolean bootstrap)
-      throws IOException {
-    return open(conf, FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf),
-        regionInfo, bootstrap);
-  }
-
-  public static RegionFileSystem open(Configuration conf, FileSystem fs, Path rootDir,
-      HRegionInfo regionInfo, boolean bootstrap) throws IOException {
-    // Cover both bases, the old way of setting default fs and the new.
-    // We're supposed to run on 0.20 and 0.21 anyways.
-    fs = rootDir.getFileSystem(conf);
-    FSUtils.setFsDefault(conf, new Path(fs.getUri()));
-    // make sure the fs has the same conf
-    fs.setConf(conf);
-
-    RegionFileSystem rfs = getInstance(conf, fs, rootDir, regionInfo);
-    if (bootstrap) {
-      // TODO: are bootstrap and create two different things?
-      // should switch to bootstrap & read-only
-      // legacy region wants to recover the .regioninfo :(
-      rfs.bootstrap();
-    }
-    return rfs;
-  }
-
-  public static void destroy(Configuration conf, HRegionInfo regionInfo) throws IOException {
-    destroy(conf, FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), regionInfo);
-  }
-
-  public static void destroy(Configuration conf, FileSystem fs,
-      Path rootDir, HRegionInfo regionInfo) throws IOException {
-    getInstance(conf, fs, rootDir, regionInfo).destroy();
-  }
-
-  private static RegionFileSystem getInstance(Configuration conf, FileSystem fs,
-      Path rootDir, HRegionInfo regionInfo) throws IOException {
-    String fsType = conf.get("hbase.fs.layout.type", "legacy").toLowerCase();
-    switch (fsType) {
-      case "legacy":
-        return new LegacyRegionFileSystem(conf, fs, rootDir, regionInfo);
-      default:
-        throw new IOException("Invalid filesystem type " + fsType);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
new file mode 100644
index 0000000..ba7d0bf
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
@@ -0,0 +1,342 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.ClusterId;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyRegionStorage;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.FSUtilsWithRetries;
+import org.apache.hadoop.hbase.fs.FsContext;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.MetaUtils;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.regionserver.*;
+import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSHDFSUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
+
+@InterfaceAudience.Private
+public abstract class RegionStorage<IDENTIFIER extends StorageIdentifier> {
+  private static Log LOG = LogFactory.getLog(RegionStorage.class);
+
+  private final Configuration conf;
+  private final HRegionInfo hri;
+  private final FileSystem fs;
+  private final IDENTIFIER rootContainer;
+
+  protected RegionStorage(Configuration conf, FileSystem fs, IDENTIFIER rootContainer, HRegionInfo hri) {
+    this.conf = conf;
+    this.rootContainer = rootContainer;
+    this.hri = hri;
+    this.fs = fs;
+  }
+
+  /* Probably remove */ 
+  public Configuration getConfiguration() { return conf; }
+  /* TODO Definitely remove */
+  public FileSystem getFileSystem() { return fs; }
+  public IDENTIFIER getRootContainer() { return rootContainer; }
+
+  public HRegionInfo getRegionInfo() { return hri; }
+  public TableName getTable() { return getRegionInfo().getTable(); }
+
+  // ==========================================================================
+  //  PUBLIC Interfaces - Visitors
+  // ==========================================================================
+  public interface StoreFileVisitor {
+    void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
+       throws IOException;
+  }
+
+  public void visitStoreFiles(final StoreFileVisitor visitor) throws IOException {
+    for (String familyName: getFamilies()) {
+      for (StoreFileInfo storeFile: getStoreFiles(familyName)) {
+        visitor.storeFile(getRegionInfo(), familyName, storeFile);
+      }
+    }
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - Families Related
+  // ==========================================================================
+
+  /**
+   * @return the set of families present on disk
+   * @throws IOException
+   */
+  public abstract Collection<String> getFamilies() throws IOException;
+
+  public void deleteFamily(byte[] familyName, boolean hasMob) throws IOException {
+    deleteFamily(Bytes.toString(familyName), hasMob);
+  }
+
+  public abstract void deleteFamily(String familyName, boolean hasMob) throws IOException;
+
+  // ==========================================================================
+  //  PUBLIC Methods - Store Files related
+  // ==========================================================================
+
+  /**
+   * Returns the store files available for the family.
+   * This methods performs the filtering based on the valid store files.
+   * @param familyName Column Family Name
+   * @return a set of {@link StoreFileInfo} for the specified family.
+   */
+  public Collection<StoreFileInfo> getStoreFiles(final byte[] familyName) throws IOException {
+    return getStoreFiles(Bytes.toString(familyName));
+  }
+
+  public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
+    return getStoreFiles(familyName, true);
+  }
+
+  /**
+   * Returns the store files available for the family.
+   * This methods performs the filtering based on the valid store files.
+   * @param familyName Column Family Name
+   * @return a set of {@link StoreFileInfo} for the specified family.
+   */
+  public abstract Collection<StoreFileInfo> getStoreFiles(String familyName, boolean validate)
+      throws IOException;
+
+  // ==========================================================================
+  //  PUBLIC bootstrap
+  // ==========================================================================
+  protected abstract void bootstrap() throws IOException;
+  protected abstract void destroy() throws IOException;
+
+  // ==========================================================================
+  //  NOOOOO
+  // ==========================================================================
+  // TODO refactor to checkRegionInfoInStorage or remove
+  public abstract void checkRegionInfoOnFilesystem() throws IOException;
+  /**
+   * Get an opaque handle to the backing storage associated with this region.
+   *
+   */
+  public abstract IDENTIFIER getRegionContainer();
+  
+  public abstract IDENTIFIER getTableContainer();
+
+  public abstract IDENTIFIER getTempContainer();
+
+  public HRegionInfo getRegionInfoForFS() { return hri; }
+
+  /**
+   * Retrieve a referene to the backing storage associated with a particular family within this region.
+   */
+  public abstract IDENTIFIER getStoreContainer(final String familyName);
+   
+  public abstract IDENTIFIER getTempIdentifier();
+  public abstract IDENTIFIER createStoreContainer(final String familyName) throws IOException;
+  /**
+   * Move the given identifier into given family and return a StoreFile object with properly
+   * initialized info and reader objects.
+   */
+  public abstract StoreFile bulkLoadStoreFile(final String familyName, IDENTIFIER srcPath, long seqNum, final CacheConfig cacheConfig, final BloomType cfBloomType, final RegionCoprocessorHost coprocessorHost)
+      throws IOException;
+
+  public abstract void cleanupTempContainer() throws IOException;
+  public abstract void cleanupSplitsContainer() throws IOException;
+  public abstract void cleanupMergesContainer() throws IOException;
+  public abstract void cleanupAnySplitDetritus() throws IOException;
+
+  public abstract IDENTIFIER commitDaughterRegion(final HRegionInfo regionInfo)
+      throws IOException;
+  public abstract void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException;
+  public abstract StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
+      throws IOException;
+
+  public abstract StoreFile commitStoreFile(final String familyName, final IDENTIFIER uncommitted, final CacheConfig cacheConf, final BloomType cfBloomType, final RegionCoprocessorHost coprocessorHost) throws IOException;
+  public abstract void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException;
+
+  public abstract void removeStoreFiles(final String familyName, final Collection<StoreFile> storeFiles)
+      throws IOException;
+
+  public abstract boolean hasReferences(final String familyName) throws IOException;
+  public abstract boolean hasReferences(final HTableDescriptor htd) throws IOException;
+
+  public abstract IDENTIFIER getStoreFileStorageIdentifier(final String familyName, final String fileName);
+
+  public abstract long getStoreFileLen(final StoreFile store) throws IOException;
+
+  public abstract void logFileSystemState(final Log LOG) throws IOException;
+
+  public abstract void createSplitsContainer() throws IOException;
+  public abstract IDENTIFIER getSplitsContainer();
+  public abstract IDENTIFIER getSplitsContainer(final HRegionInfo hri);
+
+  public abstract IDENTIFIER getMergesContainer();
+  public abstract void createMergesContainer() throws IOException;
+
+  public abstract IDENTIFIER mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
+      final StoreFile f, final IDENTIFIER mergedContainer)
+      throws IOException;
+
+  public abstract void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException;
+
+  public abstract IDENTIFIER splitStoreFile(final HRegionInfo hri, final String familyName,
+      final StoreFile f, final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy)
+          throws IOException;
+
+  public abstract void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException;
+
+  // ==========================================================================
+  //  PUBLIC
+  // ==========================================================================
+  public static RegionStorage open(Configuration conf, StorageIdentifier regionContainer, boolean bootstrap) throws IOException {
+    RegionStorage rs = getInstance(conf, FSUtils.getCurrentFileSystem(conf), new LegacyPathIdentifier(FSUtils.getRootDir(conf)),
+        regionContainer);
+    if (bootstrap) {
+      rs.bootstrap();
+    }
+    return rs;
+  }
+
+  public static RegionStorage open(Configuration conf, HRegionInfo regionInfo, boolean bootstrap)
+      throws IOException {
+    return open(conf, FSUtils.getCurrentFileSystem(conf), new LegacyPathIdentifier(FSUtils.getRootDir(conf)),
+        regionInfo, bootstrap);
+  }
+
+  /* TODO remove this method and have the RegionStorage implementation determine the root container */
+  public static RegionStorage open(Configuration conf, FileSystem fs, StorageIdentifier rootContainer,
+      HRegionInfo regionInfo, boolean bootstrap) throws IOException {
+
+    if (rootContainer instanceof LegacyPathIdentifier) {
+      // Cover both bases, the old way of setting default fs and the new.
+      // We're supposed to run on 0.20 and 0.21 anyways.
+      fs = ((LegacyPathIdentifier)rootContainer).path.getFileSystem(conf);
+      FSUtils.setFsDefault(conf, new Path(fs.getUri()));
+      // make sure the fs has the same conf
+      fs.setConf(conf);
+    } else {
+      LOG.debug("skipping override of the default FS, since the root container is not a LegacyPathIdentifier.");
+    }
+
+    RegionStorage rfs = getInstance(conf, fs, rootContainer, regionInfo);
+    if (bootstrap) {
+      // TODO: are bootstrap and create two different things?
+      // should switch to bootstrap & read-only
+      // legacy region wants to recover the .regioninfo :(
+      rfs.bootstrap();
+    }
+    return rfs;
+  }
+
+  public static void destroy(Configuration conf, HRegionInfo regionInfo) throws IOException {
+    destroy(conf, FSUtils.getCurrentFileSystem(conf), new LegacyPathIdentifier(FSUtils.getRootDir(conf)), regionInfo);
+  }
+
+  public static void destroy(Configuration conf, FileSystem fs,
+      StorageIdentifier rootContainer, HRegionInfo regionInfo) throws IOException {
+    getInstance(conf, fs, rootContainer, regionInfo).destroy();
+  }
+
+  private static RegionStorage getInstance(Configuration conf, FileSystem fs,
+      StorageIdentifier rootContainer, HRegionInfo regionInfo) throws IOException {
+    String fsType = conf.get("hbase.fs.storage.type", "legacy").toLowerCase();
+    switch (fsType) {
+      case "legacy":
+        return new LegacyRegionStorage(conf, fs, (LegacyPathIdentifier)rootContainer, regionInfo);
+      default:
+        throw new IOException("Invalid filesystem type " + fsType);
+    }
+  }
+
+  private static RegionStorage getInstance(Configuration conf, FileSystem fs,
+      StorageIdentifier rootContainer, StorageIdentifier regionContainer) throws IOException {
+    String fsType = conf.get("hbase.fs.storage.type", "legacy").toLowerCase();
+    switch (fsType) {
+      case "legacy":
+        return new LegacyRegionStorage(conf, fs, (LegacyPathIdentifier)rootContainer, (LegacyPathIdentifier)regionContainer);
+      default:
+        throw new IOException("Invalid filesystem type " + fsType);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/StorageIdentifier.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/StorageIdentifier.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/StorageIdentifier.java
new file mode 100644
index 0000000..69c1430
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/StorageIdentifier.java
@@ -0,0 +1,26 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+/**
+ * An opaque holder for storage implementation specific information.
+ */
+public interface StorageIdentifier {
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
index f6201a1..5cf363b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
@@ -217,8 +217,9 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
   // ==========================================================================
   @Override
   protected void bootstrapMeta() throws IOException {
+    // TODO ask RegionStorage
     if (!FSUtils.metaRegionExists(getFileSystem(), getRootDir())) {
-      bootstrapMeta(getRootDir(), getConfiguration());
+      bootstrapMeta(getConfiguration());
     }
 
     // Create tableinfo-s for hbase:meta if not already there.
@@ -228,7 +229,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
     createTableDescriptor(HTableDescriptor.metaTableDescriptor(getConfiguration()), false);
   }
 
-  private static void bootstrapMeta(final Path rd, final Configuration c) throws IOException {
+  private static void bootstrapMeta(final Configuration c) throws IOException {
     LOG.info("BOOTSTRAP: creating hbase:meta region");
     try {
       // Bootstrapping, make sure blockcache is off.  Else, one will be
@@ -238,7 +239,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
       HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
       HTableDescriptor metaDescriptor = HTableDescriptor.metaTableDescriptor(c);
       MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, false);
-      HRegion meta = HRegion.createHRegion(c, rd, metaDescriptor, metaHRI, null);
+      HRegion meta = HRegion.createHRegion(c, metaDescriptor, metaHRI, null);
       MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, true);
       meta.close();
     } catch (IOException e) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyPathIdentifier.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyPathIdentifier.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyPathIdentifier.java
new file mode 100644
index 0000000..aaebfec
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyPathIdentifier.java
@@ -0,0 +1,51 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs.legacy;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
+
+/**
+ * Wraps HDFS Paths for use as identifiers
+ */
+@InterfaceAudience.Private
+public class LegacyPathIdentifier implements StorageIdentifier {
+  public final Path path;
+
+  public LegacyPathIdentifier(Path path) {
+    this.path = path;
+  }
+
+  @Override
+  public String toString() {
+    return path.toString();
+  }
+
+  @Override
+  public int hashCode() {
+    return path.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    return path.equals(other);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/9fddf3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java
deleted file mode 100644
index ce914a0..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java
+++ /dev/null
@@ -1,777 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.fs.legacy;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.FSUtilsWithRetries;
-import org.apache.hadoop.hbase.fs.FsContext;
-import org.apache.hadoop.hbase.fs.RegionFileSystem;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.MetaUtils;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.regionserver.*;
-import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSHDFSUtils;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
-
-@InterfaceAudience.Private
-public class LegacyRegionFileSystem extends RegionFileSystem {
-  private static final Log LOG = LogFactory.getLog(LegacyRegionFileSystem.class);
-
-  private final Path tableDir;
-  private final Path regionDir;
-  private final Path mobDir;
-
-  // regionInfo for interacting with FS (getting encodedName, etc)
-  private final HRegionInfo regionInfoForFs;
-
-  private final FSUtilsWithRetries fsWithRetries;
-
-  public LegacyRegionFileSystem(Configuration conf, FileSystem fs, Path rootDir, HRegionInfo hri) {
-    super(conf, fs, rootDir, hri);
-
-    Path dataDir = LegacyLayout.getDataDir(rootDir);
-    this.tableDir = LegacyLayout.getTableDir(dataDir, hri.getTable());
-    this.regionDir = LegacyLayout.getRegionDir(tableDir, hri);
-    this.mobDir = LegacyLayout.getDataDir(LegacyLayout.getMobDir(rootDir));
-    this.fsWithRetries = new FSUtilsWithRetries(conf, fs);
-
-    this.regionInfoForFs = ServerRegionReplicaUtil.getRegionInfoForFs(hri);
-  }
-
-  public Path getRegionDir() {
-    return regionDir;
-  }
-
-  public Path getTableDir() {
-    return tableDir;
-  }
-
-  // ==========================================================================
-  //  PUBLIC Methods - Families Related
-  // ==========================================================================
-  @Override
-  public Collection<String> getFamilies() throws IOException {
-    FileSystem fs = getFileSystem();
-    FileStatus[] fds = FSUtils.listStatus(fs, regionDir, new FSUtils.FamilyDirFilter(fs));
-    if (fds == null) return Collections.emptyList();
-
-    ArrayList<String> families = new ArrayList<String>(fds.length);
-    for (FileStatus status: fds) {
-      families.add(status.getPath().getName());
-    }
-    return families;
-  }
-
-  @Override
-  public void deleteFamily(String familyName, boolean hasMob) throws IOException {
-    // archive family store files
-    byte[] bFamilyName = Bytes.toBytes(familyName);
-
-    FileSystem fs = getFileSystem();
-    HFileArchiver.archiveFamily(fs, getConfiguration(), getRegionInfo(), tableDir, bFamilyName);
-
-    // delete the family folder
-    HRegionInfo region = getRegionInfo();
-    Path familyDir = new Path(tableDir, new Path(region.getEncodedName(), familyName));
-    if (!fsWithRetries.deleteDir(familyDir)) {
-      throw new IOException("Could not delete family "
-          + familyName + " from FileSystem for region "
-          + region.getRegionNameAsString() + "(" + region.getEncodedName()
-          + ")");
-    }
-
-    // archive and delete mob files
-    if (hasMob) {
-      Path mobTableDir = LegacyLayout.getTableDir(mobDir, getTable());
-      HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(getTable());
-      Path mobRegionDir = LegacyLayout.getRegionDir(mobTableDir, mobRegionInfo);
-      Path mobFamilyDir = LegacyLayout.getFamilyDir(mobRegionDir, familyName);
-      // archive mob family store files
-      MobUtils.archiveMobStoreFiles(getConfiguration(), getFileSystem(),
-          mobRegionInfo, mobFamilyDir, bFamilyName);
-
-      if (!fsWithRetries.deleteDir(mobFamilyDir)) {
-        throw new IOException("Could not delete mob store files for family "
-            + familyName + " from FileSystem region "
-            + mobRegionInfo.getRegionNameAsString() + "(" + mobRegionInfo.getEncodedName() + ")");
-      }
-    }
-  }
-
-  // ===========================================================================
-  //  Temp Helpers
-  // ===========================================================================
-  /** @return {@link Path} to the region's temp directory, used for file creations */
-  public Path getTempDir() {
-    return LegacyLayout.getRegionTempDir(regionDir);
-  }
-
-  /**
-   * Clean up any temp detritus that may have been left around from previous operation attempts.
-   */
-  public void cleanupTempDir() throws IOException {
-    fsWithRetries.deleteDir(getTempDir());
-  }
-
-  // ===========================================================================
-  //  Store/StoreFile Helpers
-  // ===========================================================================
-  /**
-   * Returns the directory path of the specified family
-   * @param familyName Column Family Name
-   * @return {@link Path} to the directory of the specified family
-   */
-  public Path getStoreDir(final String familyName) {
-    return LegacyLayout.getFamilyDir(getRegionDir(), familyName);
-  }
-
-  /**
-   * Create the store directory for the specified family name
-   * @param familyName Column Family Name
-   * @return {@link Path} to the directory of the specified family
-   * @throws IOException if the directory creation fails.
-   */
-  public Path createStoreDir(final String familyName) throws IOException {
-    Path storeDir = getStoreDir(familyName);
-    if (!fsWithRetries.createDir(storeDir))
-      throw new IOException("Failed creating "+storeDir);
-    return storeDir;
-  }
-
-  // ==========================================================================
-  //  PUBLIC Methods - Store Files related
-  // ==========================================================================
-
-  @Override
-  public Collection<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate)
-      throws IOException {
-    Path familyDir = getStoreDir(familyName);
-    FileStatus[] files = FSUtils.listStatus(getFileSystem(), familyDir);
-    if (files == null) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("No StoreFiles for: " + familyDir);
-      }
-      return null;
-    }
-
-    ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(files.length);
-    for (FileStatus status: files) {
-      if (validate && !StoreFileInfo.isValid(status)) {
-        LOG.warn("Invalid StoreFile: " + status.getPath());
-        continue;
-      }
-      StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(getConfiguration(),
-          getFileSystem(), getRegionInfo(), regionInfoForFs, familyName, status.getPath());
-      storeFiles.add(info);
-
-    }
-    return storeFiles;
-  }
-
-
-  /**
-   * Return Qualified Path of the specified family/file
-   *
-   * @param familyName Column Family Name
-   * @param fileName File Name
-   * @return The qualified Path for the specified family/file
-   */
-  public Path getStoreFilePath(final String familyName, final String fileName) {
-    Path familyDir = getStoreDir(familyName);
-    return LegacyLayout.getStoreFile(familyDir, fileName).makeQualified(getFileSystem());
-  }
-
-  /**
-   * Return the store file information of the specified family/file.
-   *
-   * @param familyName Column Family Name
-   * @param fileName File Name
-   * @return The {@link StoreFileInfo} for the specified family/file
-   */
-  public StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
-      throws IOException {
-    Path familyDir = getStoreDir(familyName);
-    return ServerRegionReplicaUtil.getStoreFileInfo(getConfiguration(),
-      getFileSystem(), getRegionInfo(), regionInfoForFs, familyName,
-      LegacyLayout.getStoreFile(familyDir, fileName));
-  }
-
-  /**
-   * Returns true if the specified family has reference files
-   * @param familyName Column Family Name
-   * @return true if family contains reference files
-   * @throws IOException
-   */
-  public boolean hasReferences(final String familyName) throws IOException {
-    FileStatus[] files = FSUtils.listStatus(getFileSystem(), getStoreDir(familyName));
-    if (files != null) {
-      for(FileStatus stat: files) {
-        if(stat.isDirectory()) {
-          continue;
-        }
-        if(StoreFileInfo.isReference(stat.getPath())) {
-          return true;
-        }
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Check whether region has Reference file
-   * @param htd table desciptor of the region
-   * @return true if region has reference file
-   * @throws IOException
-   */
-  public boolean hasReferences(final HTableDescriptor htd) throws IOException {
-    for (HColumnDescriptor family : htd.getFamilies()) {
-      if (hasReferences(family.getNameAsString())) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-
-  /**
-   * Generate a unique file name, used by createTempName() and commitStoreFile()
-   * @param suffix extra information to append to the generated name
-   * @return Unique file name
-   */
-  private static String generateUniqueName(final String suffix) {
-    String name = UUID.randomUUID().toString().replaceAll("-", "");
-    if (suffix != null) name += suffix;
-    return name;
-  }
-
-  /**
-   * Generate a unique temporary Path. Used in conjuction with commitStoreFile()
-   * to get a safer file creation.
-   * <code>
-   * Path file = fs.createTempName();
-   * ...StoreFile.Writer(file)...
-   * fs.commitStoreFile("family", file);
-   * </code>
-   *
-   * @return Unique {@link Path} of the temporary file
-   */
-  public Path createTempName() {
-    return createTempName(null);
-  }
-
-  /**
-   * Generate a unique temporary Path. Used in conjuction with commitStoreFile()
-   * to get a safer file creation.
-   * <code>
-   * Path file = fs.createTempName();
-   * ...StoreFile.Writer(file)...
-   * fs.commitStoreFile("family", file);
-   * </code>
-   *
-   * @param suffix extra information to append to the generated name
-   * @return Unique {@link Path} of the temporary file
-   */
-  public Path createTempName(final String suffix) {
-    return new Path(getTempDir(), generateUniqueName(suffix));
-  }
-
-  /**
-   * Move the file from a build/temp location to the main family store directory.
-   * @param familyName Family that will gain the file
-   * @param buildPath {@link Path} to the file to commit.
-   * @return The new {@link Path} of the committed file
-   * @throws IOException
-   */
-  public Path commitStoreFile(final String familyName, final Path buildPath) throws IOException {
-    return commitStoreFile(familyName, buildPath, -1, false);
-  }
-
-  /**
-   * Move the file from a build/temp location to the main family store directory.
-   * @param familyName Family that will gain the file
-   * @param buildPath {@link Path} to the file to commit.
-   * @param seqNum Sequence Number to append to the file name (less then 0 if no sequence number)
-   * @param generateNewName False if you want to keep the buildPath name
-   * @return The new {@link Path} of the committed file
-   * @throws IOException
-   */
-  private Path commitStoreFile(final String familyName, final Path buildPath,
-      final long seqNum, final boolean generateNewName) throws IOException {
-    Path storeDir = getStoreDir(familyName);
-    if(!fsWithRetries.createDir(storeDir))
-      throw new IOException("Failed creating " + storeDir);
-
-    String name = buildPath.getName();
-    if (generateNewName) {
-      name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_");
-    }
-    Path dstPath = new Path(storeDir, name);
-    if (!fsWithRetries.exists(buildPath)) {
-      throw new FileNotFoundException(buildPath.toString());
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Committing store file " + buildPath + " as " + dstPath);
-    }
-    // buildPath exists, therefore not doing an exists() check.
-    if (!fsWithRetries.rename(buildPath, dstPath)) {
-      throw new IOException("Failed rename of " + buildPath + " to " + dstPath);
-    }
-    return dstPath;
-  }
-
-
-  /**
-   * Moves multiple store files to the relative region's family store directory.
-   * @param storeFiles list of store files divided by family
-   * @throws IOException
-   */
-  public void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException {
-    for (Map.Entry<byte[], List<StoreFile>> es: storeFiles.entrySet()) {
-      String familyName = Bytes.toString(es.getKey());
-      for (StoreFile sf: es.getValue()) {
-        commitStoreFile(familyName, sf.getPath());
-      }
-    }
-  }
-
-  /**
-   * Archives the specified store file from the specified family.
-   * @param familyName Family that contains the store files
-   * @param filePath {@link Path} to the store file to remove
-   * @throws IOException if the archiving fails
-   */
-  public void removeStoreFile(final String familyName, final Path filePath)
-      throws IOException {
-    HFileArchiver.archiveStoreFile(getConfiguration(), getFileSystem(), this.regionInfoForFs,
-        this.tableDir, Bytes.toBytes(familyName), filePath);
-  }
-
-  /**
-   * Closes and archives the specified store files from the specified family.
-   * @param familyName Family that contains the store files
-   * @param storeFiles set of store files to remove
-   * @throws IOException if the archiving fails
-   */
-  public void removeStoreFiles(final String familyName, final Collection<StoreFile> storeFiles)
-      throws IOException {
-    HFileArchiver.archiveStoreFiles(getConfiguration(), getFileSystem(), this.regionInfoForFs,
-        this.tableDir, Bytes.toBytes(familyName), storeFiles);
-  }
-
-  /**
-   * Bulk load: Add a specified store file to the specified family.
-   * If the source file is on the same different file-system is moved from the
-   * source location to the destination location, otherwise is copied over.
-   *
-   * @param familyName Family that will gain the file
-   * @param srcPath {@link Path} to the file to import
-   * @param seqNum Bulk Load sequence number
-   * @return The destination {@link Path} of the bulk loaded file
-   * @throws IOException
-   */
-  public Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
-      throws IOException {
-    // Copy the file if it's on another filesystem
-    FileSystem fs = getFileSystem();
-    FileSystem srcFs = srcPath.getFileSystem(getConfiguration());
-    FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs;
-
-    // We can't compare FileSystem instances as equals() includes UGI instance
-    // as part of the comparison and won't work when doing SecureBulkLoad
-    // TODO deal with viewFS
-    if (!FSHDFSUtils.isSameHdfs(getConfiguration(), srcFs, desFs)) {
-      LOG.info("Bulk-load file " + srcPath + " is on different filesystem than " +
-          "the destination store. Copying file over to destination filesystem.");
-      Path tmpPath = createTempName();
-      FileUtil.copy(srcFs, srcPath, fs, tmpPath, false, getConfiguration());
-      LOG.info("Copied " + srcPath + " to temporary path on destination filesystem: " + tmpPath);
-      srcPath = tmpPath;
-    }
-
-    return commitStoreFile(familyName, srcPath, seqNum, true);
-  }
-
-  // ===========================================================================
-  //  Splits Helpers
-  // ===========================================================================
-  /** @return {@link Path} to the temp directory used during split operations */
-  public Path getSplitsDir() {
-    return LegacyLayout.getRegionSplitsDir(getRegionDir());
-  }
-
-  public Path getSplitsDir(final HRegionInfo hri) {
-    return LegacyLayout.getRegionSplitsDir(getSplitsDir(), hri);
-  }
-
-  /**
-   * Clean up any split detritus that may have been left around from previous split attempts.
-   */
-  public void cleanupSplitsDir() throws IOException {
-    fsWithRetries.deleteDir(getSplitsDir());
-  }
-
-  /**
-   * Clean up any split detritus that may have been left around from previous
-   * split attempts.
-   * Call this method on initial region deploy.
-   * @throws IOException
-   */
-  public void cleanupAnySplitDetritus() throws IOException {
-    Path splitdir = this.getSplitsDir();
-    if (!fsWithRetries.exists(splitdir)) return;
-    // Look at the splitdir.  It could have the encoded names of the daughter
-    // regions we tried to make.  See if the daughter regions actually got made
-    // out under the tabledir.  If here under splitdir still, then the split did
-    // not complete.  Try and do cleanup.  This code WILL NOT catch the case
-    // where we successfully created daughter a but regionserver crashed during
-    // the creation of region b.  In this case, there'll be an orphan daughter
-    // dir in the filesystem.  TOOD: Fix.
-    FileSystem fs = getFileSystem();
-    FileStatus[] daughters = FSUtils.listStatus(fs, splitdir, new FSUtils.DirFilter(fs));
-    if (daughters != null) {
-      for (int i = 0; i < daughters.length; ++i) {
-        Path daughterDir = new Path(this.tableDir, daughters[i].getPath().getName());
-        if (!fsWithRetries.deleteDir(daughterDir)) {
-          throw new IOException("Failed delete of " + daughterDir);
-        }
-      }
-    }
-    cleanupSplitsDir();
-    LOG.info("Cleaned up old failed split transaction detritus: " + splitdir);
-  }
-
-  /**
-   * Remove daughter region
-   * @param regionInfo daughter {@link HRegionInfo}
-   * @throws IOException
-   */
-  public void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException {
-    Path regionDir = LegacyLayout.getRegionDir(tableDir, regionInfo);
-    if (!fsWithRetries.deleteDir(regionDir)) {
-      throw new IOException("Failed delete of " + regionDir);
-    }
-  }
-
-  /**
-   * Commit a daughter region, moving it from the split temporary directory
-   * to the proper location in the filesystem.
-   *
-   * @param regionInfo                 daughter {@link org.apache.hadoop.hbase.HRegionInfo}
-   * @throws IOException
-   */
-  public Path commitDaughterRegion(final HRegionInfo regionInfo)
-      throws IOException {
-    Path regionDir = LegacyLayout.getRegionDir(tableDir, regionInfo);
-    Path daughterTmpDir = this.getSplitsDir(regionInfo);
-
-    if (fsWithRetries.exists(daughterTmpDir)) {
-
-      // Write HRI to a file in case we need to recover hbase:meta
-      Path regionInfoFile = LegacyLayout.getRegionInfoFile(daughterTmpDir);
-      byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
-      writeRegionInfoFileContent(getConfiguration(), getFileSystem(), regionInfoFile, regionInfoContent);
-
-      // Move the daughter temp dir to the table dir
-      if (!fsWithRetries.rename(daughterTmpDir, regionDir)) {
-        throw new IOException("Unable to rename " + daughterTmpDir + " to " + regionDir);
-      }
-    }
-
-    return regionDir;
-  }
-
-  /**
-   * Create the region splits directory.
-   */
-  public void createSplitsDir() throws IOException {
-    createTempDir(getSplitsDir());
-  }
-
-  void createTempDir(Path dir) throws IOException {
-    if (fsWithRetries.exists(dir)) {
-      LOG.info("The " + dir + " directory exists.  Hence deleting it to recreate it");
-      if (!fsWithRetries.deleteDir(dir)) {
-        throw new IOException("Failed deletion of " + dir + " before creating them again.");
-      }
-    }
-    // dir doesn't exists now. No need to do an exists() call for it.
-    if (!fsWithRetries.createDir(dir)) {
-      throw new IOException("Failed create of " + dir);
-    }
-  }
-
-  /**
-   * Write out a split reference. Package local so it doesnt leak out of
-   * regionserver.
-   * @param hri {@link HRegionInfo} of the destination
-   * @param familyName Column Family Name
-   * @param f File to split.
-   * @param splitRow Split Row
-   * @param top True if we are referring to the top half of the hfile.
-   * @param splitPolicy
-   * @return Path to created reference.
-   * @throws IOException
-   */
-  public Path splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f,
-      final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy)
-          throws IOException {
-
-    if (splitPolicy == null || !splitPolicy.skipStoreFileRangeCheck(familyName)) {
-      // Check whether the split row lies in the range of the store file
-      // If it is outside the range, return directly.
-      try {
-        if (top) {
-          //check if larger than last key.
-          Cell splitKey = CellUtil.createFirstOnRow(splitRow);
-          Cell lastKey = f.getLastKey();
-          // If lastKey is null means storefile is empty.
-          if (lastKey == null) {
-            return null;
-          }
-          if (f.getComparator().compare(splitKey, lastKey) > 0) {
-            return null;
-          }
-        } else {
-          //check if smaller than first key
-          Cell splitKey = CellUtil.createLastOnRow(splitRow);
-          Cell firstKey = f.getFirstKey();
-          // If firstKey is null means storefile is empty.
-          if (firstKey == null) {
-            return null;
-          }
-          if (f.getComparator().compare(splitKey, firstKey) < 0) {
-            return null;
-          }
-        }
-      } finally {
-        f.closeReader(true);
-      }
-    }
-
-    Path splitDir = new Path(getSplitsDir(hri), familyName);
-    // A reference to the bottom half of the hsf store file.
-    Reference r =
-      top ? Reference.createTopReference(splitRow): Reference.createBottomReference(splitRow);
-    // Add the referred-to regions name as a dot separated suffix.
-    // See REF_NAME_REGEX regex above.  The referred-to regions name is
-    // up in the path of the passed in <code>f</code> -- parentdir is family,
-    // then the directory above is the region name.
-    String parentRegionName = regionInfoForFs.getEncodedName();
-    // Write reference with same file id only with the other region name as
-    // suffix and into the new region location (under same family).
-    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
-    return r.write(getFileSystem(), p);
-  }
-
-  // ===========================================================================
-  //  Merge Helpers
-  // ===========================================================================
-  /** @return {@link Path} to the temp directory used during merge operations */
-  public Path getMergesDir() {
-    return LegacyLayout.getRegionMergesDir(getRegionDir());
-  }
-
-  Path getMergesDir(final HRegionInfo hri) {
-    return LegacyLayout.getRegionMergesDir(getMergesDir(), hri);
-  }
-
-  /**
-   * Clean up any merge detritus that may have been left around from previous merge attempts.
-   */
-  public void cleanupMergesDir() throws IOException {
-    fsWithRetries.deleteDir(getMergesDir());
-  }
-
-  /**
-   * Remove merged region
-   * @param mergedRegion {@link HRegionInfo}
-   * @throws IOException
-   */
-  public void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException {
-    Path regionDir = LegacyLayout.getRegionDir(tableDir, mergedRegion);
-    if (fsWithRetries.deleteDir(regionDir)) {
-      throw new IOException("Failed delete of " + regionDir);
-    }
-  }
-
-  /**
-   * Create the region merges directory.
-   * @throws IOException If merges dir already exists or we fail to create it.
-   * @see HRegionFileSystem#cleanupMergesDir()
-   */
-  public void createMergesDir() throws IOException {
-    createTempDir(getMergesDir());
-  }
-
-  /**
-   * Write out a merge reference under the given merges directory. Package local
-   * so it doesnt leak out of regionserver.
-   * @param mergedRegion {@link HRegionInfo} of the merged region
-   * @param familyName Column Family Name
-   * @param f File to create reference.
-   * @param mergedDir
-   * @return Path to created reference.
-   * @throws IOException
-   */
-  public Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
-      final StoreFile f, final Path mergedDir)
-      throws IOException {
-    Path referenceDir = new Path(new Path(mergedDir,
-        mergedRegion.getEncodedName()), familyName);
-    // A whole reference to the store file.
-    Reference r = Reference.createTopReference(regionInfoForFs.getStartKey());
-    // Add the referred-to regions name as a dot separated suffix.
-    // See REF_NAME_REGEX regex above. The referred-to regions name is
-    // up in the path of the passed in <code>f</code> -- parentdir is family,
-    // then the directory above is the region name.
-    String mergingRegionName = regionInfoForFs.getEncodedName();
-    // Write reference with same file id only with the other region name as
-    // suffix and into the new region location (under same family).
-    Path p = new Path(referenceDir, f.getPath().getName() + "."
-        + mergingRegionName);
-    return r.write(getFileSystem(), p);
-  }
-
-  /**
-   * Commit a merged region, moving it from the merges temporary directory to
-   * the proper location in the filesystem.
-   * @param mergedRegionInfo merged region {@link HRegionInfo}
-   * @throws IOException
-   */
-  public void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException {
-    Path regionDir = new Path(this.tableDir, mergedRegionInfo.getEncodedName());
-    Path mergedRegionTmpDir = this.getMergesDir(mergedRegionInfo);
-    // Move the tmp dir in the expected location
-    if (mergedRegionTmpDir != null && fsWithRetries.exists(mergedRegionTmpDir)) {
-      if (!fsWithRetries.rename(mergedRegionTmpDir, regionDir)) {
-        throw new IOException("Unable to rename " + mergedRegionTmpDir + " to "
-            + regionDir);
-      }
-    }
-  }
-
-  // ===========================================================================
-  //  Create/Open/Delete Helpers
-  // ===========================================================================
-  /**
-   * Log the current state of the region
-   * @param LOG log to output information
-   * @throws IOException if an unexpected exception occurs
-   */
-  public void logFileSystemState(final Log LOG) throws IOException {
-    FSUtils.logFileSystemState(getFileSystem(), this.getRegionDir(), LOG);
-  }
-
-  /**
-   * @param hri
-   * @return Content of the file we write out to the filesystem under a region
-   * @throws IOException
-   */
-  private static byte[] getRegionInfoFileContent(final HRegionInfo hri) throws IOException {
-    return hri.toDelimitedByteArray();
-  }
-
-  /**
-   * Write the .regioninfo file on-disk.
-   */
-  private static void writeRegionInfoFileContent(final Configuration conf, final FileSystem fs,
-      final Path regionInfoFile, final byte[] content) throws IOException {
-    // First check to get the permissions
-    FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
-    // Write the RegionInfo file content
-    FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null);
-    try {
-      out.write(content);
-    } finally {
-      out.close();
-    }
-  }
-
-  // ==========================================================================
-  //  PUBLIC bootstrap
-  // ==========================================================================
-  @Override
-  protected void bootstrap() throws IOException {
-    fsWithRetries.createDir(getRegionDir());
-
-    // Cleanup temporary directories
-    cleanupTempDir();
-    cleanupSplitsDir();
-    cleanupMergesDir();
-
-    // if it doesn't exists, Write HRI to a file, in case we need to recover hbase:meta
-    checkRegionInfoOnFilesystem();
-  }
-
-  public void checkRegionInfoOnFilesystem() throws IOException {
-    writeRegionInfoFileContent(getConfiguration(), getFileSystem(),
-      LegacyLayout.getRegionInfoFile(getRegionDir()),
-      getRegionInfoFileContent(getRegionInfo()));
-  }
-
-  @Override
-  protected void destroy() throws IOException {
-    fsWithRetries.deleteDir(regionDir);
-  }
-}


[08/18] hbase git commit: HBASE-14439 Moved from MasterFileSystem to MasterStorage abstract class for storage abstraction layer.

Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index 4c87b57..d471eda 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.master.procedure;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -43,8 +42,9 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.exceptions.HBaseException;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
 import org.apache.hadoop.hbase.mob.MobConstants;
@@ -262,11 +262,12 @@ public class DeleteTableProcedure
   protected static void deleteFromFs(final MasterProcedureEnv env,
       final TableName tableName, final List<HRegionInfo> regions,
       final boolean archive) throws IOException {
-    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
-    final FileSystem fs = mfs.getFileSystem();
-    final Path tempdir = mfs.getTempDir();
+    final MasterStorage ms = env.getMasterServices().getMasterStorage();
+    final FileSystem fs = ms.getFileSystem();
+    final Path tempdir = ((LegacyPathIdentifier)ms.getTempContainer()).path;
 
-    final Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
+    final Path tableDir = FSUtils.getTableDir(((LegacyPathIdentifier)ms.getRootContainer()).path,
+        tableName);
     final Path tempTableDir = FSUtils.getTableDir(tempdir, tableName);
 
     if (fs.exists(tableDir)) {
@@ -290,7 +291,8 @@ public class DeleteTableProcedure
           if (files != null && files.length > 0) {
             for (int i = 0; i < files.length; ++i) {
               if (!files[i].isDir()) continue;
-              HFileArchiver.archiveRegion(fs, mfs.getRootDir(), tempTableDir, files[i].getPath());
+              HFileArchiver.archiveRegion(fs, ((LegacyPathIdentifier) ms.getRootContainer()).path,
+                  tempTableDir, files[i].getPath());
             }
           }
           fs.delete(tempdir, true);
@@ -303,19 +305,20 @@ public class DeleteTableProcedure
     if (archive) {
       for (HRegionInfo hri : regions) {
         LOG.debug("Archiving region " + hri.getRegionNameAsString() + " from FS");
-        HFileArchiver.archiveRegion(fs, mfs.getRootDir(),
+        HFileArchiver.archiveRegion(fs, ((LegacyPathIdentifier) ms.getRootContainer()).path,
             tempTableDir, HRegion.getRegionDir(tempTableDir, hri.getEncodedName()));
       }
       LOG.debug("Table '" + tableName + "' archived!");
     }
 
     // Archive mob data
-    Path mobTableDir = FSUtils.getTableDir(new Path(mfs.getRootDir(), MobConstants.MOB_DIR_NAME),
-            tableName);
+    Path mobTableDir = FSUtils.getTableDir(new Path(((LegacyPathIdentifier) ms.getRootContainer())
+            .path, MobConstants.MOB_DIR_NAME), tableName);
     Path regionDir =
             new Path(mobTableDir, MobUtils.getMobRegionInfo(tableName).getEncodedName());
     if (fs.exists(regionDir)) {
-      HFileArchiver.archiveRegion(fs, mfs.getRootDir(), mobTableDir, regionDir);
+      HFileArchiver.archiveRegion(fs, ((LegacyPathIdentifier) ms.getRootContainer()).path,
+          mobTableDir, regionDir);
     }
 
     // Delete table directory from FS (temp directory)

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
index d37159b..dbcb7c7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
@@ -31,12 +31,10 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotDisabledException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.RegionLocator;
-import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.BulkReOpen;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -62,7 +60,7 @@ public final class MasterDDLOperationHelper {
       List<HRegionInfo> regionInfoList,
       final byte[] familyName,
       boolean hasMob) throws IOException {
-    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+    final MasterStorage ms = env.getMasterServices().getMasterStorage();
     if (LOG.isDebugEnabled()) {
       LOG.debug("Removing family=" + Bytes.toString(familyName) + " from table=" + tableName);
     }
@@ -71,7 +69,7 @@ public final class MasterDDLOperationHelper {
     }
     for (HRegionInfo hri : regionInfoList) {
       // Delete the family directory in FS for all the regions one by one
-      mfs.deleteFamilyFromFS(hri, familyName, hasMob);
+      ms.deleteFamilyFromStorage(hri, familyName, hasMob);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
index feb9228..9c373bd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
@@ -41,7 +41,8 @@ import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.master.MetricsSnapshot;
 import org.apache.hadoop.hbase.master.RegionStates;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
@@ -333,12 +334,12 @@ public class RestoreSnapshotProcedure
 
     if (!getTableName().isSystemTable()) {
       // Table already exist. Check and update the region quota for this table namespace.
-      final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+      final MasterStorage ms = env.getMasterServices().getMasterStorage();
       SnapshotManifest manifest = SnapshotManifest.open(
         env.getMasterConfiguration(),
-        mfs.getFileSystem(),
-        SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()),
-        snapshot);
+        ms.getFileSystem(),
+        SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, ((LegacyPathIdentifier) ms
+            .getRootContainer()).path), snapshot);
       int snapshotRegionCount = manifest.getRegionManifestsMap().size();
       int tableRegionCount =
           ProcedureSyncWait.getMasterQuotaManager(env).getRegionCountOfTable(tableName);
@@ -365,9 +366,9 @@ public class RestoreSnapshotProcedure
    * @throws IOException
    **/
   private void restoreSnapshot(final MasterProcedureEnv env) throws IOException {
-    MasterFileSystem fileSystemManager = env.getMasterServices().getMasterFileSystem();
+    MasterStorage fileSystemManager = env.getMasterServices().getMasterStorage();
     FileSystem fs = fileSystemManager.getFileSystem();
-    Path rootDir = fileSystemManager.getRootDir();
+    Path rootDir = ((LegacyPathIdentifier) fileSystemManager.getRootContainer()).path;
     final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher();
 
     LOG.info("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot));

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
index 40f5845..6e14f47 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
@@ -90,7 +90,7 @@ public final class MasterSnapshotVerifier {
    * @param rootDir root directory of the hbase installation.
    */
   public MasterSnapshotVerifier(MasterServices services, SnapshotDescription snapshot, Path rootDir) {
-    this.fs = services.getMasterFileSystem().getFileSystem();
+    this.fs = services.getMasterStorage().getFileSystem();
     this.services = services;
     this.snapshot = snapshot;
     this.rootDir = rootDir;

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index 3a5b974..a2c0ea0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -27,10 +27,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -50,8 +47,9 @@ import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.client.TableState;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.ipc.RpcServer;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.MetricsMaster;
@@ -88,7 +86,6 @@ import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.KeyLocker;
-import org.apache.hadoop.hbase.wal.WAL;
 import org.apache.zookeeper.KeeperException;
 
 /**
@@ -188,8 +185,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
       throws IOException, UnsupportedOperationException {
     this.master = master;
 
-    this.rootDir = master.getMasterFileSystem().getRootDir();
-    checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
+    this.rootDir = ((LegacyPathIdentifier) master.getMasterStorage().getRootContainer()).path;
+    checkSnapshotSupport(master.getConfiguration(), master.getMasterStorage());
 
     this.coordinator = coordinator;
     this.executorService = pool;
@@ -214,7 +211,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
   private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
     List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
     // first create the snapshot root path and check to see if it exists
-    FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    FileSystem fs = master.getMasterStorage().getFileSystem();
     if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
 
     // if there are no snapshots, return an empty list
@@ -274,8 +271,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
   void resetTempDir() throws IOException {
     // cleanup any existing snapshots.
     Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
-    if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) {
-      if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
+    if (master.getMasterStorage().getFileSystem().exists(tmpdir)) {
+      if (!master.getMasterStorage().getFileSystem().delete(tmpdir, true)) {
         LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
       }
     }
@@ -295,7 +292,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
 
     String snapshotName = snapshot.getName();
     // first create the snapshot description and check to see if it exists
-    FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    FileSystem fs = master.getMasterStorage().getFileSystem();
     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
     // Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
     // just the "name" and it does not contains the "real" snapshot information
@@ -428,7 +425,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
    */
   private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
       throws HBaseSnapshotException {
-    FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    FileSystem fs = master.getMasterStorage().getFileSystem();
     Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
     TableName snapshotTable =
         TableName.valueOf(snapshot.getTable());
@@ -524,7 +521,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
       // cleanup the working directory by trying to delete it from the fs.
       Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
       try {
-        if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
+        if (!this.master.getMasterStorage().getFileSystem().delete(workingDir, true)) {
           LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
               ClientSnapshotDescriptionUtils.toString(snapshot));
         }
@@ -662,7 +659,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
   private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
     try {
       final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
-      FileSystem fs = master.getMasterFileSystem().getFileSystem();
+      FileSystem fs = master.getMasterStorage().getFileSystem();
       // check to see if the snapshot already exists
       return fs.exists(snapshotDir);
     } catch (IllegalArgumentException iae) {
@@ -765,7 +762,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
       SnapshotDescription reqSnapshot,
       final long nonceGroup,
       final long nonce) throws IOException {
-    FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    FileSystem fs = master.getMasterStorage().getFileSystem();
     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
 
     // check if the snapshot exists
@@ -1045,12 +1042,12 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
    * starting the master if there're snapshots present but the cleaners needed are missing.
    * Otherwise we can end up with snapshot data loss.
    * @param conf The {@link Configuration} object to use
-   * @param mfs The MasterFileSystem to use
+   * @param ms The MasterFileSystem to use
    * @throws IOException in case of file-system operation failure
    * @throws UnsupportedOperationException in case cleaners are missing and
    *         there're snapshot in the system
    */
-  private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
+  private void checkSnapshotSupport(final Configuration conf, final MasterStorage ms)
       throws IOException, UnsupportedOperationException {
     // Verify if snapshot is disabled by the user
     String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
@@ -1067,8 +1064,9 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
     if (cleaners != null) Collections.addAll(logCleaners, cleaners);
 
     // check if an older version of snapshot directory was present
-    Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
-    FileSystem fs = mfs.getFileSystem();
+    Path oldSnapshotDir = new Path(((LegacyPathIdentifier) ms.getRootContainer()).path, HConstants
+        .OLD_SNAPSHOT_DIR_NAME);
+    FileSystem fs = ms.getFileSystem();
     List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
     if (ss != null && !ss.isEmpty()) {
       LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
@@ -1109,7 +1107,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
     // otherwise we end up with snapshot data loss.
     if (!snapshotEnabled) {
       LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
-      Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
+      Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(((LegacyPathIdentifier) ms
+          .getRootContainer()).path);
       if (fs.exists(snapshotDir)) {
         FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
           new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
@@ -1126,8 +1125,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
       IOException, UnsupportedOperationException {
     this.master = master;
 
-    this.rootDir = master.getMasterFileSystem().getRootDir();
-    checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
+    this.rootDir = ((LegacyPathIdentifier) master.getMasterStorage().getRootContainer()).path;
+    checkSnapshotSupport(master.getConfiguration(), master.getMasterStorage());
 
     // get the configuration for the coordinator
     Configuration conf = master.getConfiguration();

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index a0e5b93..503f346 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.executor.EventType;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.MetricsSnapshot;
 import org.apache.hadoop.hbase.master.SnapshotSentinel;
@@ -107,8 +108,8 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
     this.snapshotManager = snapshotManager;
     this.snapshotTable = TableName.valueOf(snapshot.getTable());
     this.conf = this.master.getConfiguration();
-    this.fs = this.master.getMasterFileSystem().getFileSystem();
-    this.rootDir = this.master.getMasterFileSystem().getRootDir();
+    this.fs = this.master.getMasterStorage().getFileSystem();
+    this.rootDir = ((LegacyPathIdentifier) this.master.getMasterStorage().getRootContainer()).path;
     this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
     this.workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
     this.monitor = new ForeignExceptionDispatcher(snapshot.getName());

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index ebbe5df..7f03435 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -31,15 +31,14 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Coprocessor;
-import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.TableInfoMissingException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
 
 /**
  * Implementation of {@link TableDescriptors} that reads descriptors from the
@@ -63,7 +62,7 @@ import org.apache.hadoop.hbase.fs.MasterFileSystem;
 public class FSTableDescriptors implements TableDescriptors {
   private static final Log LOG = LogFactory.getLog(FSTableDescriptors.class);
 
-  private final MasterFileSystem mfs;
+  private final MasterStorage<? extends StorageIdentifier> ms;
   private final boolean fsreadonly;
 
   private volatile boolean usecache;
@@ -103,17 +102,18 @@ public class FSTableDescriptors implements TableDescriptors {
    */
   public FSTableDescriptors(final Configuration conf, final FileSystem fs,
       final Path rootdir, final boolean fsreadonly, final boolean usecache) throws IOException {
-    this(MasterFileSystem.open(conf, fs, rootdir, false), fsreadonly, usecache);
+    this(MasterStorage.open(conf, rootdir, false), fsreadonly, usecache);
   }
 
-  private FSTableDescriptors(final MasterFileSystem mfs, boolean fsreadonly, boolean usecache)
+  private FSTableDescriptors(final MasterStorage<? extends StorageIdentifier> ms, boolean
+      fsreadonly, boolean usecache)
       throws IOException {
     super();
-    this.mfs = mfs;
+    this.ms = ms;
     this.fsreadonly = fsreadonly;
     this.usecache = usecache;
 
-    this.metaTableDescriptor = HTableDescriptor.metaTableDescriptor(mfs.getConfiguration());
+    this.metaTableDescriptor = HTableDescriptor.metaTableDescriptor(ms.getConfiguration());
   }
 
   @Override
@@ -164,7 +164,7 @@ public class FSTableDescriptors implements TableDescriptors {
     }
     HTableDescriptor tdmt = null;
     try {
-      tdmt = mfs.getTableDescriptor(tablename);
+      tdmt = ms.getTableDescriptor(tablename);
     } catch (NullPointerException e) {
       LOG.debug("Exception during readTableDecriptor. Current table name = " + tablename, e);
     } catch (TableInfoMissingException e) {
@@ -198,7 +198,7 @@ public class FSTableDescriptors implements TableDescriptors {
       LOG.debug("Fetching table descriptors from the filesystem.");
       boolean allvisited = true;
 
-      for (TableName table: mfs.getTables()) {
+      for (TableName table: ms.getTables()) {
         HTableDescriptor htd = null;
         try {
           htd = get(table);
@@ -239,7 +239,7 @@ public class FSTableDescriptors implements TableDescriptors {
   @Override
   public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
     Map<String, HTableDescriptor> htds = new TreeMap<String, HTableDescriptor>();
-    for (TableName table: mfs.getTables(name)) {
+    for (TableName table: ms.getTables(name)) {
       HTableDescriptor htd = null;
       try {
         htd = get(table);
@@ -284,7 +284,7 @@ public class FSTableDescriptors implements TableDescriptors {
     if (fsreadonly) {
       throw new NotImplementedException("Cannot remove a table descriptor - in read only mode");
     }
-    mfs.deleteTable(tablename); // for test only??
+    ms.deleteTable(tablename); // for test only??
     HTableDescriptor descriptor = this.cache.remove(tablename);
     return descriptor;
   }
@@ -299,7 +299,7 @@ public class FSTableDescriptors implements TableDescriptors {
     if (fsreadonly) {
       throw new NotImplementedException("Cannot update a table descriptor - in read only mode");
     }
-    mfs.updateTableDescriptor(td);
+    ms.updateTableDescriptor(td);
     if (usecache) {
       this.cache.put(td.getTableName(), td);
     }
@@ -328,6 +328,6 @@ public class FSTableDescriptors implements TableDescriptors {
     }
 
     // forceCreation???
-    return mfs.createTableDescriptor(htd, forceCreation);
+    return ms.createTableDescriptor(htd, forceCreation);
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 662f20d..0eacc2b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -40,13 +40,8 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Vector;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.FutureTask;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
@@ -73,13 +68,12 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.fs.RegionStorage;
-import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
-import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.FSProtos;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -93,7 +87,6 @@ import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 
-import com.google.common.base.Throwables;
 import com.google.common.collect.Iterators;
 import com.google.common.primitives.Ints;
 
@@ -1088,12 +1081,12 @@ public abstract class FSUtils {
     int cfCountTotal = 0;
     int cfFragTotal = 0;
 
-    MasterFileSystem mfs = master.getMasterFileSystem();
-    for (TableName table: mfs.getTables()) {
+    MasterStorage<? extends StorageIdentifier> ms = master.getMasterStorage();
+    for (TableName table: ms.getTables()) {
       int cfCount = 0;
       int cfFrag = 0;
-      for (HRegionInfo hri: mfs.getRegions(table)) {
-        RegionStorage rfs = mfs.getRegionStorage(hri);
+      for (HRegionInfo hri: ms.getRegions(table)) {
+        RegionStorage rfs = ms.getRegionStorage(hri);
         final Collection<String> families = rfs.getFamilies();
         for (String family: families) {
           cfCount++;
@@ -1782,14 +1775,14 @@ public abstract class FSUtils {
       throws IOException {
     long startTime = EnvironmentEdgeManager.currentTime();
 
-    MasterFileSystem mfs = MasterFileSystem.open(conf, false);
+    MasterStorage<? extends StorageIdentifier> ms = MasterStorage.open(conf, false);
     Collection<HRegionInfo> hris;
     if (desiredTable != null) {
-      hris = mfs.getRegions(TableName.valueOf(desiredTable));
+      hris = ms.getRegions(TableName.valueOf(desiredTable));
     } else {
       hris = new ArrayList<HRegionInfo>();
-      for (TableName tableName: mfs.getTables()) {
-        hris.addAll(mfs.getRegions(tableName));
+      for (TableName tableName: ms.getTables()) {
+        hris.addAll(ms.getRegions(tableName));
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 8a14cca..45c9989 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -110,7 +110,7 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
 import org.apache.hadoop.hbase.fs.StorageIdentifier;
@@ -118,7 +118,6 @@ import org.apache.hadoop.hbase.fs.legacy.LegacyLayout;
 import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
 import org.apache.hadoop.hbase.fs.legacy.LegacyRegionStorage;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.master.RegionState;
@@ -130,7 +129,6 @@ import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
 import org.apache.hadoop.hbase.security.UserProvider;
-import org.apache.hadoop.hbase.util.MetaUtils;
 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
@@ -240,7 +238,7 @@ public class HBaseFsck extends Configured implements Closeable {
   // successful
   private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
 
-  private final MasterFileSystem mfs;
+  private final MasterStorage ms;
 
   /***********
    * Options
@@ -361,7 +359,7 @@ public class HBaseFsck extends Configured implements Closeable {
     // Disable usage of meta replicas in hbck
     getConf().setBoolean(HConstants.USE_META_REPLICAS, false);
 
-    mfs = MasterFileSystem.open(getConf(), false);
+    ms = MasterStorage.open(getConf(), false);
 
     errors = getErrorReporter(getConf());
     this.executor = exec;
@@ -1095,14 +1093,14 @@ public class HBaseFsck extends Configured implements Closeable {
   private void offlineReferenceFileRepair() throws IOException {
     clearState();
     LOG.info("Validating mapping using HDFS state");
-    mfs.visitStoreFiles(new StoreFileVisitor() {
+    ms.visitStoreFiles(new StoreFileVisitor() {
       @Override
       public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
           throws IOException {
         if (errors != null) errors.progress();
         if (!storeFile.isReference()) return;
 
-        FileSystem fs = mfs.getFileSystem();
+        FileSystem fs = ms.getFileSystem();
         Path path = storeFile.getPath();
         Path referredToFile = StoreFileInfo.getReferredToFile(path);
         if (fs.exists(referredToFile)) return;  // good, expected

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java
index 4abf350..c3effc1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java
@@ -26,10 +26,9 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -158,7 +157,7 @@ public class TestHColumnDescriptorDefaultVersions {
     verifyHColumnDescriptor(expected, hcds, tableName, families);
 
     // Verify descriptor from HDFS
-    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    MasterStorage mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterStorage();
     Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
     HTableDescriptor td = LegacyTableDescriptor.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir);
     hcds = td.getColumnFamilies();

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
index 2dca6b1..baaa14b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
@@ -230,7 +230,7 @@ public class TestNamespace {
     TEST_UTIL.waitTableAvailable(desc.getTableName().getName(), 10000);
     FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
     assertTrue(fs.exists(
-        new Path(master.getMasterFileSystem().getRootDir(),
+        new Path(master.getMasterStorage().getRootDir(),
             new Path(HConstants.BASE_NAMESPACE_DIR,
                 new Path(nsName, desc.getTableName().getQualifierAsString())))));
     assertEquals(1, admin.listTables().length);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index f5f380f..9b4206c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -26,7 +26,6 @@ import java.util.HashSet;
 import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.CategoryBasedTimeout;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -34,7 +33,7 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
@@ -44,7 +43,6 @@ import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.testclassification.ClientTests;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -304,7 +302,7 @@ public class TestRestoreSnapshotFromClient {
   }
 
   private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
-    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    MasterStorage mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterStorage();
     final Set<String> families = new HashSet<String>();
     mfs.visitStoreFiles(tableName, new StoreFileVisitor() {
       @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
index 2d4b4c9..565da24 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.testclassification.ClientTests;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
 import org.junit.After;
@@ -121,8 +120,8 @@ public class TestSnapshotCloneIndependence {
 
   @Before
   public void setup() throws Exception {
-    fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
-    rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    fs = UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
+    rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
 
     admin = UTIL.getHBaseAdmin();
     originalTableName = TableName.valueOf("test" + testName.getMethodName());

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
index fbd347a..702b80a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
@@ -221,10 +221,10 @@ public class TestSnapshotFromClient {
         SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
 
     // make sure its a valid snapshot
-    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
-    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
+    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
     LOG.debug("FS state after snapshot:");
-    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+    UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
 
     SnapshotTestingUtils.confirmSnapshotValid(
       ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM,
@@ -291,10 +291,10 @@ public class TestSnapshotFromClient {
         SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
 
     // make sure its a valid snapshot
-    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
-    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
+    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
     LOG.debug("FS state after snapshot:");
-    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+    UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
 
     List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
     List<byte[]> nonEmptyCfs = Lists.newArrayList();

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java
index 1a774e5..e043290 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java
@@ -98,8 +98,8 @@ public class TestSnapshotMetadata {
     setupConf(UTIL.getConfiguration());
     UTIL.startMiniCluster(NUM_RS);
 
-    fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
-    rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    fs = UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
+    rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
   }
 
   @AfterClass

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
index 8b9428f..b357066 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
@@ -58,7 +58,7 @@ public class TestTableSnapshotScanner {
   public void setupCluster() throws Exception {
     setupConf(UTIL.getConfiguration());
     UTIL.startMiniCluster(NUM_REGION_SERVERS, true);
-    rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
     fs = rootDir.getFileSystem(UTIL.getConfiguration());
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
index 3180c50..8b869a5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
@@ -280,7 +280,7 @@ public class TestBlockReorder {
             "/" + targetRs.getServerName().toString()).toUri().getPath();
 
     DistributedFileSystem mdfs = (DistributedFileSystem)
-        hbm.getMaster().getMasterFileSystem().getFileSystem();
+        hbm.getMaster().getMasterStorage().getFileSystem();
 
 
     int nbTest = 0;

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
index 70c879a..30bc3e9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.hbase.mapreduce;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.CategoryBasedTimeout;
@@ -63,7 +62,7 @@ public abstract class TableSnapshotInputFormatTestBase {
   public void setupCluster() throws Exception {
     setupConf(UTIL.getConfiguration());
     UTIL.startMiniCluster(NUM_REGION_SERVERS, true);
-    rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
     fs = rootDir.getFileSystem(UTIL.getConfiguration());
   }
 
@@ -127,7 +126,7 @@ public abstract class TableSnapshotInputFormatTestBase {
 
       testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir);
 
-      UTIL.getHBaseCluster().getMaster().getMasterFileSystem().visitStoreFiles(tableName,
+      UTIL.getHBaseCluster().getMaster().getMasterStorage().visitStoreFiles(tableName,
           new StoreFileVisitor() {
         @Override
         public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
index c17d408..d109907 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
@@ -108,7 +108,7 @@ public class TestWALPlayer {
     // replay the WAL, map table 1 to table 2
     WAL log = cluster.getRegionServer(0).getWAL(null);
     log.rollWriter();
-    String walInputDir = new Path(cluster.getMaster().getMasterFileSystem()
+    String walInputDir = new Path(cluster.getMaster().getMasterStorage()
         .getRootDir(), HConstants.HREGION_LOGDIR_NAME).toString();
 
     Configuration configuration= TEST_UTIL.getConfiguration();

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index 56a8522..ee232d5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -103,7 +103,7 @@ public class MockNoopMasterServices implements MasterServices, Server {
   }
 
   @Override
-  public MasterFileSystem getMasterFileSystem() {
+  public MasterFileSystem getMasterStorage() {
     return null;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 4348d2b..fa8a74f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
 import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
 import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails;
 import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
@@ -90,7 +90,7 @@ public class TestCatalogJanitor {
    */
   class MockMasterServices extends MockNoopMasterServices {
     private final ClusterConnection connection;
-    private final MasterFileSystem mfs;
+    private final MasterStorage mfs;
     private final AssignmentManager asm;
     private final ServerManager sm;
 
@@ -134,7 +134,7 @@ public class TestCatalogJanitor {
       FSUtils.setRootDir(getConfiguration(), rootdir);
       Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
 
-      this.mfs = new MasterFileSystem(this);
+      this.mfs = new MasterStorage(this);
       this.asm = Mockito.mock(AssignmentManager.class);
       this.sm = Mockito.mock(ServerManager.class);
     }
@@ -145,7 +145,7 @@ public class TestCatalogJanitor {
     }
 
     @Override
-    public MasterFileSystem getMasterFileSystem() {
+    public MasterStorage getMasterStorage() {
       return this.mfs;
     }
 
@@ -244,7 +244,7 @@ public class TestCatalogJanitor {
       // remove the parent.
       Result r = createResult(parent, splita, splitb);
       // Add a reference under splitA directory so we don't clear out the parent.
-      Path rootdir = services.getMasterFileSystem().getRootDir();
+      Path rootdir = services.getMasterStorage().getRootDir();
       Path tabledir =
         FSUtils.getTableDir(rootdir, htd.getTableName());
       Path storedir = HStore.getStoreHomedir(tabledir, splita,
@@ -253,7 +253,7 @@ public class TestCatalogJanitor {
       long now = System.currentTimeMillis();
       // Reference name has this format: StoreFile#REF_NAME_PARSER
       Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
-      FileSystem fs = services.getMasterFileSystem().getFileSystem();
+      FileSystem fs = services.getMasterStorage().getFileSystem();
       Path path = ref.write(fs, p);
       assertTrue(fs.exists(path));
       assertFalse(janitor.cleanParent(parent, r));
@@ -580,7 +580,7 @@ public class TestCatalogJanitor {
     // remove the parent.
     Result parentMetaRow = createResult(parent, splita, splitb);
     FileSystem fs = FileSystem.get(htu.getConfiguration());
-    Path rootdir = services.getMasterFileSystem().getRootDir();
+    Path rootdir = services.getMasterStorage().getRootDir();
     // have to set the root directory since we use it in HFileDisposer to figure out to get to the
     // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
     // the single test passes, but when the full suite is run, things get borked).
@@ -663,7 +663,7 @@ public class TestCatalogJanitor {
 
     FileSystem fs = FileSystem.get(htu.getConfiguration());
 
-    Path rootdir = services.getMasterFileSystem().getRootDir();
+    Path rootdir = services.getMasterStorage().getRootDir();
     // have to set the root directory since we use it in HFileDisposer to figure out to get to the
     // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
     // the single test passes, but when the full suite is run, things get borked).
@@ -708,7 +708,7 @@ public class TestCatalogJanitor {
   private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
       throws IOException {
     // get the existing store files
-    FileSystem fs = services.getMasterFileSystem().getFileSystem();
+    FileSystem fs = services.getMasterStorage().getFileSystem();
     fs.mkdirs(storedir);
     // create the store files in the parent
     for (int i = 0; i < count; i++) {
@@ -748,7 +748,7 @@ public class TestCatalogJanitor {
       final HTableDescriptor htd, final HRegionInfo parent,
       final HRegionInfo daughter, final byte [] midkey, final boolean top)
   throws IOException {
-    Path rootdir = services.getMasterFileSystem().getRootDir();
+    Path rootdir = services.getMasterStorage().getRootDir();
     Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
     Path storedir = HStore.getStoreHomedir(tabledir, daughter,
       htd.getColumnFamilies()[0].getName());
@@ -757,7 +757,7 @@ public class TestCatalogJanitor {
     long now = System.currentTimeMillis();
     // Reference name has this format: StoreFile#REF_NAME_PARSER
     Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
-    FileSystem fs = services.getMasterFileSystem().getFileSystem();
+    FileSystem fs = services.getMasterStorage().getFileSystem();
     ref.write(fs, p);
     return p;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 002438a..1583f33 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -101,7 +101,6 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
-import org.apache.hadoop.hbase.wal.FSHLogProvider;
 import org.apache.hadoop.hbase.wal.WAL;
 import org.apache.hadoop.hbase.wal.WALFactory;
 import org.apache.hadoop.hbase.wal.WALSplitter;
@@ -217,7 +216,7 @@ public class TestDistributedLogSplitting {
     // turn off load balancing to prevent regions from moving around otherwise
     // they will consume recovered.edits
     master.balanceSwitch(false);
-    FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    FileSystem fs = master.getMasterStorage().getFileSystem();
 
     List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
 
@@ -867,7 +866,7 @@ public class TestDistributedLogSplitting {
       });
 
       int count = 0;
-      FileSystem fs = master.getMasterFileSystem().getFileSystem();
+      FileSystem fs = master.getMasterStorage().getFileSystem();
       Path rootdir = FSUtils.getRootDir(conf);
       Path tdir = FSUtils.getTableDir(rootdir, TableName.valueOf("disableTable"));
       for (HRegionInfo hri : regions) {
@@ -1005,7 +1004,7 @@ public class TestDistributedLogSplitting {
     startCluster(3);
     final int NUM_LOG_LINES = 10000;
     final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager();
-    FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    FileSystem fs = master.getMasterStorage().getFileSystem();
 
     final List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
     HRegionServer hrs = findRSToKill(false, "table");
@@ -1124,7 +1123,7 @@ public class TestDistributedLogSplitting {
     LOG.info("testDelayedDeleteOnFailure");
     startCluster(1);
     final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager();
-    final FileSystem fs = master.getMasterFileSystem().getFileSystem();
+    final FileSystem fs = master.getMasterStorage().getFileSystem();
     final Path logDir = new Path(FSUtils.getRootDir(conf), "x");
     fs.mkdirs(logDir);
     ExecutorService executor = null;
@@ -1442,7 +1441,7 @@ public class TestDistributedLogSplitting {
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     Table ht = installTable(zkw, "table", "family", 10);
     try {
-      FileSystem fs = master.getMasterFileSystem().getFileSystem();
+      FileSystem fs = master.getMasterStorage().getFileSystem();
       Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), TableName.valueOf("table"));
       List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir);
       long newSeqId = WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0), 1L, 1000L);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
index 48e143d..3a692d9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
@@ -26,8 +26,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -58,7 +57,7 @@ public class TestMasterFileSystem {
   @Test
   public void testFsUriSetProperly() throws Exception {
     HMaster master = UTIL.getMiniHBaseCluster().getMaster();
-    MasterFileSystem fs = master.getMasterFileSystem();
+    MasterStorage fs = master.getMasterStorage();
     Path masterRoot = FSUtils.getRootDir(fs.getConfiguration());
     Path rootDir = FSUtils.getRootDir(fs.getFileSystem().getConf());
     // make sure the fs and the found root dir have the same scheme

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
index 650171d..6725422 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
@@ -102,7 +102,7 @@ public class TestSnapshotFromMaster {
     UTIL.startMiniCluster(NUM_RS);
     fs = UTIL.getDFSCluster().getFileSystem();
     master = UTIL.getMiniHBaseCluster().getMaster();
-    rootDir = master.getMasterFileSystem().getRootDir();
+    rootDir = master.getMasterStorage().getRootDir();
     archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index a6a882d..245aa88 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -124,8 +124,8 @@ public class MasterProcedureTestingUtility {
   public static void validateTableCreation(final HMaster master, final TableName tableName,
       final HRegionInfo[] regions, boolean hasFamilyDirs, String... family) throws IOException {
     // check filesystem
-    final FileSystem fs = master.getMasterFileSystem().getFileSystem();
-    final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
+    final FileSystem fs = master.getMasterStorage().getFileSystem();
+    final Path tableDir = FSUtils.getTableDir(master.getMasterStorage().getRootDir(), tableName);
     assertTrue(fs.exists(tableDir));
     FSUtils.logFileSystemState(fs, tableDir, LOG);
     List<Path> allRegionDirs = FSUtils.getRegionDirs(fs, tableDir);
@@ -167,8 +167,8 @@ public class MasterProcedureTestingUtility {
   public static void validateTableDeletion(
       final HMaster master, final TableName tableName) throws IOException {
     // check filesystem
-    final FileSystem fs = master.getMasterFileSystem().getFileSystem();
-    final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
+    final FileSystem fs = master.getMasterStorage().getFileSystem();
+    final Path tableDir = FSUtils.getTableDir(master.getMasterStorage().getRootDir(), tableName);
     assertFalse(fs.exists(tableDir));
 
     // check meta
@@ -243,7 +243,7 @@ public class MasterProcedureTestingUtility {
     assertFalse(htd.hasFamily(deletedFamily.getBytes()));
 
     // verify fs
-    master.getMasterFileSystem().visitStoreFiles(tableName, new StoreFileVisitor() {
+    master.getMasterStorage().visitStoreFiles(tableName, new StoreFileVisitor() {
       @Override
       public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
           throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
index e949fc5..0e54151 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
@@ -326,6 +326,6 @@ public class TestMasterFailoverWithProcedures {
   }
 
   private Path getRootDir() {
-    return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    return UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureWalLease.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureWalLease.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureWalLease.java
index 408da81..ac85920 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureWalLease.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureWalLease.java
@@ -108,7 +108,7 @@ public class TestMasterProcedureWalLease {
     Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration();
     Mockito.doReturn(true).when(backupMaster3).isActiveMaster();
     final WALProcedureStore backupStore3 = new WALProcedureStore(firstMaster.getConfiguration(),
-        firstMaster.getMasterFileSystem().getFileSystem(),
+        firstMaster.getMasterStorage().getFileSystem(),
         ((WALProcedureStore)masterStore).getLogDir(),
         new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3));
     // Abort Latch for the test store
@@ -188,7 +188,7 @@ public class TestMasterProcedureWalLease {
     Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration();
     Mockito.doReturn(true).when(backupMaster3).isActiveMaster();
     final WALProcedureStore procStore2 = new WALProcedureStore(firstMaster.getConfiguration(),
-        firstMaster.getMasterFileSystem().getFileSystem(),
+        firstMaster.getMasterStorage().getFileSystem(),
         ((WALProcedureStore)procStore).getLogDir(),
         new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3));
 
@@ -227,4 +227,4 @@ public class TestMasterProcedureWalLease {
   private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
     return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java
index c143b9a..78e6e7d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java
@@ -32,12 +32,11 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -273,7 +272,7 @@ public class TestTableDescriptorModificationFromClient {
     verifyTableDescriptor(htd, tableName, families);
 
     // Verify descriptor from HDFS
-    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    MasterStorage mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterStorage();
     Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
     HTableDescriptor td =
         LegacyTableDescriptor.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
index 035b17b..817a2d2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.MetricsMaster;
@@ -54,7 +54,7 @@ public class TestSnapshotManager {
   MetricsMaster metrics = Mockito.mock(MetricsMaster.class);
   ProcedureCoordinator coordinator = Mockito.mock(ProcedureCoordinator.class);
   ExecutorService pool = Mockito.mock(ExecutorService.class);
-  MasterFileSystem mfs = Mockito.mock(MasterFileSystem.class);
+  MasterStorage mfs = Mockito.mock(MasterStorage.class);
   FileSystem fs;
   {
     try {
@@ -72,7 +72,7 @@ public class TestSnapshotManager {
       throws IOException, KeeperException {
     Mockito.reset(services);
     Mockito.when(services.getConfiguration()).thenReturn(conf);
-    Mockito.when(services.getMasterFileSystem()).thenReturn(mfs);
+    Mockito.when(services.getMasterStorage()).thenReturn(mfs);
     Mockito.when(mfs.getFileSystem()).thenReturn(fs);
     Mockito.when(mfs.getRootDir()).thenReturn(UTIL.getDataTestDir());
     return new SnapshotManager(services, metrics, coordinator, pool);

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
index 49a290c..aa1c7ef 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
@@ -63,7 +63,7 @@ import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.RegionState;
@@ -81,7 +81,6 @@ import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.zookeeper.KeeperException;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -189,7 +188,7 @@ public class TestNamespaceAuditor {
   @Test
   public void testValidQuotas() throws Exception {
     boolean exceptionCaught = false;
-    MasterFileSystem mfs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem();
+    MasterStorage mfs = UTIL.getHBaseCluster().getMaster().getMasterStorage();
     NamespaceDescriptor nspDesc =
         NamespaceDescriptor.create(prefix + "vq1")
             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
index 6c851de..97238cf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
@@ -60,7 +60,7 @@ public class TestCompactSplitThread {
     setupConf(TEST_UTIL.getConfiguration());
     TEST_UTIL.startMiniCluster(NUM_RS);
     fs = TEST_UTIL.getDFSCluster().getFileSystem();
-    rootDir = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    rootDir = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterStorage().getRootDir();
 
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
index 140b038..331ef7b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
@@ -35,15 +35,12 @@ import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.fs.FsContext;
+import org.apache.hadoop.hbase.fs.StorageContext;
 import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.TestTableName;
 
 import org.junit.After;
@@ -106,8 +103,8 @@ public class TestCorruptedRegionStoreFile {
 
     // get the store file paths
     storeFiles.clear();
-    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().visitStoreFiles(
-        FsContext.DATA, tableName, new StoreFileVisitor() {
+    UTIL.getHBaseCluster().getMaster().getMasterStorage().visitStoreFiles(
+        StorageContext.DATA, tableName, new StoreFileVisitor() {
       public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
           throws IOException {
         storeFiles.add(storeFile.getPath());
@@ -190,11 +187,11 @@ public class TestCorruptedRegionStoreFile {
   //  Helpers
   // ==========================================================================
   private FileSystem getFileSystem() {
-    return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+    return UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
   }
 
   private Path getRootDir() {
-    return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    return UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
   }
 
   private void evictHFileCache(final Path hfile) throws Exception {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
index ee3d7d6..f824517 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
@@ -239,8 +239,8 @@ public class TestRegionMergeTransactionOnCluster {
       PairOfSameType<HRegionInfo> p = MetaTableAccessor.getMergeRegions(mergedRegionResult);
       HRegionInfo regionA = p.getFirst();
       HRegionInfo regionB = p.getSecond();
-      FileSystem fs = MASTER.getMasterFileSystem().getFileSystem();
-      Path rootDir = MASTER.getMasterFileSystem().getRootDir();
+      FileSystem fs = MASTER.getMasterStorage().getFileSystem();
+      Path rootDir = MASTER.getMasterStorage().getRootDir();
 
       Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
       Path regionAdir = new Path(tabledir, regionA.getEncodedName());

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerRetriableFailure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerRetriableFailure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerRetriableFailure.java
index c89860c..508b5dc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerRetriableFailure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerRetriableFailure.java
@@ -120,11 +120,11 @@ public class TestScannerRetriableFailure {
   //  Helpers
   // ==========================================================================
   private FileSystem getFileSystem() {
-    return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+    return UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
   }
 
   private Path getRootDir() {
-    return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    return UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
   }
 
   public void loadTable(final Table table, int numRows) throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index c980ae6..165acd0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -31,7 +31,6 @@ import java.io.InterruptedIOException;
 import java.util.Collection;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -39,7 +38,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.CoordinatedStateManager;
@@ -57,7 +55,6 @@ import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.UnknownRegionException;
-import org.apache.hadoop.hbase.Waiter;
 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Connection;
@@ -93,8 +90,6 @@ import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.HBaseFsck;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.apache.hadoop.hbase.util.PairOfSameType;
 import org.apache.hadoop.hbase.util.Threads;
@@ -769,7 +764,7 @@ public class TestSplitTransactionOnCluster {
     try {
       // Precondition: we created a table with no data, no store files.
       printOutRegions(regionServer, "Initial regions: ");
-      cluster.getMaster().getMasterFileSystem().logFileSystemState(LOG);
+      cluster.getMaster().getMasterStorage().logStorageState(LOG);
       List<StoreFileInfo> storefiles = getStoreFiles(tableName);
       assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
 
@@ -793,7 +788,7 @@ public class TestSplitTransactionOnCluster {
       assertTrue(daughters.size() == 2);
 
       // check dirs
-      cluster.getMaster().getMasterFileSystem().logFileSystemState(LOG);
+      cluster.getMaster().getMasterStorage().logStorageState(LOG);
       List<StoreFileInfo> storefilesAfter = getStoreFiles(tableName);
       assertEquals("Expected nothing but found " + storefilesAfter.toString(),
           storefilesAfter.size(), 0);
@@ -949,7 +944,7 @@ public class TestSplitTransactionOnCluster {
       SplitTransactionImpl st = new SplitTransactionImpl(regions.get(0), Bytes.toBytes("r3"));
       st.prepare();
       st.stepsBeforePONR(regionServer, regionServer, false);
-      assertEquals(3, cluster.getMaster().getMasterFileSystem().getRegions(desc.getTableName()).size());
+      assertEquals(3, cluster.getMaster().getMasterStorage().getRegions(desc.getTableName()).size());
       cluster.startRegionServer();
       regionServer.kill();
       // Before we check deadServerInProgress, we should ensure server is dead at master side.
@@ -965,7 +960,7 @@ public class TestSplitTransactionOnCluster {
       AssignmentManager am = cluster.getMaster().getAssignmentManager();
       assertEquals(am.getRegionStates().getRegionsInTransition().toString(), 0, am
           .getRegionStates().getRegionsInTransition().size());
-      assertEquals(1, cluster.getMaster().getMasterFileSystem().getRegions(desc.getTableName()).size());
+      assertEquals(1, cluster.getMaster().getMasterStorage().getRegions(desc.getTableName()).size());
     } finally {
       TESTING_UTIL.deleteTable(table);
     }
@@ -1364,7 +1359,7 @@ public class TestSplitTransactionOnCluster {
 
   private List<StoreFileInfo> getStoreFiles(TableName table) throws IOException {
     final ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>();
-    cluster.getMaster().getMasterFileSystem().visitStoreFiles(table, new StoreFileVisitor() {
+    cluster.getMaster().getMasterStorage().visitStoreFiles(table, new StoreFileVisitor() {
       @Override
       public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
           throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9f759626/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index d8ceafb..d824d70 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -55,7 +55,7 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
 import org.apache.hadoop.hbase.fs.RegionStorage;
 import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.io.HFileLink;
@@ -162,7 +162,7 @@ public final class SnapshotTestingUtils {
   public static void confirmSnapshotValid(HBaseTestingUtility testUtil,
       HBaseProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
       throws IOException {
-    MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem();
+    MasterStorage mfs = testUtil.getHBaseCluster().getMaster().getMasterStorage();
     confirmSnapshotValid(snapshotDescriptor, tableName, family,
         mfs.getRootDir(), testUtil.getHBaseAdmin(), mfs.getFileSystem());
   }
@@ -424,7 +424,7 @@ public final class SnapshotTestingUtils {
    */
   public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
       throws IOException {
-    final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
+    final MasterStorage mfs = util.getHBaseCluster().getMaster().getMasterStorage();
     final FileSystem fs = mfs.getFileSystem();
 
     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
@@ -854,7 +854,7 @@ public final class SnapshotTestingUtils {
   public static void deleteArchiveDirectory(final HBaseTestingUtility util)
       throws IOException {
     // Ensure the archiver to be empty
-    MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    MasterStorage mfs = util.getMiniHBaseCluster().getMaster().getMasterStorage();
     Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
     mfs.getFileSystem().delete(archiveDir, true);
   }


[11/18] hbase git commit: HBASE-14439 Move fs stuff out pt2 - region fs

Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiver.java
new file mode 100644
index 0000000..d682ccc
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiver.java
@@ -0,0 +1,690 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.io.MultipleIOException;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Lists;
+
+/**
+ * Utility class to handle the removal of HFiles (or the respective {@link StoreFile StoreFiles})
+ * for a HRegion from the {@link FileSystem}. The hfiles will be archived or deleted, depending on
+ * the state of the system.
+ */
+public class HFileArchiver {
+  private static final Log LOG = LogFactory.getLog(HFileArchiver.class);
+  private static final String SEPARATOR = ".";
+
+  /** Number of retries in case of fs operation failure */
+  private static final int DEFAULT_RETRIES_NUMBER = 3;
+
+  private HFileArchiver() {
+    // hidden ctor since this is just a util
+  }
+
+  /**
+   * Cleans up all the files for a HRegion by archiving the HFiles to the
+   * archive directory
+   * @param conf the configuration to use
+   * @param fs the file system object
+   * @param info HRegionInfo for region to be deleted
+   * @throws IOException
+   */
+  public static void archiveRegion(Configuration conf, FileSystem fs, HRegionInfo info)
+      throws IOException {
+    Path rootDir = FSUtils.getRootDir(conf);
+    archiveRegion(fs, rootDir, FSUtils.getTableDir(rootDir, info.getTable()),
+      HRegion.getRegionDir(rootDir, info));
+  }
+
+  /**
+   * Remove an entire region from the table directory via archiving the region's hfiles.
+   * @param fs {@link FileSystem} from which to remove the region
+   * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
+   *          the archive path)
+   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
+   * @param regionDir {@link Path} to where a region is being stored (for building the archive path)
+   * @return <tt>true</tt> if the region was sucessfully deleted. <tt>false</tt> if the filesystem
+   *         operations could not complete.
+   * @throws IOException if the request cannot be completed
+   */
+  public static boolean archiveRegion(FileSystem fs, Path rootdir, Path tableDir, Path regionDir)
+      throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("ARCHIVING " + regionDir.toString());
+    }
+
+    // otherwise, we archive the files
+    // make sure we can archive
+    if (tableDir == null || regionDir == null) {
+      LOG.error("No archive directory could be found because tabledir (" + tableDir
+          + ") or regiondir (" + regionDir + "was null. Deleting files instead.");
+      deleteRegionWithoutArchiving(fs, regionDir);
+      // we should have archived, but failed to. Doesn't matter if we deleted
+      // the archived files correctly or not.
+      return false;
+    }
+
+    // make sure the regiondir lives under the tabledir
+    Preconditions.checkArgument(regionDir.toString().startsWith(tableDir.toString()));
+    Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(rootdir,
+        FSUtils.getTableName(tableDir),
+        regionDir.getName());
+
+    FileStatusConverter getAsFile = new FileStatusConverter(fs);
+    // otherwise, we attempt to archive the store files
+
+    // build collection of just the store directories to archive
+    Collection<File> toArchive = new ArrayList<File>();
+    final PathFilter dirFilter = new FSUtils.DirFilter(fs);
+    PathFilter nonHidden = new PathFilter() {
+      @Override
+      public boolean accept(Path file) {
+        return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
+      }
+    };
+    FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
+    // if there no files, we can just delete the directory and return;
+    if (storeDirs == null) {
+      LOG.debug("Region directory (" + regionDir + ") was empty, just deleting and returning!");
+      return deleteRegionWithoutArchiving(fs, regionDir);
+    }
+
+    // convert the files in the region to a File
+    toArchive.addAll(Lists.transform(Arrays.asList(storeDirs), getAsFile));
+    LOG.debug("Archiving " + toArchive);
+    boolean success = false;
+    try {
+      success = resolveAndArchive(fs, regionArchiveDir, toArchive);
+    } catch (IOException e) {
+      LOG.error("Failed to archive " + toArchive, e);
+      success = false;
+    }
+
+    // if that was successful, then we delete the region
+    if (success) {
+      return deleteRegionWithoutArchiving(fs, regionDir);
+    }
+
+    throw new IOException("Received error when attempting to archive files (" + toArchive
+        + "), cannot delete region directory. ");
+  }
+
+  /**
+   * Remove from the specified region the store files of the specified column family,
+   * either by archiving them or outright deletion
+   * @param fs the filesystem where the store files live
+   * @param conf {@link Configuration} to examine to determine the archive directory
+   * @param parent Parent region hosting the store files
+   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
+   * @param family the family hosting the store files
+   * @throws IOException if the files could not be correctly disposed.
+   */
+  public static void archiveFamily(FileSystem fs, Configuration conf,
+      HRegionInfo parent, Path tableDir, byte[] family) throws IOException {
+    Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family)));
+    FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir);
+    if (storeFiles == null) {
+      LOG.debug("No store files to dispose for region=" + parent.getRegionNameAsString() +
+          ", family=" + Bytes.toString(family));
+      return;
+    }
+
+    FileStatusConverter getAsFile = new FileStatusConverter(fs);
+    Collection<File> toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile);
+    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, tableDir, family);
+
+    // do the actual archive
+    if (!resolveAndArchive(fs, storeArchiveDir, toArchive)) {
+      throw new IOException("Failed to archive/delete all the files for region:"
+          + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family)
+          + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
+    }
+  }
+
+  /**
+   * Remove the store files, either by archiving them or outright deletion
+   * @param conf {@link Configuration} to examine to determine the archive directory
+   * @param fs the filesystem where the store files live
+   * @param regionInfo {@link HRegionInfo} of the region hosting the store files
+   * @param family the family hosting the store files
+   * @param compactedFiles files to be disposed of. No further reading of these files should be
+   *          attempted; otherwise likely to cause an {@link IOException}
+   * @throws IOException if the files could not be correctly disposed.
+   */
+  public static void archiveStoreFiles(Configuration conf, FileSystem fs, HRegionInfo regionInfo,
+      Path tableDir, byte[] family, Collection<StoreFile> compactedFiles) throws IOException {
+
+    // sometimes in testing, we don't have rss, so we need to check for that
+    if (fs == null) {
+      LOG.warn("Passed filesystem is null, so just deleting the files without archiving for region:"
+          + Bytes.toString(regionInfo.getRegionName()) + ", family:" + Bytes.toString(family));
+      deleteStoreFilesWithoutArchiving(compactedFiles);
+      return;
+    }
+
+    // short circuit if we don't have any files to delete
+    if (compactedFiles.size() == 0) {
+      LOG.debug("No store files to dispose, done!");
+      return;
+    }
+
+    // build the archive path
+    if (regionInfo == null || family == null) throw new IOException(
+        "Need to have a region and a family to archive from.");
+
+    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family);
+
+    // make sure we don't archive if we can't and that the archive dir exists
+    if (!fs.mkdirs(storeArchiveDir)) {
+      throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
+          + Bytes.toString(family) + ", deleting compacted files instead.");
+    }
+
+    // otherwise we attempt to archive the store files
+    if (LOG.isDebugEnabled()) LOG.debug("Archiving compacted store files.");
+
+    // Wrap the storefile into a File
+    StoreToFile getStorePath = new StoreToFile(fs);
+    Collection<File> storeFiles = Collections2.transform(compactedFiles, getStorePath);
+
+    // do the actual archive
+    if (!resolveAndArchive(fs, storeArchiveDir, storeFiles)) {
+      throw new IOException("Failed to archive/delete all the files for region:"
+          + Bytes.toString(regionInfo.getRegionName()) + ", family:" + Bytes.toString(family)
+          + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
+    }
+  }
+
+  /**
+   * Archive the store file
+   * @param fs the filesystem where the store files live
+   * @param regionInfo region hosting the store files
+   * @param conf {@link Configuration} to examine to determine the archive directory
+   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
+   * @param family the family hosting the store files
+   * @param storeFile file to be archived
+   * @throws IOException if the files could not be correctly disposed.
+   */
+  public static void archiveStoreFile(Configuration conf, FileSystem fs, HRegionInfo regionInfo,
+      Path tableDir, byte[] family, Path storeFile) throws IOException {
+    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family);
+    // make sure we don't archive if we can't and that the archive dir exists
+    if (!fs.mkdirs(storeArchiveDir)) {
+      throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
+          + Bytes.toString(family) + ", deleting compacted files instead.");
+    }
+
+    // do the actual archive
+    long start = EnvironmentEdgeManager.currentTime();
+    File file = new FileablePath(fs, storeFile);
+    if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) {
+      throw new IOException("Failed to archive/delete the file for region:"
+          + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family)
+          + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
+    }
+  }
+
+  /**
+   * Archive the given files and resolve any conflicts with existing files via appending the time
+   * archiving started (so all conflicts in the same group have the same timestamp appended).
+   * <p>
+   * If any of the passed files to archive are directories, archives all the files under that
+   * directory. Archive directory structure for children is the base archive directory name + the
+   * parent directory and is built recursively is passed files are directories themselves.
+   * @param fs {@link FileSystem} on which to archive the files
+   * @param baseArchiveDir base archive directory to archive the given files
+   * @param toArchive files to be archived
+   * @return <tt>true</tt> on success, <tt>false</tt> otherwise
+   * @throws IOException on unexpected failure
+   */
+  private static boolean resolveAndArchive(FileSystem fs, Path baseArchiveDir,
+      Collection<File> toArchive) throws IOException {
+    if (LOG.isTraceEnabled()) LOG.trace("Starting to archive " + toArchive);
+    long start = EnvironmentEdgeManager.currentTime();
+    List<File> failures = resolveAndArchive(fs, baseArchiveDir, toArchive, start);
+
+    // notify that some files were not archived.
+    // We can't delete the files otherwise snapshots or other backup system
+    // that relies on the archiver end up with data loss.
+    if (failures.size() > 0) {
+      LOG.warn("Failed to complete archive of: " + failures +
+        ". Those files are still in the original location, and they may slow down reads.");
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Resolve any conflict with an existing archive file via timestamp-append
+   * renaming of the existing file and then archive the passed in files.
+   * @param fs {@link FileSystem} on which to archive the files
+   * @param baseArchiveDir base archive directory to store the files. If any of
+   *          the files to archive are directories, will append the name of the
+   *          directory to the base archive directory name, creating a parallel
+   *          structure.
+   * @param toArchive files/directories that need to be archvied
+   * @param start time the archiving started - used for resolving archive
+   *          conflicts.
+   * @return the list of failed to archive files.
+   * @throws IOException if an unexpected file operation exception occured
+   */
+  private static List<File> resolveAndArchive(FileSystem fs, Path baseArchiveDir,
+      Collection<File> toArchive, long start) throws IOException {
+    // short circuit if no files to move
+    if (toArchive.size() == 0) return Collections.emptyList();
+
+    if (LOG.isTraceEnabled()) LOG.trace("moving files to the archive directory: " + baseArchiveDir);
+
+    // make sure the archive directory exists
+    if (!fs.exists(baseArchiveDir)) {
+      if (!fs.mkdirs(baseArchiveDir)) {
+        throw new IOException("Failed to create the archive directory:" + baseArchiveDir
+            + ", quitting archive attempt.");
+      }
+      if (LOG.isTraceEnabled()) LOG.trace("Created archive directory:" + baseArchiveDir);
+    }
+
+    List<File> failures = new ArrayList<File>();
+    String startTime = Long.toString(start);
+    for (File file : toArchive) {
+      // if its a file archive it
+      try {
+        if (LOG.isTraceEnabled()) LOG.trace("Archiving: " + file);
+        if (file.isFile()) {
+          // attempt to archive the file
+          if (!resolveAndArchiveFile(baseArchiveDir, file, startTime)) {
+            LOG.warn("Couldn't archive " + file + " into backup directory: " + baseArchiveDir);
+            failures.add(file);
+          }
+        } else {
+          // otherwise its a directory and we need to archive all files
+          if (LOG.isTraceEnabled()) LOG.trace(file + " is a directory, archiving children files");
+          // so we add the directory name to the one base archive
+          Path parentArchiveDir = new Path(baseArchiveDir, file.getName());
+          // and then get all the files from that directory and attempt to
+          // archive those too
+          Collection<File> children = file.getChildren();
+          failures.addAll(resolveAndArchive(fs, parentArchiveDir, children, start));
+        }
+      } catch (IOException e) {
+        LOG.warn("Failed to archive " + file, e);
+        failures.add(file);
+      }
+    }
+    return failures;
+  }
+
+  /**
+   * Attempt to archive the passed in file to the archive directory.
+   * <p>
+   * If the same file already exists in the archive, it is moved to a timestamped directory under
+   * the archive directory and the new file is put in its place.
+   * @param archiveDir {@link Path} to the directory that stores the archives of the hfiles
+   * @param currentFile {@link Path} to the original HFile that will be archived
+   * @param archiveStartTime time the archiving started, to resolve naming conflicts
+   * @return <tt>true</tt> if the file is successfully archived. <tt>false</tt> if there was a
+   *         problem, but the operation still completed.
+   * @throws IOException on failure to complete {@link FileSystem} operations.
+   */
+  private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile,
+      String archiveStartTime) throws IOException {
+    // build path as it should be in the archive
+    String filename = currentFile.getName();
+    Path archiveFile = new Path(archiveDir, filename);
+    FileSystem fs = currentFile.getFileSystem();
+
+    // if the file already exists in the archive, move that one to a timestamped backup. This is a
+    // really, really unlikely situtation, where we get the same name for the existing file, but
+    // is included just for that 1 in trillion chance.
+    if (fs.exists(archiveFile)) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("File:" + archiveFile + " already exists in archive, moving to "
+            + "timestamped backup and overwriting current.");
+      }
+
+      // move the archive file to the stamped backup
+      Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime);
+      if (!fs.rename(archiveFile, backedupArchiveFile)) {
+        LOG.error("Could not rename archive file to backup: " + backedupArchiveFile
+            + ", deleting existing file in favor of newer.");
+        // try to delete the exisiting file, if we can't rename it
+        if (!fs.delete(archiveFile, false)) {
+          throw new IOException("Couldn't delete existing archive file (" + archiveFile
+              + ") or rename it to the backup file (" + backedupArchiveFile
+              + ") to make room for similarly named file.");
+        }
+      }
+      LOG.debug("Backed up archive file from " + archiveFile);
+    }
+
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("No existing file in archive for: " + archiveFile +
+        ", free to archive original file.");
+    }
+
+    // at this point, we should have a free spot for the archive file
+    boolean success = false;
+    for (int i = 0; !success && i < DEFAULT_RETRIES_NUMBER; ++i) {
+      if (i > 0) {
+        // Ensure that the archive directory exists.
+        // The previous "move to archive" operation has failed probably because
+        // the cleaner has removed our archive directory (HBASE-7643).
+        // (we're in a retry loop, so don't worry too much about the exception)
+        try {
+          if (!fs.exists(archiveDir)) {
+            if (fs.mkdirs(archiveDir)) {
+              LOG.debug("Created archive directory:" + archiveDir);
+            }
+          }
+        } catch (IOException e) {
+          LOG.warn("Failed to create directory: " + archiveDir, e);
+        }
+      }
+
+      try {
+        success = currentFile.moveAndClose(archiveFile);
+      } catch (IOException e) {
+        LOG.warn("Failed to archive " + currentFile + " on try #" + i, e);
+        success = false;
+      }
+    }
+
+    if (!success) {
+      LOG.error("Failed to archive " + currentFile);
+      return false;
+    }
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Finished archiving from " + currentFile + ", to " + archiveFile);
+    }
+    return true;
+  }
+
+  /**
+   * Without regard for backup, delete a region. Should be used with caution.
+   * @param regionDir {@link Path} to the region to be deleted.
+   * @param fs FileSystem from which to delete the region
+   * @return <tt>true</tt> on successful deletion, <tt>false</tt> otherwise
+   * @throws IOException on filesystem operation failure
+   */
+  private static boolean deleteRegionWithoutArchiving(FileSystem fs, Path regionDir)
+      throws IOException {
+    if (fs.delete(regionDir, true)) {
+      LOG.debug("Deleted all region files in: " + regionDir);
+      return true;
+    }
+    LOG.debug("Failed to delete region directory:" + regionDir);
+    return false;
+  }
+
+  /**
+   * Just do a simple delete of the given store files
+   * <p>
+   * A best effort is made to delete each of the files, rather than bailing on the first failure.
+   * <p>
+   * This method is preferable to {@link #deleteFilesWithoutArchiving(Collection)} since it consumes
+   * less resources, but is limited in terms of usefulness
+   * @param compactedFiles store files to delete from the file system.
+   * @throws IOException if a file cannot be deleted. All files will be attempted to deleted before
+   *           throwing the exception, rather than failing at the first file.
+   */
+  private static void deleteStoreFilesWithoutArchiving(Collection<StoreFile> compactedFiles)
+      throws IOException {
+    LOG.debug("Deleting store files without archiving.");
+    List<IOException> errors = new ArrayList<IOException>(0);
+    for (StoreFile hsf : compactedFiles) {
+      try {
+        hsf.deleteReader();
+      } catch (IOException e) {
+        LOG.error("Failed to delete store file:" + hsf.getPath());
+        errors.add(e);
+      }
+    }
+    if (errors.size() > 0) {
+      throw MultipleIOException.createIOException(errors);
+    }
+  }
+
+  /**
+   * Adapt a type to match the {@link File} interface, which is used internally for handling
+   * archival/removal of files
+   * @param <T> type to adapt to the {@link File} interface
+   */
+  private static abstract class FileConverter<T> implements Function<T, File> {
+    protected final FileSystem fs;
+
+    public FileConverter(FileSystem fs) {
+      this.fs = fs;
+    }
+  }
+
+  /**
+   * Convert a FileStatus to something we can manage in the archiving
+   */
+  private static class FileStatusConverter extends FileConverter<FileStatus> {
+    public FileStatusConverter(FileSystem fs) {
+      super(fs);
+    }
+
+    @Override
+    public File apply(FileStatus input) {
+      return new FileablePath(fs, input.getPath());
+    }
+  }
+
+  /**
+   * Convert the {@link StoreFile} into something we can manage in the archive
+   * methods
+   */
+  private static class StoreToFile extends FileConverter<StoreFile> {
+    public StoreToFile(FileSystem fs) {
+      super(fs);
+    }
+
+    @Override
+    public File apply(StoreFile input) {
+      return new FileableStoreFile(fs, input);
+    }
+  }
+
+  /**
+   * Wrapper to handle file operations uniformly
+   */
+  private static abstract class File {
+    protected final FileSystem fs;
+
+    public File(FileSystem fs) {
+      this.fs = fs;
+    }
+
+    /**
+     * Delete the file
+     * @throws IOException on failure
+     */
+    abstract void delete() throws IOException;
+
+    /**
+     * Check to see if this is a file or a directory
+     * @return <tt>true</tt> if it is a file, <tt>false</tt> otherwise
+     * @throws IOException on {@link FileSystem} connection error
+     */
+    abstract boolean isFile() throws IOException;
+
+    /**
+     * @return if this is a directory, returns all the children in the
+     *         directory, otherwise returns an empty list
+     * @throws IOException
+     */
+    abstract Collection<File> getChildren() throws IOException;
+
+    /**
+     * close any outside readers of the file
+     * @throws IOException
+     */
+    abstract void close() throws IOException;
+
+    /**
+     * @return the name of the file (not the full fs path, just the individual
+     *         file name)
+     */
+    abstract String getName();
+
+    /**
+     * @return the path to this file
+     */
+    abstract Path getPath();
+
+    /**
+     * Move the file to the given destination
+     * @param dest
+     * @return <tt>true</tt> on success
+     * @throws IOException
+     */
+    public boolean moveAndClose(Path dest) throws IOException {
+      this.close();
+      Path p = this.getPath();
+      return FSUtils.renameAndSetModifyTime(fs, p, dest);
+    }
+
+    /**
+     * @return the {@link FileSystem} on which this file resides
+     */
+    public FileSystem getFileSystem() {
+      return this.fs;
+    }
+
+    @Override
+    public String toString() {
+      return this.getClass() + ", file:" + getPath().toString();
+    }
+  }
+
+  /**
+   * A {@link File} that wraps a simple {@link Path} on a {@link FileSystem}.
+   */
+  private static class FileablePath extends File {
+    private final Path file;
+    private final FileStatusConverter getAsFile;
+
+    public FileablePath(FileSystem fs, Path file) {
+      super(fs);
+      this.file = file;
+      this.getAsFile = new FileStatusConverter(fs);
+    }
+
+    @Override
+    public void delete() throws IOException {
+      if (!fs.delete(file, true)) throw new IOException("Failed to delete:" + this.file);
+    }
+
+    @Override
+    public String getName() {
+      return file.getName();
+    }
+
+    @Override
+    public Collection<File> getChildren() throws IOException {
+      if (fs.isFile(file)) return Collections.emptyList();
+      return Collections2.transform(Arrays.asList(fs.listStatus(file)), getAsFile);
+    }
+
+    @Override
+    public boolean isFile() throws IOException {
+      return fs.isFile(file);
+    }
+
+    @Override
+    public void close() throws IOException {
+      // NOOP - files are implicitly closed on removal
+    }
+
+    @Override
+    Path getPath() {
+      return file;
+    }
+  }
+
+  /**
+   * {@link File} adapter for a {@link StoreFile} living on a {@link FileSystem}
+   * .
+   */
+  private static class FileableStoreFile extends File {
+    StoreFile file;
+
+    public FileableStoreFile(FileSystem fs, StoreFile store) {
+      super(fs);
+      this.file = store;
+    }
+
+    @Override
+    public void delete() throws IOException {
+      file.deleteReader();
+    }
+
+    @Override
+    public String getName() {
+      return file.getPath().getName();
+    }
+
+    @Override
+    public boolean isFile() {
+      return true;
+    }
+
+    @Override
+    public Collection<File> getChildren() throws IOException {
+      // storefiles don't have children
+      return Collections.emptyList();
+    }
+
+    @Override
+    public void close() throws IOException {
+      file.closeReader(true);
+    }
+
+    @Override
+    Path getPath() {
+      return file.getPath();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyArchiver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyArchiver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyArchiver.java
new file mode 100644
index 0000000..1386fe4
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyArchiver.java
@@ -0,0 +1,29 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs.legacy;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+
+public final class LegacyArchiver {
+
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
index 8f54bae..8c26cdf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
@@ -23,10 +23,20 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.mob.MobConstants;
 
 public final class LegacyLayout {
   /** Name of the region info file that resides just under the region directory. */
-  public final static String REGION_INFO_FILE = ".regioninfo";
+  private final static String REGION_INFO_FILE = ".regioninfo";
+
+  /** Temporary subdirectory of the region directory used for merges. */
+  public static final String REGION_MERGES_DIR = ".merges";
+
+  /** Temporary subdirectory of the region directory used for splits. */
+  public static final String REGION_SPLITS_DIR = ".splits";
+
+  /** Temporary subdirectory of the region directory used for compaction output. */
+  private static final String REGION_TEMP_DIR = ".tmp";
 
   private LegacyLayout() {}
 
@@ -63,8 +73,44 @@ public final class LegacyLayout {
     return new Path(nsDir, table.getQualifierAsString());
   }
 
-  public static Path getRegionDir(Path baseDataDir, TableName table, HRegionInfo hri) {
-    return new Path(getTableDir(baseDataDir, table), hri.getEncodedName());
+  public static Path getRegionDir(Path tableDir, HRegionInfo hri) {
+    return new Path(tableDir, hri.getEncodedName());
+  }
+
+  public static Path getFamilyDir(Path regionDir, String familyName) {
+    return new Path(regionDir, familyName);
+  }
+
+  public static Path getStoreFile(Path familyDir, String fileName) {
+    return new Path(familyDir, fileName);
+  }
+
+  public static Path getRegionInfoFile(Path regionDir) {
+    return new Path(regionDir, REGION_INFO_FILE);
+  }
+
+  public static Path getRegionTempDir(Path regionDir) {
+    return new Path(regionDir, REGION_TEMP_DIR);
+  }
+
+  public static Path getRegionMergesDir(Path regionDir) {
+    return new Path(regionDir, REGION_MERGES_DIR);
+  }
+
+  public static Path getRegionMergesDir(Path mergeDir, HRegionInfo hri) {
+    return new Path(mergeDir, hri.getEncodedName());
+  }
+
+  public static Path getRegionSplitsDir(Path regionDir) {
+    return new Path(regionDir, REGION_SPLITS_DIR);
+  }
+
+  public static Path getRegionSplitsDir(Path splitDir, HRegionInfo hri) {
+    return new Path(splitDir, hri.getEncodedName());
+  }
+
+  public static Path getMobDir(Path rootDir) {
+    return new Path(rootDir, MobConstants.MOB_DIR_NAME);
   }
 
   public static Path getBulkDir(Path rootDir) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
index 992dce1..b2675a0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
@@ -28,6 +28,7 @@ import java.util.Collections;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
@@ -186,6 +187,32 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
   }
 
   // ==========================================================================
+  //  PUBLIC Methods - Table Regions related
+  // ==========================================================================
+  @Override
+  public Collection<HRegionInfo> getRegions(FsContext ctx, TableName tableName)
+      throws IOException {
+    FileStatus[] stats = FSUtils.listStatus(getFileSystem(),
+        getTableDir(ctx, tableName), new FSUtils.RegionDirFilter(getFileSystem()));
+    if (stats == null) return Collections.emptyList();
+
+    ArrayList<HRegionInfo> regions = new ArrayList<HRegionInfo>(stats.length);
+    for (int i = 0; i < stats.length; ++i) {
+      regions.add(loadRegionInfo(stats[i].getPath()));
+    }
+    return regions;
+  }
+
+  protected HRegionInfo loadRegionInfo(Path regionDir) throws IOException {
+    FSDataInputStream in = getFileSystem().open(LegacyLayout.getRegionInfoFile(regionDir));
+    try {
+      return HRegionInfo.parseFrom(in);
+    } finally {
+      in.close();
+    }
+  }
+
+  // ==========================================================================
   //  PROTECTED Methods - Bootstrap
   // ==========================================================================
   @Override
@@ -365,7 +392,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
   }
 
   protected Path getRegionDir(FsContext ctx, TableName table, HRegionInfo hri) {
-    return LegacyLayout.getRegionDir(getBaseDirFromContext(ctx), table, hri);
+    return LegacyLayout.getRegionDir(getTableDir(ctx, table), hri);
   }
 
   public Path getTempDir() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java
new file mode 100644
index 0000000..d07a1f1
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyRegionFileSystem.java
@@ -0,0 +1,758 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs.legacy;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableDescriptor;
+import org.apache.hadoop.hbase.fs.FSUtilsWithRetries;
+import org.apache.hadoop.hbase.fs.FsContext;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.MetaUtils;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.regionserver.*;
+import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSHDFSUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
+
+@InterfaceAudience.Private
+public class LegacyRegionFileSystem extends RegionFileSystem {
+  private static final Log LOG = LogFactory.getLog(LegacyRegionFileSystem.class);
+
+  private final Path tableDir;
+  private final Path regionDir;
+  private final Path mobDir;
+
+  // regionInfo for interacting with FS (getting encodedName, etc)
+  private final HRegionInfo regionInfoForFs;
+
+  private final FSUtilsWithRetries fsWithRetries;
+
+  public LegacyRegionFileSystem(Configuration conf, FileSystem fs, Path rootDir, HRegionInfo hri) {
+    super(conf, fs, rootDir, hri);
+
+    Path dataDir = LegacyLayout.getDataDir(rootDir);
+    this.tableDir = LegacyLayout.getTableDir(dataDir, hri.getTable());
+    this.regionDir = LegacyLayout.getRegionDir(tableDir, hri);
+    this.mobDir = LegacyLayout.getDataDir(LegacyLayout.getMobDir(rootDir));
+    this.fsWithRetries = new FSUtilsWithRetries(conf, fs);
+
+    this.regionInfoForFs = ServerRegionReplicaUtil.getRegionInfoForFs(hri);
+  }
+
+  private Path getRegionDir() {
+    return regionDir;
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - Families Related
+  // ==========================================================================
+  @Override
+  public Collection<String> getFamilies() throws IOException {
+    FileSystem fs = getFileSystem();
+    FileStatus[] fds = FSUtils.listStatus(fs, regionDir, new FSUtils.FamilyDirFilter(fs));
+    if (fds == null) return null;
+
+    ArrayList<String> families = new ArrayList<String>(fds.length);
+    for (FileStatus status: fds) {
+      families.add(status.getPath().getName());
+    }
+    return families;
+  }
+
+  @Override
+  public void deleteFamily(String familyName, boolean hasMob) throws IOException {
+    // archive family store files
+    byte[] bFamilyName = Bytes.toBytes(familyName);
+
+    FileSystem fs = getFileSystem();
+    HFileArchiver.archiveFamily(fs, getConfiguration(), getRegionInfo(), tableDir, bFamilyName);
+
+    // delete the family folder
+    HRegionInfo region = getRegionInfo();
+    Path familyDir = new Path(tableDir, new Path(region.getEncodedName(), familyName));
+    if (!fsWithRetries.deleteDir(familyDir)) {
+      throw new IOException("Could not delete family "
+          + familyName + " from FileSystem for region "
+          + region.getRegionNameAsString() + "(" + region.getEncodedName()
+          + ")");
+    }
+
+    // archive and delete mob files
+    if (hasMob) {
+      Path mobTableDir = LegacyLayout.getTableDir(mobDir, getTable());
+      HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(getTable());
+      Path mobRegionDir = LegacyLayout.getRegionDir(mobTableDir, mobRegionInfo);
+      Path mobFamilyDir = LegacyLayout.getFamilyDir(mobRegionDir, familyName);
+      // archive mob family store files
+      MobUtils.archiveMobStoreFiles(getConfiguration(), getFileSystem(),
+          mobRegionInfo, mobFamilyDir, bFamilyName);
+
+      if (!fsWithRetries.deleteDir(mobFamilyDir)) {
+        throw new IOException("Could not delete mob store files for family "
+            + familyName + " from FileSystem region "
+            + mobRegionInfo.getRegionNameAsString() + "(" + mobRegionInfo.getEncodedName() + ")");
+      }
+    }
+  }
+
+  // ===========================================================================
+  //  Temp Helpers
+  // ===========================================================================
+  /** @return {@link Path} to the region's temp directory, used for file creations */
+  Path getTempDir() {
+    return LegacyLayout.getRegionTempDir(regionDir);
+  }
+
+  /**
+   * Clean up any temp detritus that may have been left around from previous operation attempts.
+   */
+  void cleanupTempDir() throws IOException {
+    fsWithRetries.deleteDir(getTempDir());
+  }
+
+  // ===========================================================================
+  //  Store/StoreFile Helpers
+  // ===========================================================================
+  /**
+   * Returns the directory path of the specified family
+   * @param familyName Column Family Name
+   * @return {@link Path} to the directory of the specified family
+   */
+  public Path getStoreDir(final String familyName) {
+    return LegacyLayout.getFamilyDir(getRegionDir(), familyName);
+  }
+
+  /**
+   * Create the store directory for the specified family name
+   * @param familyName Column Family Name
+   * @return {@link Path} to the directory of the specified family
+   * @throws IOException if the directory creation fails.
+   */
+  Path createStoreDir(final String familyName) throws IOException {
+    Path storeDir = getStoreDir(familyName);
+    if (!fsWithRetries.createDir(storeDir))
+      throw new IOException("Failed creating "+storeDir);
+    return storeDir;
+  }
+
+  // ==========================================================================
+  //  PUBLIC Methods - Store Files related
+  // ==========================================================================
+
+  @Override
+  public Collection<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate)
+      throws IOException {
+    Path familyDir = getStoreDir(familyName);
+    FileStatus[] files = FSUtils.listStatus(getFileSystem(), familyDir);
+    if (files == null) {
+      LOG.debug("No StoreFiles for: " + familyDir);
+      return null;
+    }
+
+    ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(files.length);
+    for (FileStatus status: files) {
+      if (validate && !StoreFileInfo.isValid(status)) {
+        LOG.warn("Invalid StoreFile: " + status.getPath());
+        continue;
+      }
+      StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(getConfiguration(),
+          getFileSystem(), getRegionInfo(), regionInfoForFs, familyName, status.getPath());
+      storeFiles.add(info);
+
+    }
+    return storeFiles;
+  }
+
+
+  /**
+   * Return Qualified Path of the specified family/file
+   *
+   * @param familyName Column Family Name
+   * @param fileName File Name
+   * @return The qualified Path for the specified family/file
+   */
+  Path getStoreFilePath(final String familyName, final String fileName) {
+    Path familyDir = getStoreDir(familyName);
+    return LegacyLayout.getStoreFile(familyDir, fileName).makeQualified(getFileSystem());
+  }
+
+  /**
+   * Return the store file information of the specified family/file.
+   *
+   * @param familyName Column Family Name
+   * @param fileName File Name
+   * @return The {@link StoreFileInfo} for the specified family/file
+   */
+  StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
+      throws IOException {
+    Path familyDir = getStoreDir(familyName);
+    return ServerRegionReplicaUtil.getStoreFileInfo(getConfiguration(),
+      getFileSystem(), getRegionInfo(), regionInfoForFs, familyName,
+      LegacyLayout.getStoreFile(familyDir, fileName));
+  }
+
+  /**
+   * Returns true if the specified family has reference files
+   * @param familyName Column Family Name
+   * @return true if family contains reference files
+   * @throws IOException
+   */
+  public boolean hasReferences(final String familyName) throws IOException {
+    FileStatus[] files = FSUtils.listStatus(getFileSystem(),
+        getStoreDir(familyName), new FSUtils.ReferenceFileFilter(getFileSystem()));
+    return files != null && files.length > 0;
+  }
+
+  /**
+   * Check whether region has Reference file
+   * @param htd table desciptor of the region
+   * @return true if region has reference file
+   * @throws IOException
+   */
+  public boolean hasReferences(final HTableDescriptor htd) throws IOException {
+    for (HColumnDescriptor family : htd.getFamilies()) {
+      if (hasReferences(family.getNameAsString())) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+
+  /**
+   * Generate a unique file name, used by createTempName() and commitStoreFile()
+   * @param suffix extra information to append to the generated name
+   * @return Unique file name
+   */
+  private static String generateUniqueName(final String suffix) {
+    String name = UUID.randomUUID().toString().replaceAll("-", "");
+    if (suffix != null) name += suffix;
+    return name;
+  }
+
+  /**
+   * Generate a unique temporary Path. Used in conjuction with commitStoreFile()
+   * to get a safer file creation.
+   * <code>
+   * Path file = fs.createTempName();
+   * ...StoreFile.Writer(file)...
+   * fs.commitStoreFile("family", file);
+   * </code>
+   *
+   * @return Unique {@link Path} of the temporary file
+   */
+  public Path createTempName() {
+    return createTempName(null);
+  }
+
+  /**
+   * Generate a unique temporary Path. Used in conjuction with commitStoreFile()
+   * to get a safer file creation.
+   * <code>
+   * Path file = fs.createTempName();
+   * ...StoreFile.Writer(file)...
+   * fs.commitStoreFile("family", file);
+   * </code>
+   *
+   * @param suffix extra information to append to the generated name
+   * @return Unique {@link Path} of the temporary file
+   */
+  public Path createTempName(final String suffix) {
+    return new Path(getTempDir(), generateUniqueName(suffix));
+  }
+
+  /**
+   * Move the file from a build/temp location to the main family store directory.
+   * @param familyName Family that will gain the file
+   * @param buildPath {@link Path} to the file to commit.
+   * @return The new {@link Path} of the committed file
+   * @throws IOException
+   */
+  public Path commitStoreFile(final String familyName, final Path buildPath) throws IOException {
+    return commitStoreFile(familyName, buildPath, -1, false);
+  }
+
+  /**
+   * Move the file from a build/temp location to the main family store directory.
+   * @param familyName Family that will gain the file
+   * @param buildPath {@link Path} to the file to commit.
+   * @param seqNum Sequence Number to append to the file name (less then 0 if no sequence number)
+   * @param generateNewName False if you want to keep the buildPath name
+   * @return The new {@link Path} of the committed file
+   * @throws IOException
+   */
+  private Path commitStoreFile(final String familyName, final Path buildPath,
+      final long seqNum, final boolean generateNewName) throws IOException {
+    Path storeDir = getStoreDir(familyName);
+    if(!fsWithRetries.createDir(storeDir))
+      throw new IOException("Failed creating " + storeDir);
+
+    String name = buildPath.getName();
+    if (generateNewName) {
+      name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_");
+    }
+    Path dstPath = new Path(storeDir, name);
+    if (!fsWithRetries.exists(buildPath)) {
+      throw new FileNotFoundException(buildPath.toString());
+    }
+    LOG.debug("Committing store file " + buildPath + " as " + dstPath);
+    // buildPath exists, therefore not doing an exists() check.
+    if (!fsWithRetries.rename(buildPath, dstPath)) {
+      throw new IOException("Failed rename of " + buildPath + " to " + dstPath);
+    }
+    return dstPath;
+  }
+
+
+  /**
+   * Moves multiple store files to the relative region's family store directory.
+   * @param storeFiles list of store files divided by family
+   * @throws IOException
+   */
+  void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException {
+    for (Map.Entry<byte[], List<StoreFile>> es: storeFiles.entrySet()) {
+      String familyName = Bytes.toString(es.getKey());
+      for (StoreFile sf: es.getValue()) {
+        commitStoreFile(familyName, sf.getPath());
+      }
+    }
+  }
+
+  /**
+   * Archives the specified store file from the specified family.
+   * @param familyName Family that contains the store files
+   * @param filePath {@link Path} to the store file to remove
+   * @throws IOException if the archiving fails
+   */
+  public void removeStoreFile(final String familyName, final Path filePath)
+      throws IOException {
+    HFileArchiver.archiveStoreFile(getConfiguration(), getFileSystem(), this.regionInfoForFs,
+        this.tableDir, Bytes.toBytes(familyName), filePath);
+  }
+
+  /**
+   * Closes and archives the specified store files from the specified family.
+   * @param familyName Family that contains the store files
+   * @param storeFiles set of store files to remove
+   * @throws IOException if the archiving fails
+   */
+  public void removeStoreFiles(final String familyName, final Collection<StoreFile> storeFiles)
+      throws IOException {
+    HFileArchiver.archiveStoreFiles(getConfiguration(), getFileSystem(), this.regionInfoForFs,
+        this.tableDir, Bytes.toBytes(familyName), storeFiles);
+  }
+
+  /**
+   * Bulk load: Add a specified store file to the specified family.
+   * If the source file is on the same different file-system is moved from the
+   * source location to the destination location, otherwise is copied over.
+   *
+   * @param familyName Family that will gain the file
+   * @param srcPath {@link Path} to the file to import
+   * @param seqNum Bulk Load sequence number
+   * @return The destination {@link Path} of the bulk loaded file
+   * @throws IOException
+   */
+  Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
+      throws IOException {
+    // Copy the file if it's on another filesystem
+    FileSystem fs = getFileSystem();
+    FileSystem srcFs = srcPath.getFileSystem(getConfiguration());
+    FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs;
+
+    // We can't compare FileSystem instances as equals() includes UGI instance
+    // as part of the comparison and won't work when doing SecureBulkLoad
+    // TODO deal with viewFS
+    if (!FSHDFSUtils.isSameHdfs(getConfiguration(), srcFs, desFs)) {
+      LOG.info("Bulk-load file " + srcPath + " is on different filesystem than " +
+          "the destination store. Copying file over to destination filesystem.");
+      Path tmpPath = createTempName();
+      FileUtil.copy(srcFs, srcPath, fs, tmpPath, false, getConfiguration());
+      LOG.info("Copied " + srcPath + " to temporary path on destination filesystem: " + tmpPath);
+      srcPath = tmpPath;
+    }
+
+    return commitStoreFile(familyName, srcPath, seqNum, true);
+  }
+
+  // ===========================================================================
+  //  Splits Helpers
+  // ===========================================================================
+  /** @return {@link Path} to the temp directory used during split operations */
+  Path getSplitsDir() {
+    return LegacyLayout.getRegionSplitsDir(getRegionDir());
+  }
+
+  Path getSplitsDir(final HRegionInfo hri) {
+    return LegacyLayout.getRegionSplitsDir(getSplitsDir(), hri);
+  }
+
+  /**
+   * Clean up any split detritus that may have been left around from previous split attempts.
+   */
+  void cleanupSplitsDir() throws IOException {
+    fsWithRetries.deleteDir(getSplitsDir());
+  }
+
+  /**
+   * Clean up any split detritus that may have been left around from previous
+   * split attempts.
+   * Call this method on initial region deploy.
+   * @throws IOException
+   */
+  void cleanupAnySplitDetritus() throws IOException {
+    Path splitdir = this.getSplitsDir();
+    if (!fsWithRetries.exists(splitdir)) return;
+    // Look at the splitdir.  It could have the encoded names of the daughter
+    // regions we tried to make.  See if the daughter regions actually got made
+    // out under the tabledir.  If here under splitdir still, then the split did
+    // not complete.  Try and do cleanup.  This code WILL NOT catch the case
+    // where we successfully created daughter a but regionserver crashed during
+    // the creation of region b.  In this case, there'll be an orphan daughter
+    // dir in the filesystem.  TOOD: Fix.
+    FileSystem fs = getFileSystem();
+    FileStatus[] daughters = FSUtils.listStatus(fs, splitdir, new FSUtils.DirFilter(fs));
+    if (daughters != null) {
+      for (int i = 0; i < daughters.length; ++i) {
+        Path daughterDir = new Path(this.tableDir, daughters[i].getPath().getName());
+        if (!fsWithRetries.deleteDir(daughterDir)) {
+          throw new IOException("Failed delete of " + daughterDir);
+        }
+      }
+    }
+    cleanupSplitsDir();
+    LOG.info("Cleaned up old failed split transaction detritus: " + splitdir);
+  }
+
+  /**
+   * Remove daughter region
+   * @param regionInfo daughter {@link HRegionInfo}
+   * @throws IOException
+   */
+  void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException {
+    Path regionDir = LegacyLayout.getRegionDir(tableDir, regionInfo);
+    if (!fsWithRetries.deleteDir(regionDir)) {
+      throw new IOException("Failed delete of " + regionDir);
+    }
+  }
+
+  /**
+   * Commit a daughter region, moving it from the split temporary directory
+   * to the proper location in the filesystem.
+   *
+   * @param regionInfo                 daughter {@link org.apache.hadoop.hbase.HRegionInfo}
+   * @throws IOException
+   */
+  Path commitDaughterRegion(final HRegionInfo regionInfo)
+      throws IOException {
+    Path regionDir = LegacyLayout.getRegionDir(tableDir, regionInfo);
+    Path daughterTmpDir = this.getSplitsDir(regionInfo);
+
+    if (fsWithRetries.exists(daughterTmpDir)) {
+
+      // Write HRI to a file in case we need to recover hbase:meta
+      Path regionInfoFile = LegacyLayout.getRegionInfoFile(daughterTmpDir);
+      byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
+      writeRegionInfoFileContent(getConfiguration(), getFileSystem(), regionInfoFile, regionInfoContent);
+
+      // Move the daughter temp dir to the table dir
+      if (!fsWithRetries.rename(daughterTmpDir, regionDir)) {
+        throw new IOException("Unable to rename " + daughterTmpDir + " to " + regionDir);
+      }
+    }
+
+    return regionDir;
+  }
+
+  /**
+   * Create the region splits directory.
+   */
+  void createSplitsDir() throws IOException {
+    createTempDir(getSplitsDir());
+  }
+
+  void createTempDir(Path dir) throws IOException {
+    if (fsWithRetries.exists(dir)) {
+      LOG.info("The " + dir + " directory exists.  Hence deleting it to recreate it");
+      if (!fsWithRetries.deleteDir(dir)) {
+        throw new IOException("Failed deletion of " + dir + " before creating them again.");
+      }
+    }
+    // dir doesn't exists now. No need to do an exists() call for it.
+    if (!fsWithRetries.createDir(dir)) {
+      throw new IOException("Failed create of " + dir);
+    }
+  }
+
+  /**
+   * Write out a split reference. Package local so it doesnt leak out of
+   * regionserver.
+   * @param hri {@link HRegionInfo} of the destination
+   * @param familyName Column Family Name
+   * @param f File to split.
+   * @param splitRow Split Row
+   * @param top True if we are referring to the top half of the hfile.
+   * @param splitPolicy
+   * @return Path to created reference.
+   * @throws IOException
+   */
+  Path splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f,
+      final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy)
+          throws IOException {
+
+    if (splitPolicy == null || !splitPolicy.skipStoreFileRangeCheck(familyName)) {
+      // Check whether the split row lies in the range of the store file
+      // If it is outside the range, return directly.
+      try {
+        if (top) {
+          //check if larger than last key.
+          KeyValue splitKey = KeyValueUtil.createFirstOnRow(splitRow);
+          Cell lastKey = f.createReader().getLastKey();
+          // If lastKey is null means storefile is empty.
+          if (lastKey == null) {
+            return null;
+          }
+          if (f.getReader().getComparator().compare(splitKey, lastKey) > 0) {
+            return null;
+          }
+        } else {
+          //check if smaller than first key
+          KeyValue splitKey = KeyValueUtil.createLastOnRow(splitRow);
+          Cell firstKey = f.createReader().getFirstKey();
+          // If firstKey is null means storefile is empty.
+          if (firstKey == null) {
+            return null;
+          }
+          if (f.getReader().getComparator().compare(splitKey, firstKey) < 0) {
+            return null;
+          }
+        }
+      } finally {
+        f.closeReader(true);
+      }
+    }
+
+    Path splitDir = new Path(getSplitsDir(hri), familyName);
+    // A reference to the bottom half of the hsf store file.
+    Reference r =
+      top ? Reference.createTopReference(splitRow): Reference.createBottomReference(splitRow);
+    // Add the referred-to regions name as a dot separated suffix.
+    // See REF_NAME_REGEX regex above.  The referred-to regions name is
+    // up in the path of the passed in <code>f</code> -- parentdir is family,
+    // then the directory above is the region name.
+    String parentRegionName = regionInfoForFs.getEncodedName();
+    // Write reference with same file id only with the other region name as
+    // suffix and into the new region location (under same family).
+    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
+    return r.write(getFileSystem(), p);
+  }
+
+  // ===========================================================================
+  //  Merge Helpers
+  // ===========================================================================
+  /** @return {@link Path} to the temp directory used during merge operations */
+  Path getMergesDir() {
+    return LegacyLayout.getRegionMergesDir(getRegionDir());
+  }
+
+  Path getMergesDir(final HRegionInfo hri) {
+    return LegacyLayout.getRegionMergesDir(getMergesDir(), hri);
+  }
+
+  /**
+   * Clean up any merge detritus that may have been left around from previous merge attempts.
+   */
+  void cleanupMergesDir() throws IOException {
+    fsWithRetries.deleteDir(getMergesDir());
+  }
+
+  /**
+   * Remove merged region
+   * @param mergedRegion {@link HRegionInfo}
+   * @throws IOException
+   */
+  void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException {
+    Path regionDir = LegacyLayout.getRegionDir(tableDir, mergedRegion);
+    if (fsWithRetries.deleteDir(regionDir)) {
+      throw new IOException("Failed delete of " + regionDir);
+    }
+  }
+
+  /**
+   * Create the region merges directory.
+   * @throws IOException If merges dir already exists or we fail to create it.
+   * @see HRegionFileSystem#cleanupMergesDir()
+   */
+  void createMergesDir() throws IOException {
+    createTempDir(getMergesDir());
+  }
+
+  /**
+   * Write out a merge reference under the given merges directory. Package local
+   * so it doesnt leak out of regionserver.
+   * @param mergedRegion {@link HRegionInfo} of the merged region
+   * @param familyName Column Family Name
+   * @param f File to create reference.
+   * @param mergedDir
+   * @return Path to created reference.
+   * @throws IOException
+   */
+  Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
+      final StoreFile f, final Path mergedDir)
+      throws IOException {
+    Path referenceDir = new Path(new Path(mergedDir,
+        mergedRegion.getEncodedName()), familyName);
+    // A whole reference to the store file.
+    Reference r = Reference.createTopReference(regionInfoForFs.getStartKey());
+    // Add the referred-to regions name as a dot separated suffix.
+    // See REF_NAME_REGEX regex above. The referred-to regions name is
+    // up in the path of the passed in <code>f</code> -- parentdir is family,
+    // then the directory above is the region name.
+    String mergingRegionName = regionInfoForFs.getEncodedName();
+    // Write reference with same file id only with the other region name as
+    // suffix and into the new region location (under same family).
+    Path p = new Path(referenceDir, f.getPath().getName() + "."
+        + mergingRegionName);
+    return r.write(getFileSystem(), p);
+  }
+
+  /**
+   * Commit a merged region, moving it from the merges temporary directory to
+   * the proper location in the filesystem.
+   * @param mergedRegionInfo merged region {@link HRegionInfo}
+   * @throws IOException
+   */
+  void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException {
+    Path regionDir = new Path(this.tableDir, mergedRegionInfo.getEncodedName());
+    Path mergedRegionTmpDir = this.getMergesDir(mergedRegionInfo);
+    // Move the tmp dir in the expected location
+    if (mergedRegionTmpDir != null && fsWithRetries.exists(mergedRegionTmpDir)) {
+      if (!fsWithRetries.rename(mergedRegionTmpDir, regionDir)) {
+        throw new IOException("Unable to rename " + mergedRegionTmpDir + " to "
+            + regionDir);
+      }
+    }
+  }
+
+  // ===========================================================================
+  //  Create/Open/Delete Helpers
+  // ===========================================================================
+  /**
+   * Log the current state of the region
+   * @param LOG log to output information
+   * @throws IOException if an unexpected exception occurs
+   */
+  void logFileSystemState(final Log LOG) throws IOException {
+    FSUtils.logFileSystemState(getFileSystem(), this.getRegionDir(), LOG);
+  }
+
+  /**
+   * @param hri
+   * @return Content of the file we write out to the filesystem under a region
+   * @throws IOException
+   */
+  private static byte[] getRegionInfoFileContent(final HRegionInfo hri) throws IOException {
+    return hri.toDelimitedByteArray();
+  }
+
+  /**
+   * Write the .regioninfo file on-disk.
+   */
+  private static void writeRegionInfoFileContent(final Configuration conf, final FileSystem fs,
+      final Path regionInfoFile, final byte[] content) throws IOException {
+    // First check to get the permissions
+    FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
+    // Write the RegionInfo file content
+    FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null);
+    try {
+      out.write(content);
+    } finally {
+      out.close();
+    }
+  }
+
+  // ==========================================================================
+  //  PUBLIC bootstrap
+  // ==========================================================================
+  @Override
+  protected void bootstrap() throws IOException {
+    // Cleanup temporary directories
+    cleanupTempDir();
+    cleanupSplitsDir();
+    cleanupMergesDir();
+
+    // if it doesn't exists, Write HRI to a file, in case we need to recover hbase:meta
+    checkRegionInfoOnFilesystem();
+  }
+
+  private void checkRegionInfoOnFilesystem() throws IOException {
+    // TODO
+  }
+
+  @Override
+  protected void destroy() throws IOException {
+    fsWithRetries.deleteDir(regionDir);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index ba08a05..e875f30 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -677,10 +677,9 @@ public class RegionStates {
         // This is RPC to meta table. It is done while we have a synchronize on
         // regionstates. No progress will be made if meta is not available at this time.
         // This is a cleanup task. Not critical.
-        if (MetaTableAccessor.getRegion(server.getConnection(), hri.getEncodedNameAsBytes()) ==
-            null) {
+        if (MetaTableAccessor.getRegion(server.getConnection(), hri.getEncodedNameAsBytes()) == null) {
           regionOffline(hri);
-          FSUtils.deleteRegionDir(server.getConfiguration(), hri);
+          server.getMasterFileSystem().deleteRegion(hri);
         }
       } catch (IOException e) {
         LOG.warn("Got exception while deleting " + hri + " directories from file system.", e);

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
index 3c12045..d022c99 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
@@ -28,6 +28,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -277,19 +278,48 @@ public class StoreFileInfo {
     return reader;
   }
 
+  private interface ComputeFileInfo<T> {
+    T compute(FileSystem fs, FileStatus status, long offset, long length)
+      throws IOException;
+  }
+
   /**
    * Compute the HDFS Block Distribution for this StoreFile
    */
   public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
       throws IOException {
+    return computeFileInfo(fs, new ComputeFileInfo<HDFSBlocksDistribution>() {
+      @Override
+      public HDFSBlocksDistribution compute(FileSystem fs, FileStatus status,
+          long offset, long length) throws IOException {
+        return FSUtils.computeHDFSBlocksDistribution(fs, status, offset, length);
+      }
+    });
+  }
 
+  public BlockLocation[] getFileBlockLocations(final FileSystem fs)
+      throws IOException {
+    return computeFileInfo(fs, new ComputeFileInfo<BlockLocation[]>() {
+      @Override
+      public BlockLocation[] compute(FileSystem fs, FileStatus status,
+          long offset, long length) throws IOException {
+        return fs.getFileBlockLocations(status, offset, length);
+      }
+    });
+  }
+
+  /**
+   * Compute the HDFS Block Distribution for this StoreFile
+   */
+  private <T> T computeFileInfo(final FileSystem fs,
+      final ComputeFileInfo<T> computeObj) throws IOException {
     // guard against the case where we get the FileStatus from link, but by the time we
     // call compute the file is moved again
     if (this.link != null) {
       FileNotFoundException exToThrow = null;
       for (int i = 0; i < this.link.getLocations().length; i++) {
         try {
-          return computeHDFSBlocksDistributionInternal(fs);
+          return computeFileInfoInternal(fs, computeObj);
         } catch (FileNotFoundException ex) {
           // try the other location
           exToThrow = ex;
@@ -297,21 +327,52 @@ public class StoreFileInfo {
       }
       throw exToThrow;
     } else {
-      return computeHDFSBlocksDistributionInternal(fs);
+      return computeFileInfoInternal(fs, computeObj);
     }
   }
 
-  private HDFSBlocksDistribution computeHDFSBlocksDistributionInternal(final FileSystem fs)
+  private <T> T computeFileInfoInternal(final FileSystem fs, final ComputeFileInfo<T> computeObj)
       throws IOException {
     FileStatus status = getReferencedFileStatus(fs);
     if (this.reference != null) {
-      return computeRefFileHDFSBlockDistribution(fs, reference, status);
+      return computeRefFileInfo(fs, reference, status, computeObj);
     } else {
-      return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
+      return computeObj.compute(fs, status, 0, status.getLen());
     }
   }
 
   /**
+   * helper function to compute HDFS blocks distribution of a given reference
+   * file.For reference file, we don't compute the exact value. We use some
+   * estimate instead given it might be good enough. we assume bottom part
+   * takes the first half of reference file, top part takes the second half
+   * of the reference file. This is just estimate, given
+   * midkey ofregion != midkey of HFile, also the number and size of keys vary.
+   * If this estimate isn't good enough, we can improve it later.
+   * @param fs  The FileSystem
+   * @param reference  The reference
+   * @param status  The reference FileStatus
+   * @return HDFS blocks distribution
+   */
+  private static <T> T computeRefFileInfo(final FileSystem fs, final Reference reference,
+      final FileStatus status, final ComputeFileInfo<T> computeObj) throws IOException {
+    if (status == null) {
+      return null;
+    }
+
+    long start = 0;
+    long length = 0;
+    if (Reference.isTopFileRegion(reference.getFileRegion())) {
+      start = status.getLen()/2;
+      length = status.getLen() - status.getLen()/2;
+    } else {
+      start = 0;
+      length = status.getLen()/2;
+    }
+    return computeObj.compute(fs, status, start, length);
+  }
+
+  /**
    * Get the {@link FileStatus} of the file referenced by this StoreFileInfo
    * @param fs The current file system to use.
    * @return The {@link FileStatus} of the file referenced by this StoreFileInfo
@@ -496,39 +557,6 @@ public class StoreFileInfo {
     return validateStoreFileName(p.getName());
   }
 
-  /**
-   * helper function to compute HDFS blocks distribution of a given reference
-   * file.For reference file, we don't compute the exact value. We use some
-   * estimate instead given it might be good enough. we assume bottom part
-   * takes the first half of reference file, top part takes the second half
-   * of the reference file. This is just estimate, given
-   * midkey ofregion != midkey of HFile, also the number and size of keys vary.
-   * If this estimate isn't good enough, we can improve it later.
-   * @param fs  The FileSystem
-   * @param reference  The reference
-   * @param status  The reference FileStatus
-   * @return HDFS blocks distribution
-   */
-  private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
-      final FileSystem fs, final Reference reference, final FileStatus status)
-      throws IOException {
-    if (status == null) {
-      return null;
-    }
-
-    long start = 0;
-    long length = 0;
-
-    if (Reference.isTopFileRegion(reference.getFileRegion())) {
-      start = status.getLen()/2;
-      length = status.getLen() - status.getLen()/2;
-    } else {
-      start = 0;
-      length = status.getLen()/2;
-    }
-    return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
-  }
-
   @Override
   public boolean equals(Object that) {
     if (this == that) return true;

http://git-wip-us.apache.org/repos/asf/hbase/blob/7546190a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
index b0af52b..d38d5c1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
@@ -25,7 +25,13 @@ import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.RegionFileSystem;
+import org.apache.hadoop.hbase.fs.RegionFileSystem.StoreFileVisitor;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -41,30 +47,18 @@ import org.apache.hadoop.hbase.util.FSUtils;
 class FSRegionScanner implements Runnable {
   static private final Log LOG = LogFactory.getLog(FSRegionScanner.class);
 
-  private Path regionPath;
-
-  /**
-   * The file system used
-   */
-  private FileSystem fs;
-
-  /**
-   * Maps each region to the RS with highest locality for that region.
-   */
-  private Map<String,String> regionToBestLocalityRSMapping;
+  private final RegionFileSystem rfs;
 
   /**
    * Maps region encoded names to maps of hostnames to fractional locality of
    * that region on that host.
    */
-  private Map<String, Map<String, Float>> regionDegreeLocalityMapping;
+  private final Map<String, Map<String, Float>> regionDegreeLocalityMapping;
 
-  FSRegionScanner(FileSystem fs, Path regionPath,
-                  Map<String, String> regionToBestLocalityRSMapping,
-                  Map<String, Map<String, Float>> regionDegreeLocalityMapping) {
-    this.fs = fs;
-    this.regionPath = regionPath;
-    this.regionToBestLocalityRSMapping = regionToBestLocalityRSMapping;
+  FSRegionScanner(Configuration conf, HRegionInfo hri,
+                  Map<String, Map<String, Float>> regionDegreeLocalityMapping)
+      throws IOException {
+    this.rfs = RegionFileSystem.open(conf, hri, true);
     this.regionDegreeLocalityMapping = regionDegreeLocalityMapping;
   }
 
@@ -72,38 +66,17 @@ class FSRegionScanner implements Runnable {
   public void run() {
     try {
       // empty the map for each region
-      Map<String, AtomicInteger> blockCountMap = new HashMap<String, AtomicInteger>();
-
-      //get table name
-      String tableName = regionPath.getParent().getName();
-      int totalBlkCount = 0;
+      final Map<String, AtomicInteger> blockCountMap = new HashMap<String, AtomicInteger>();
+      final AtomicInteger totalBlkCount = new AtomicInteger(0);
 
-      // ignore null
-      FileStatus[] cfList = fs.listStatus(regionPath, new FSUtils.FamilyDirFilter(fs));
-      if (null == cfList) {
-        return;
-      }
-
-      // for each cf, get all the blocks information
-      for (FileStatus cfStatus : cfList) {
-        if (!cfStatus.isDirectory()) {
-          // skip because this is not a CF directory
-          continue;
-        }
-
-        FileStatus[] storeFileLists = fs.listStatus(cfStatus.getPath());
-        if (null == storeFileLists) {
-          continue;
-        }
-
-        for (FileStatus storeFile : storeFileLists) {
-          BlockLocation[] blkLocations =
-            fs.getFileBlockLocations(storeFile, 0, storeFile.getLen());
-          if (null == blkLocations) {
-            continue;
-          }
+      rfs.visitStoreFiles(new StoreFileVisitor() {
+        @Override
+        public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
+            throws IOException {
+          BlockLocation[] blkLocations = storeFile.getFileBlockLocations(rfs.getFileSystem());
+          if (blkLocations == null) return;
 
-          totalBlkCount += blkLocations.length;
+          totalBlkCount.addAndGet(blkLocations.length);
           for(BlockLocation blk: blkLocations) {
             for (String host: blk.getHosts()) {
               AtomicInteger count = blockCountMap.get(host);
@@ -115,36 +88,9 @@ class FSRegionScanner implements Runnable {
             }
           }
         }
-      }
-
-      if (regionToBestLocalityRSMapping != null) {
-        int largestBlkCount = 0;
-        String hostToRun = null;
-        for (Map.Entry<String, AtomicInteger> entry : blockCountMap.entrySet()) {
-          String host = entry.getKey();
-
-          int tmp = entry.getValue().get();
-          if (tmp > largestBlkCount) {
-            largestBlkCount = tmp;
-            hostToRun = host;
-          }
-        }
-
-        // empty regions could make this null
-        if (null == hostToRun) {
-          return;
-        }
-
-        if (hostToRun.endsWith(".")) {
-          hostToRun = hostToRun.substring(0, hostToRun.length()-1);
-        }
-        String name = tableName + ":" + regionPath.getName();
-        synchronized (regionToBestLocalityRSMapping) {
-          regionToBestLocalityRSMapping.put(name,  hostToRun);
-        }
-      }
+      });
 
-      if (regionDegreeLocalityMapping != null && totalBlkCount > 0) {
+      if (regionDegreeLocalityMapping != null && totalBlkCount.get() > 0) {
         Map<String, Float> hostLocalityMap = new HashMap<String, Float>();
         for (Map.Entry<String, AtomicInteger> entry : blockCountMap.entrySet()) {
           String host = entry.getKey();
@@ -152,12 +98,12 @@ class FSRegionScanner implements Runnable {
             host = host.substring(0, host.length() - 1);
           }
           // Locality is fraction of blocks local to this host.
-          float locality = ((float)entry.getValue().get()) / totalBlkCount;
+          float locality = ((float)entry.getValue().get()) / totalBlkCount.get();
           hostLocalityMap.put(host, locality);
         }
         // Put the locality map into the result map, keyed by the encoded name
         // of the region.
-        regionDegreeLocalityMapping.put(regionPath.getName(), hostLocalityMap);
+        regionDegreeLocalityMapping.put(rfs.getRegionInfo().getEncodedName(), hostLocalityMap);
       }
     } catch (IOException e) {
       LOG.warn("Problem scanning file system", e);


[13/18] hbase git commit: HBASE-14439 Move fs stuff out pt1 - master fs + table desc

Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
index 0dde420..047f1a0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
@@ -38,14 +38,15 @@ import org.apache.hadoop.hbase.TableExistsException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
 import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.fs.FsContext;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CreateTableState;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
 import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
@@ -297,10 +298,9 @@ public class CreateTableProcedure
 
     // 1. Create Table Descriptor
     // using a copy of descriptor, table will be created enabling first
+    HTableDescriptor underConstruction = new HTableDescriptor(hTableDescriptor);
     final Path tempTableDir = FSUtils.getTableDir(tempdir, hTableDescriptor.getTableName());
-    ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors()))
-        .createTableDescriptorForTableDirectory(
-          tempTableDir, hTableDescriptor, false);
+    mfs.createTableDescriptor(FsContext.TEMP, underConstruction, false);
 
     // 2. Create Regions
     newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir,
@@ -319,6 +319,7 @@ public class CreateTableProcedure
     final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
     final Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), hTableDescriptor.getTableName());
     FileSystem fs = mfs.getFileSystem();
+
     if (!fs.delete(tableDir, true) && fs.exists(tableDir)) {
       throw new IOException("Couldn't delete " + tableDir);
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
index 2b042dd..c6ca374 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.NamespaceNotFoundException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.constraint.ConstraintException;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.master.TableNamespaceManager;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
@@ -275,23 +275,7 @@ public class DeleteNamespaceProcedure
   protected static void deleteDirectory(
       final MasterProcedureEnv env,
       final String namespaceName) throws IOException {
-    MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
-    FileSystem fs = mfs.getFileSystem();
-    Path p = FSUtils.getNamespaceDir(mfs.getRootDir(), namespaceName);
-
-    try {
-      for(FileStatus status : fs.listStatus(p)) {
-        if (!HConstants.HBASE_NON_TABLE_DIRS.contains(status.getPath().getName())) {
-          throw new IOException("Namespace directory contains table dir: " + status.getPath());
-        }
-      }
-      if (!fs.delete(FSUtils.getNamespaceDir(mfs.getRootDir(), namespaceName), true)) {
-        throw new IOException("Failed to remove namespace: " + namespaceName);
-      }
-    } catch (FileNotFoundException e) {
-      // File already deleted, continue
-      LOG.debug("deleteDirectory throws exception: " + e);
-    }
+    env.getMasterServices().getMasterFileSystem().deleteNamespace(namespaceName);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index fdf0b82..4c87b57 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -43,10 +43,10 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.exceptions.HBaseException;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.mob.MobConstants;
 import org.apache.hadoop.hbase.mob.MobUtils;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
index 8fa4183..d37159b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.RegionLocator;
 import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.BulkReOpen;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.util.Bytes;
 
 import com.google.common.collect.Lists;

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
index b27a3b8..feb9228 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.master.MetricsSnapshot;
 import org.apache.hadoop.hbase.master.RegionStates;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index abd69ad..3a5b974 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -51,8 +51,8 @@ import org.apache.hadoop.hbase.client.TableState;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.ipc.RpcServer;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.MetricsMaster;
 import org.apache.hadoop.hbase.master.SnapshotSentinel;

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
index e35c686..65b32a4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
@@ -60,7 +61,6 @@ import org.apache.hadoop.hbase.mapreduce.JobUtil;
 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 
 /*
@@ -111,13 +111,13 @@ public class CompactionTool extends Configured implements Tool {
       if (isFamilyDir(fs, path)) {
         Path regionDir = path.getParent();
         Path tableDir = regionDir.getParent();
-        HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir);
+        HTableDescriptor htd = LegacyTableDescriptor.getTableDescriptorFromFs(fs, tableDir);
         HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
         compactStoreFiles(tableDir, htd, hri,
             path.getName(), compactOnce, major);
       } else if (isRegionDir(fs, path)) {
         Path tableDir = path.getParent();
-        HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir);
+        HTableDescriptor htd = LegacyTableDescriptor.getTableDescriptorFromFs(fs, tableDir);
         compactRegion(tableDir, htd, path, compactOnce, major);
       } else if (isTableDir(fs, path)) {
         compactTable(path, compactOnce, major);
@@ -129,7 +129,7 @@ public class CompactionTool extends Configured implements Tool {
 
     private void compactTable(final Path tableDir, final boolean compactOnce, final boolean major)
         throws IOException {
-      HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir);
+      HTableDescriptor htd = LegacyTableDescriptor.getTableDescriptorFromFs(fs, tableDir);
       for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
         compactRegion(tableDir, htd, regionDir, compactOnce, major);
       }
@@ -198,7 +198,7 @@ public class CompactionTool extends Configured implements Tool {
   }
 
   private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException {
-    return FSTableDescriptors.getTableInfoPath(fs, path) != null;
+    return LegacyTableDescriptor.getTableInfoPath(fs, path) != null;
   }
 
   private static boolean isFamilyDir(final FileSystem fs, final Path path) throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
index 1763b2f..0409441 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
 import org.apache.hadoop.hbase.mob.MobUtils;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
@@ -55,7 +56,6 @@ import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Threads;
 
@@ -347,7 +347,7 @@ public final class SnapshotManifest {
   private void load() throws IOException {
     switch (getSnapshotFormat(desc)) {
       case SnapshotManifestV1.DESCRIPTOR_VERSION: {
-        this.htd = FSTableDescriptors.getTableDescriptorFromFs(fs, workingDir);
+        this.htd = LegacyTableDescriptor.getTableDescriptorFromFs(fs, workingDir);
         ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
         try {
           this.regionManifests =
@@ -444,8 +444,7 @@ public final class SnapshotManifest {
       Path rootDir = FSUtils.getRootDir(conf);
       LOG.info("Using old Snapshot Format");
       // write a copy of descriptor to the snapshot directory
-      new FSTableDescriptors(conf, fs, rootDir)
-        .createTableDescriptorForTableDirectory(workingDir, htd, false);
+      LegacyTableDescriptor.createTableDescriptor(fs, workingDir, htd, false);
     } else {
       LOG.debug("Convert to Single Snapshot Manifest");
       convertToV2SingleManifest();

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index 81dadd9..ebbe5df 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -19,27 +19,18 @@ package org.apache.hadoop.hbase.util;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.util.Comparator;
-import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.primitives.Ints;
 import edu.umd.cs.findbugs.annotations.Nullable;
 import org.apache.commons.lang.NotImplementedException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
@@ -48,8 +39,7 @@ import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.TableInfoMissingException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 
 /**
  * Implementation of {@link TableDescriptors} that reads descriptors from the
@@ -72,20 +62,16 @@ import org.apache.hadoop.hbase.regionserver.BloomType;
 @InterfaceAudience.Private
 public class FSTableDescriptors implements TableDescriptors {
   private static final Log LOG = LogFactory.getLog(FSTableDescriptors.class);
-  private final FileSystem fs;
-  private final Path rootdir;
+
+  private final MasterFileSystem mfs;
   private final boolean fsreadonly;
+
   private volatile boolean usecache;
   private volatile boolean fsvisited;
 
   @VisibleForTesting long cachehits = 0;
   @VisibleForTesting long invocations = 0;
 
-  /** The file name prefix used to store HTD in HDFS  */
-  static final String TABLEINFO_FILE_PREFIX = ".tableinfo";
-  static final String TABLEINFO_DIR = ".tabledesc";
-  static final String TMP_DIR = ".tmp";
-
   // This cache does not age out the old stuff.  Thinking is that the amount
   // of data we keep up in here is so small, no need to do occasional purge.
   // TODO.
@@ -116,75 +102,18 @@ public class FSTableDescriptors implements TableDescriptors {
    * operations; i.e. on remove, we do not do delete in fs.
    */
   public FSTableDescriptors(final Configuration conf, final FileSystem fs,
-    final Path rootdir, final boolean fsreadonly, final boolean usecache) throws IOException {
+      final Path rootdir, final boolean fsreadonly, final boolean usecache) throws IOException {
+    this(MasterFileSystem.open(conf, fs, rootdir, false), fsreadonly, usecache);
+  }
+
+  private FSTableDescriptors(final MasterFileSystem mfs, boolean fsreadonly, boolean usecache)
+      throws IOException {
     super();
-    this.fs = fs;
-    this.rootdir = rootdir;
+    this.mfs = mfs;
     this.fsreadonly = fsreadonly;
     this.usecache = usecache;
 
-    this.metaTableDescriptor = createMetaTableDescriptor(conf);
-  }
-
-  @VisibleForTesting
-  public static HTableDescriptor createMetaTableDescriptor(final Configuration conf)
-      throws IOException {
-    HTableDescriptor metaDescriptor = new HTableDescriptor(
-        TableName.META_TABLE_NAME,
-        new HColumnDescriptor[] {
-            new HColumnDescriptor(HConstants.CATALOG_FAMILY)
-                .setMaxVersions(conf.getInt(HConstants.HBASE_META_VERSIONS,
-                    HConstants.DEFAULT_HBASE_META_VERSIONS))
-                .setInMemory(true)
-                .setBlocksize(conf.getInt(HConstants.HBASE_META_BLOCK_SIZE,
-                    HConstants.DEFAULT_HBASE_META_BLOCK_SIZE))
-                .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
-                    // Disable blooms for meta.  Needs work.  Seems to mess w/ getClosestOrBefore.
-                .setBloomFilterType(BloomType.NONE)
-                    // Enable cache of data blocks in L1 if more than one caching tier deployed:
-                    // e.g. if using CombinedBlockCache (BucketCache).
-                .setCacheDataInL1(true),
-            new HColumnDescriptor(HConstants.REPLICATION_BARRIER_FAMILY)
-                .setMaxVersions(conf.getInt(HConstants.HBASE_META_VERSIONS,
-                    HConstants.DEFAULT_HBASE_META_VERSIONS))
-                .setInMemory(true)
-                .setBlocksize(conf.getInt(HConstants.HBASE_META_BLOCK_SIZE,
-                    HConstants.DEFAULT_HBASE_META_BLOCK_SIZE))
-                .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
-                // Disable blooms for meta.  Needs work.  Seems to mess w/ getClosestOrBefore.
-                .setBloomFilterType(BloomType.NONE)
-                // Enable cache of data blocks in L1 if more than one caching tier deployed:
-                // e.g. if using CombinedBlockCache (BucketCache).
-                .setCacheDataInL1(true),
-            new HColumnDescriptor(HConstants.REPLICATION_POSITION_FAMILY)
-                .setMaxVersions(conf.getInt(HConstants.HBASE_META_VERSIONS,
-                    HConstants.DEFAULT_HBASE_META_VERSIONS))
-                .setInMemory(true)
-                .setBlocksize(conf.getInt(HConstants.HBASE_META_BLOCK_SIZE,
-                    HConstants.DEFAULT_HBASE_META_BLOCK_SIZE))
-                .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
-                // Disable blooms for meta.  Needs work.  Seems to mess w/ getClosestOrBefore.
-                .setBloomFilterType(BloomType.NONE)
-                // Enable cache of data blocks in L1 if more than one caching tier deployed:
-                // e.g. if using CombinedBlockCache (BucketCache).
-                .setCacheDataInL1(true),
-            new HColumnDescriptor(HConstants.TABLE_FAMILY)
-                // Ten is arbitrary number.  Keep versions to help debugging.
-                .setMaxVersions(10)
-                .setInMemory(true)
-                .setBlocksize(8 * 1024)
-                .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
-                    // Disable blooms for meta.  Needs work.  Seems to mess w/ getClosestOrBefore.
-                .setBloomFilterType(BloomType.NONE)
-                    // Enable cache of data blocks in L1 if more than one caching tier deployed:
-                    // e.g. if using CombinedBlockCache (BucketCache).
-                .setCacheDataInL1(true)
-        }) {
-    };
-    metaDescriptor.addCoprocessor(
-        "org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint",
-        null, Coprocessor.PRIORITY_SYSTEM, null);
-    return metaDescriptor;
+    this.metaTableDescriptor = HTableDescriptor.metaTableDescriptor(mfs.getConfiguration());
   }
 
   @Override
@@ -235,15 +164,13 @@ public class FSTableDescriptors implements TableDescriptors {
     }
     HTableDescriptor tdmt = null;
     try {
-      tdmt = getTableDescriptorFromFs(fs, rootdir, tablename);
+      tdmt = mfs.getTableDescriptor(tablename);
     } catch (NullPointerException e) {
-      LOG.debug("Exception during readTableDecriptor. Current table name = "
-          + tablename, e);
+      LOG.debug("Exception during readTableDecriptor. Current table name = " + tablename, e);
     } catch (TableInfoMissingException e) {
       // ignore. This is regular operation
     } catch (IOException ioe) {
-      LOG.debug("Exception during readTableDecriptor. Current table name = "
-          + tablename, ioe);
+      LOG.debug("Exception during readTableDecriptor. Current table name = " + tablename, ioe);
     }
     // last HTD written wins
     if (usecache && tdmt != null) {
@@ -270,10 +197,11 @@ public class FSTableDescriptors implements TableDescriptors {
     } else {
       LOG.debug("Fetching table descriptors from the filesystem.");
       boolean allvisited = true;
-      for (Path d : FSUtils.getTableDirs(fs, rootdir)) {
+
+      for (TableName table: mfs.getTables()) {
         HTableDescriptor htd = null;
         try {
-          htd = get(FSUtils.getTableName(d));
+          htd = get(table);
         } catch (FileNotFoundException fnfe) {
           // inability of retrieving one HTD shouldn't stop getting the remaining
           LOG.warn("Trouble retrieving htd", fnfe);
@@ -309,21 +237,18 @@ public class FSTableDescriptors implements TableDescriptors {
     * @see #get(org.apache.hadoop.hbase.TableName)
     */
   @Override
-  public Map<String, HTableDescriptor> getByNamespace(String name)
-  throws IOException {
+  public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
     Map<String, HTableDescriptor> htds = new TreeMap<String, HTableDescriptor>();
-    List<Path> tableDirs =
-        FSUtils.getLocalTableDirs(fs, FSUtils.getNamespaceDir(rootdir, name));
-    for (Path d: tableDirs) {
+    for (TableName table: mfs.getTables(name)) {
       HTableDescriptor htd = null;
       try {
-        htd = get(FSUtils.getTableName(d));
+        htd = get(table);
       } catch (FileNotFoundException fnfe) {
         // inability of retrieving one HTD shouldn't stop getting the remaining
         LOG.warn("Trouble retrieving htd", fnfe);
       }
       if (htd == null) continue;
-      htds.put(FSUtils.getTableName(d).getNameAsString(), htd);
+      htds.put(table.getNameAsString(), htd);
     }
     return htds;
   }
@@ -355,372 +280,29 @@ public class FSTableDescriptors implements TableDescriptors {
    * from the FileSystem.
    */
   @Override
-  public HTableDescriptor remove(final TableName tablename)
-  throws IOException {
+  public HTableDescriptor remove(final TableName tablename) throws IOException {
     if (fsreadonly) {
       throw new NotImplementedException("Cannot remove a table descriptor - in read only mode");
     }
-    Path tabledir = getTableDir(tablename);
-    if (this.fs.exists(tabledir)) {
-      if (!this.fs.delete(tabledir, true)) {
-        throw new IOException("Failed delete of " + tabledir.toString());
-      }
-    }
+    mfs.deleteTable(tablename); // for test only??
     HTableDescriptor descriptor = this.cache.remove(tablename);
     return descriptor;
   }
 
   /**
-   * Checks if a current table info file exists for the given table
-   *
-   * @param tableName name of table
-   * @return true if exists
-   * @throws IOException
-   */
-  public boolean isTableInfoExists(TableName tableName) throws IOException {
-    return getTableInfoPath(tableName) != null;
-  }
-
-  /**
-   * Find the most current table info file for the given table in the hbase root directory.
-   * @return The file status of the current table info file or null if it does not exist
-   */
-  private FileStatus getTableInfoPath(final TableName tableName) throws IOException {
-    Path tableDir = getTableDir(tableName);
-    return getTableInfoPath(tableDir);
-  }
-
-  private FileStatus getTableInfoPath(Path tableDir)
-  throws IOException {
-    return getTableInfoPath(fs, tableDir, !fsreadonly);
-  }
-
-  /**
-   * Find the most current table info file for the table located in the given table directory.
-   *
-   * Looks within the {@link #TABLEINFO_DIR} subdirectory of the given directory for any table info
-   * files and takes the 'current' one - meaning the one with the highest sequence number if present
-   * or no sequence number at all if none exist (for backward compatibility from before there
-   * were sequence numbers).
-   *
-   * @return The file status of the current table info file or null if it does not exist
-   * @throws IOException
-   */
-  public static FileStatus getTableInfoPath(FileSystem fs, Path tableDir)
-  throws IOException {
-    return getTableInfoPath(fs, tableDir, false);
-  }
-
-  /**
-   * Find the most current table info file for the table in the given table directory.
-   *
-   * Looks within the {@link #TABLEINFO_DIR} subdirectory of the given directory for any table info
-   * files and takes the 'current' one - meaning the one with the highest sequence number if
-   * present or no sequence number at all if none exist (for backward compatibility from before
-   * there were sequence numbers).
-   * If there are multiple table info files found and removeOldFiles is true it also deletes the
-   * older files.
-   *
-   * @return The file status of the current table info file or null if none exist
-   * @throws IOException
-   */
-  private static FileStatus getTableInfoPath(FileSystem fs, Path tableDir, boolean removeOldFiles)
-  throws IOException {
-    Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
-    return getCurrentTableInfoStatus(fs, tableInfoDir, removeOldFiles);
-  }
-
-  /**
-   * Find the most current table info file in the given directory
-   *
-   * Looks within the given directory for any table info files
-   * and takes the 'current' one - meaning the one with the highest sequence number if present
-   * or no sequence number at all if none exist (for backward compatibility from before there
-   * were sequence numbers).
-   * If there are multiple possible files found
-   * and the we're not in read only mode it also deletes the older files.
-   *
-   * @return The file status of the current table info file or null if it does not exist
-   * @throws IOException
-   */
-  // only visible for FSTableDescriptorMigrationToSubdir, can be removed with that
-  static FileStatus getCurrentTableInfoStatus(FileSystem fs, Path dir, boolean removeOldFiles)
-  throws IOException {
-    FileStatus [] status = FSUtils.listStatus(fs, dir, TABLEINFO_PATHFILTER);
-    if (status == null || status.length < 1) return null;
-    FileStatus mostCurrent = null;
-    for (FileStatus file : status) {
-      if (mostCurrent == null || TABLEINFO_FILESTATUS_COMPARATOR.compare(file, mostCurrent) < 0) {
-        mostCurrent = file;
-      }
-    }
-    if (removeOldFiles && status.length > 1) {
-      // Clean away old versions
-      for (FileStatus file : status) {
-        Path path = file.getPath();
-        if (file != mostCurrent) {
-          if (!fs.delete(file.getPath(), false)) {
-            LOG.warn("Failed cleanup of " + path);
-          } else {
-            LOG.debug("Cleaned up old tableinfo file " + path);
-          }
-        }
-      }
-    }
-    return mostCurrent;
-  }
-
-  /**
-   * Compare {@link FileStatus} instances by {@link Path#getName()}. Returns in
-   * reverse order.
-   */
-  @VisibleForTesting
-  static final Comparator<FileStatus> TABLEINFO_FILESTATUS_COMPARATOR =
-  new Comparator<FileStatus>() {
-    @Override
-    public int compare(FileStatus left, FileStatus right) {
-      return right.compareTo(left);
-    }};
-
-  /**
-   * Return the table directory in HDFS
-   */
-  @VisibleForTesting Path getTableDir(final TableName tableName) {
-    return FSUtils.getTableDir(rootdir, tableName);
-  }
-
-  private static final PathFilter TABLEINFO_PATHFILTER = new PathFilter() {
-    @Override
-    public boolean accept(Path p) {
-      // Accept any file that starts with TABLEINFO_NAME
-      return p.getName().startsWith(TABLEINFO_FILE_PREFIX);
-    }};
-
-  /**
-   * Width of the sequenceid that is a suffix on a tableinfo file.
-   */
-  @VisibleForTesting static final int WIDTH_OF_SEQUENCE_ID = 10;
-
-  /*
-   * @param number Number to use as suffix.
-   * @return Returns zero-prefixed decimal version of passed
-   * number (Does absolute in case number is negative).
-   */
-  private static String formatTableInfoSequenceId(final int number) {
-    byte [] b = new byte[WIDTH_OF_SEQUENCE_ID];
-    int d = Math.abs(number);
-    for (int i = b.length - 1; i >= 0; i--) {
-      b[i] = (byte)((d % 10) + '0');
-      d /= 10;
-    }
-    return Bytes.toString(b);
-  }
-
-  /**
-   * Regex to eat up sequenceid suffix on a .tableinfo file.
-   * Use regex because may encounter oldstyle .tableinfos where there is no
-   * sequenceid on the end.
-   */
-  private static final Pattern TABLEINFO_FILE_REGEX =
-    Pattern.compile(TABLEINFO_FILE_PREFIX + "(\\.([0-9]{" + WIDTH_OF_SEQUENCE_ID + "}))?$");
-
-  /**
-   * @param p Path to a <code>.tableinfo</code> file.
-   * @return The current editid or 0 if none found.
-   */
-  @VisibleForTesting static int getTableInfoSequenceId(final Path p) {
-    if (p == null) return 0;
-    Matcher m = TABLEINFO_FILE_REGEX.matcher(p.getName());
-    if (!m.matches()) throw new IllegalArgumentException(p.toString());
-    String suffix = m.group(2);
-    if (suffix == null || suffix.length() <= 0) return 0;
-    return Integer.parseInt(m.group(2));
-  }
-
-  /**
-   * @param sequenceid
-   * @return Name of tableinfo file.
-   */
-  @VisibleForTesting static String getTableInfoFileName(final int sequenceid) {
-    return TABLEINFO_FILE_PREFIX + "." + formatTableInfoSequenceId(sequenceid);
-  }
-
-  /**
-   * Returns the latest table descriptor for the given table directly from the file system
-   * if it exists, bypassing the local cache.
-   * Returns null if it's not found.
-   */
-  public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs,
-      Path hbaseRootDir, TableName tableName) throws IOException {
-    Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName);
-    return getTableDescriptorFromFs(fs, tableDir);
-  }
-
-  /**
-   * Returns the latest table descriptor for the table located at the given directory
-   * directly from the file system if it exists.
-   * @throws TableInfoMissingException if there is no descriptor
-   */
-  public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs, Path tableDir)
-  throws IOException {
-    FileStatus status = getTableInfoPath(fs, tableDir, false);
-    if (status == null) {
-      throw new TableInfoMissingException("No table descriptor file under " + tableDir);
-    }
-    return readTableDescriptor(fs, status);
-  }
-
-  private static HTableDescriptor readTableDescriptor(FileSystem fs, FileStatus status)
-      throws IOException {
-    int len = Ints.checkedCast(status.getLen());
-    byte [] content = new byte[len];
-    FSDataInputStream fsDataInputStream = fs.open(status.getPath());
-    try {
-      fsDataInputStream.readFully(content);
-    } finally {
-      fsDataInputStream.close();
-    }
-    HTableDescriptor htd = null;
-    try {
-      htd = HTableDescriptor.parseFrom(content);
-    } catch (DeserializationException e) {
-      throw new IOException("content=" + Bytes.toShort(content), e);
-    }
-    return htd;
-  }
-
-  /**
    * Update table descriptor on the file system
    * @throws IOException Thrown if failed update.
    * @throws NotImplementedException if in read only mode
    */
-  @VisibleForTesting Path updateTableDescriptor(HTableDescriptor td)
-  throws IOException {
+  @VisibleForTesting
+  void updateTableDescriptor(HTableDescriptor td) throws IOException {
     if (fsreadonly) {
       throw new NotImplementedException("Cannot update a table descriptor - in read only mode");
     }
-    TableName tableName = td.getTableName();
-    Path tableDir = getTableDir(tableName);
-    Path p = writeTableDescriptor(fs, td, tableDir, getTableInfoPath(tableDir));
-    if (p == null) throw new IOException("Failed update");
-    LOG.info("Updated tableinfo=" + p);
+    mfs.updateTableDescriptor(td);
     if (usecache) {
       this.cache.put(td.getTableName(), td);
     }
-    return p;
-  }
-
-  /**
-   * Deletes all the table descriptor files from the file system.
-   * Used in unit tests only.
-   * @throws NotImplementedException if in read only mode
-   */
-  public void deleteTableDescriptorIfExists(TableName tableName) throws IOException {
-    if (fsreadonly) {
-      throw new NotImplementedException("Cannot delete a table descriptor - in read only mode");
-    }
-
-    Path tableDir = getTableDir(tableName);
-    Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
-    deleteTableDescriptorFiles(fs, tableInfoDir, Integer.MAX_VALUE);
-  }
-
-  /**
-   * Deletes files matching the table info file pattern within the given directory
-   * whose sequenceId is at most the given max sequenceId.
-   */
-  private static void deleteTableDescriptorFiles(FileSystem fs, Path dir, int maxSequenceId)
-  throws IOException {
-    FileStatus [] status = FSUtils.listStatus(fs, dir, TABLEINFO_PATHFILTER);
-    for (FileStatus file : status) {
-      Path path = file.getPath();
-      int sequenceId = getTableInfoSequenceId(path);
-      if (sequenceId <= maxSequenceId) {
-        boolean success = FSUtils.delete(fs, path, false);
-        if (success) {
-          LOG.debug("Deleted table descriptor at " + path);
-        } else {
-          LOG.error("Failed to delete descriptor at " + path);
-        }
-      }
-    }
-  }
-
-  /**
-   * Attempts to write a new table descriptor to the given table's directory.
-   * It first writes it to the .tmp dir then uses an atomic rename to move it into place.
-   * It begins at the currentSequenceId + 1 and tries 10 times to find a new sequence number
-   * not already in use.
-   * Removes the current descriptor file if passed in.
-   *
-   * @return Descriptor file or null if we failed write.
-   */
-  private static Path writeTableDescriptor(final FileSystem fs,
-    final HTableDescriptor htd, final Path tableDir,
-    final FileStatus currentDescriptorFile)
-  throws IOException {
-    // Get temporary dir into which we'll first write a file to avoid half-written file phenomenon.
-    // This directory is never removed to avoid removing it out from under a concurrent writer.
-    Path tmpTableDir = new Path(tableDir, TMP_DIR);
-    Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
-
-    // What is current sequenceid?  We read the current sequenceid from
-    // the current file.  After we read it, another thread could come in and
-    // compete with us writing out next version of file.  The below retries
-    // should help in this case some but its hard to do guarantees in face of
-    // concurrent schema edits.
-    int currentSequenceId = currentDescriptorFile == null ? 0 :
-      getTableInfoSequenceId(currentDescriptorFile.getPath());
-    int newSequenceId = currentSequenceId;
-
-    // Put arbitrary upperbound on how often we retry
-    int retries = 10;
-    int retrymax = currentSequenceId + retries;
-    Path tableInfoDirPath = null;
-    do {
-      newSequenceId += 1;
-      String filename = getTableInfoFileName(newSequenceId);
-      Path tempPath = new Path(tmpTableDir, filename);
-      if (fs.exists(tempPath)) {
-        LOG.debug(tempPath + " exists; retrying up to " + retries + " times");
-        continue;
-      }
-      tableInfoDirPath = new Path(tableInfoDir, filename);
-      try {
-        writeTD(fs, tempPath, htd);
-        fs.mkdirs(tableInfoDirPath.getParent());
-        if (!fs.rename(tempPath, tableInfoDirPath)) {
-          throw new IOException("Failed rename of " + tempPath + " to " + tableInfoDirPath);
-        }
-        LOG.debug("Wrote descriptor into: " + tableInfoDirPath);
-      } catch (IOException ioe) {
-        // Presume clash of names or something; go around again.
-        LOG.debug("Failed write and/or rename; retrying", ioe);
-        if (!FSUtils.deleteDirectory(fs, tempPath)) {
-          LOG.warn("Failed cleanup of " + tempPath);
-        }
-        tableInfoDirPath = null;
-        continue;
-      }
-      break;
-    } while (newSequenceId < retrymax);
-    if (tableInfoDirPath != null) {
-      // if we succeeded, remove old table info files.
-      deleteTableDescriptorFiles(fs, tableInfoDir, newSequenceId - 1);
-    }
-    return tableInfoDirPath;
-  }
-
-  private static void writeTD(final FileSystem fs, final Path p, final HTableDescriptor htd)
-  throws IOException {
-    FSDataOutputStream out = fs.create(p, false);
-    try {
-      // We used to write this file out as a serialized HTD Writable followed by two '\n's and then
-      // the toString version of HTD.  Now we just write out the pb serialization.
-      out.write(htd.toByteArray());
-    } finally {
-      out.close();
-    }
   }
 
   /**
@@ -740,42 +322,12 @@ public class FSTableDescriptors implements TableDescriptors {
    * @return True if we successfully created file.
    */
   public boolean createTableDescriptor(HTableDescriptor htd, boolean forceCreation)
-  throws IOException {
-    Path tableDir = getTableDir(htd.getTableName());
-    return createTableDescriptorForTableDirectory(tableDir, htd, forceCreation);
-  }
-
-  /**
-   * Create a new HTableDescriptor in HDFS in the specified table directory. Happens when we create
-   * a new table or snapshot a table.
-   * @param tableDir table directory under which we should write the file
-   * @param htd description of the table to write
-   * @param forceCreation if <tt>true</tt>,then even if previous table descriptor is present it will
-   *          be overwritten
-   * @return <tt>true</tt> if the we successfully created the file, <tt>false</tt> if the file
-   *         already exists and we weren't forcing the descriptor creation.
-   * @throws IOException if a filesystem error occurs
-   */
-  public boolean createTableDescriptorForTableDirectory(Path tableDir,
-      HTableDescriptor htd, boolean forceCreation) throws IOException {
+      throws IOException {
     if (fsreadonly) {
       throw new NotImplementedException("Cannot create a table descriptor - in read only mode");
     }
-    FileStatus status = getTableInfoPath(fs, tableDir);
-    if (status != null) {
-      LOG.debug("Current tableInfoPath = " + status.getPath());
-      if (!forceCreation) {
-        if (fs.exists(status.getPath()) && status.getLen() > 0) {
-          if (readTableDescriptor(fs, status).equals(htd)) {
-            LOG.debug("TableInfo already exists.. Skipping creation");
-            return false;
-          }
-        }
-      }
-    }
-    Path p = writeTableDescriptor(fs, htd, tableDir, status);
-    return p != null;
-  }
 
+    // forceCreation???
+    return mfs.createTableDescriptor(htd, forceCreation);
+  }
 }
-

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 15f079d..85a8080 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -266,8 +266,12 @@ public abstract class FSUtils {
    * @throws IOException e
    */
   public static boolean deleteDirectory(final FileSystem fs, final Path dir)
-  throws IOException {
-    return fs.exists(dir) && fs.delete(dir, true);
+      throws IOException {
+    try {
+      return !fs.delete(dir, true) ? !fs.exists(dir) : true;
+    } catch (FileNotFoundException e) {
+      return true;
+    }
   }
 
   /**
@@ -1180,7 +1184,7 @@ public abstract class FSUtils {
    * @param namespace namespace name
    * @return {@link org.apache.hadoop.fs.Path} for table
    */
-  public static Path getNamespaceDir(Path rootdir, final String namespace) {
+  private static Path getNamespaceDir(Path rootdir, final String namespace) {
     return new Path(rootdir, new Path(HConstants.BASE_NAMESPACE_DIR,
         new Path(namespace)));
   }
@@ -1322,7 +1326,7 @@ public abstract class FSUtils {
    * .logs, .oldlogs, .corrupt folders.
    * @throws IOException
    */
-  public static List<Path> getLocalTableDirs(final FileSystem fs, final Path rootdir)
+  private static List<Path> getLocalTableDirs(final FileSystem fs, final Path rootdir)
       throws IOException {
     // presumes any directory under hbase.rootdir is a table
     FileStatus[] dirs = fs.listStatus(rootdir, new UserTableDirFilter(fs));
@@ -1927,7 +1931,7 @@ public abstract class FSUtils {
    */
   public static void logFileSystemState(final FileSystem fs, final Path root, Log LOG)
       throws IOException {
-    LOG.debug("Current file system:");
+    LOG.debug("Current file system: " + root);
     logFSTree(LOG, fs, root, "|-");
   }
 
@@ -2208,4 +2212,23 @@ public abstract class FSUtils {
       return null;
     }
   }
+
+  public static void readFully(FileSystem fs, Path path, byte[] content) throws IOException {
+    FSDataInputStream in = fs.open(path);
+    try {
+      in.readFully(content);
+    } finally {
+      in.close();
+    }
+  }
+
+  public static void writeFully(FileSystem fs, Path path, byte[] content, boolean overwrite)
+      throws IOException {
+    FSDataOutputStream out = fs.create(path, overwrite);
+    try {
+      out.write(content);
+    } finally {
+      out.close();
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index a1a6099..a12425d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -110,9 +110,10 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RowMutations;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.RegionState;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
@@ -123,6 +124,7 @@ import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
 import org.apache.hadoop.hbase.security.UserProvider;
+import org.apache.hadoop.hbase.util.MetaUtils;
 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
@@ -1248,7 +1250,7 @@ public class HBaseFsck extends Configured implements Closeable {
         tablesInfo.put(tableName, modTInfo);
         try {
           HTableDescriptor htd =
-              FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
+              LegacyTableDescriptor.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
           modTInfo.htds.add(htd);
         } catch (IOException ioe) {
           if (!orphanTableDirs.containsKey(tableName)) {
@@ -1396,7 +1398,7 @@ public class HBaseFsck extends Configured implements Closeable {
     Configuration c = getConf();
     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
     HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
-    MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
+    MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, false);
     // The WAL subsystem will use the default rootDir rather than the passed in rootDir
     // unless I pass along via the conf.
     Configuration confForWAL = new Configuration(c);
@@ -1406,7 +1408,7 @@ public class HBaseFsck extends Configured implements Closeable {
         "hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8))).
         getWAL(metaHRI.getEncodedNameAsBytes(), metaHRI.getTable().getNamespace());
     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal);
-    MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
+    MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, true);
     return meta;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
index a235696..2fbaa92 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
@@ -133,7 +133,7 @@ public class HFileArchiveUtil {
 
   /**
    * Get the full path to the archive directory on the configured 
-   * {@link org.apache.hadoop.hbase.master.MasterFileSystem}
+   * {@link org.apache.hadoop.hbase.fs.MasterFileSystem}
    * @param conf to look for archive directory name and root directory. Cannot be null. Notes for
    *          testing: requires a FileSystem root directory to be specified.
    * @return the full {@link Path} to the archive directory, as defined by the configuration
@@ -145,7 +145,7 @@ public class HFileArchiveUtil {
 
   /**
    * Get the full path to the archive directory on the configured 
-   * {@link org.apache.hadoop.hbase.master.MasterFileSystem}
+   * {@link org.apache.hadoop.hbase.fs.MasterFileSystem}
    * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
    *          the archive path)
    * @return the full {@link Path} to the archive directory, as defined by the configuration

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
index 307568c..1959c73 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.wal.WALFactory;
 import org.apache.hadoop.ipc.RemoteException;
@@ -153,7 +154,7 @@ class HMerge {
 
       this.rootDir = FSUtils.getRootDir(conf);
       Path tabledir = FSUtils.getTableDir(this.rootDir, tableName);
-      this.htd = FSTableDescriptors.getTableDescriptorFromFs(this.fs, tabledir);
+      this.htd = LegacyTableDescriptor.getTableDescriptorFromFs(this.fs, tabledir);
       String logname = "merge_" + System.currentTimeMillis() + HConstants.HREGION_LOGDIR_NAME;
 
       final Configuration walConf = new Configuration(conf);

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java
index 3c81cfe..a08a9f2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.MetaTableAccessor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -135,7 +136,7 @@ public class Merge extends Configured implements Tool {
     if (info2 == null) {
       throw new NullPointerException("info2 is null using key " + meta);
     }
-    HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(FileSystem.get(getConf()),
+    HTableDescriptor htd = LegacyTableDescriptor.getTableDescriptorFromFs(FileSystem.get(getConf()),
       this.rootdir, this.tableName);
     HRegion merged = merge(htd, meta, info1, info2);
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java
index 7c89f11..1c940f7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java
@@ -30,7 +30,9 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -88,7 +90,7 @@ public class MetaUtils {
    */
   public synchronized WAL getLog(HRegionInfo info) throws IOException {
     if (this.walFactory == null) {
-      String logName = 
+      String logName =
           HConstants.HREGION_LOGDIR_NAME + "_" + System.currentTimeMillis();
       final Configuration walConf = new Configuration(this.conf);
       FSUtils.setRootDir(walConf, fs.getHomeDirectory());
@@ -152,4 +154,16 @@ public class MetaUtils {
     this.metaRegion.compactStores();
     return this.metaRegion;
   }
+
+  /**
+   * Enable in memory caching for hbase:meta
+   */
+  public static void setInfoFamilyCachingForMeta(HTableDescriptor metaDescriptor, final boolean b) {
+    for (HColumnDescriptor hcd: metaDescriptor.getColumnFamilies()) {
+      if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
+        hcd.setBlockCacheEnabled(b);
+        hcd.setInMemory(b);
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java
index f129f9d..4abf350 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java
@@ -24,9 +24,10 @@ import java.io.IOException;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
@@ -159,7 +160,7 @@ public class TestHColumnDescriptorDefaultVersions {
     // Verify descriptor from HDFS
     MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
     Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
-    HTableDescriptor td = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir);
+    HTableDescriptor td = LegacyTableDescriptor.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir);
     hcds = td.getColumnFamilies();
     verifyHColumnDescriptor(expected, hcds, tableName, families);
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index a3fc640..53de8a0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
 import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
index 8317376..fbd347a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
@@ -224,8 +224,8 @@ public class TestSnapshotFromClient {
     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
     LOG.debug("FS state after snapshot:");
-    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
-      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+
     SnapshotTestingUtils.confirmSnapshotValid(
       ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM,
       rootDir, admin, fs);
@@ -294,8 +294,7 @@ public class TestSnapshotFromClient {
     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
     LOG.debug("FS state after snapshot:");
-    FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
-      FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
+    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
 
     List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
     List<byte[]> nonEmptyCfs = Lists.newArrayList();

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 140dcd9..4348d2b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
 import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
 import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails;
 import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
index bf13e7f..48e143d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.FSUtils;

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java
index 417987b..c143b9a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java
@@ -32,7 +32,8 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -275,7 +276,7 @@ public class TestTableDescriptorModificationFromClient {
     MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
     Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
     HTableDescriptor td =
-        FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir);
+        LegacyTableDescriptor.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir);
     verifyTableDescriptor(td, tableName, families);
   }
 
@@ -288,4 +289,4 @@ public class TestTableDescriptorModificationFromClient {
       assertTrue("Expected family " + Bytes.toString(familyName), htdFamilies.contains(familyName));
     }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
index 2a97119..035b17b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
@@ -30,8 +30,8 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.executor.ExecutorService;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.MetricsMaster;
 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
index 8c9db88..49a290c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
@@ -35,8 +35,6 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.CategoryBasedTimeout;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
@@ -65,6 +63,7 @@ import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
 import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.RegionState;
@@ -190,8 +189,7 @@ public class TestNamespaceAuditor {
   @Test
   public void testValidQuotas() throws Exception {
     boolean exceptionCaught = false;
-    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
-    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+    MasterFileSystem mfs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem();
     NamespaceDescriptor nspDesc =
         NamespaceDescriptor.create(prefix + "vq1")
             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
@@ -203,7 +201,7 @@ public class TestNamespaceAuditor {
       exceptionCaught = true;
     } finally {
       assertTrue(exceptionCaught);
-      assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
+      assertFalse(mfs.getNamespaces().contains(nspDesc.getName()));
     }
     nspDesc =
         NamespaceDescriptor.create(prefix + "vq2")
@@ -216,7 +214,7 @@ public class TestNamespaceAuditor {
       exceptionCaught = true;
     } finally {
       assertTrue(exceptionCaught);
-      assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
+      assertFalse(mfs.getNamespaces().contains(nspDesc.getName()));
     }
     nspDesc =
         NamespaceDescriptor.create(prefix + "vq3")
@@ -229,7 +227,7 @@ public class TestNamespaceAuditor {
       exceptionCaught = true;
     } finally {
       assertTrue(exceptionCaught);
-      assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
+      assertFalse(mfs.getNamespaces().contains(nspDesc.getName()));
     }
     nspDesc =
         NamespaceDescriptor.create(prefix + "vq4")
@@ -242,7 +240,7 @@ public class TestNamespaceAuditor {
       exceptionCaught = true;
     } finally {
       assertTrue(exceptionCaught);
-      assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
+      assertFalse(mfs.getNamespaces().contains(nspDesc.getName()));
     }
   }
 
@@ -716,8 +714,8 @@ public class TestNamespaceAuditor {
     ADMIN.createTable(tableDescOne);
     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
   }
-  
-  @Test(expected = QuotaExceededException.class)
+
+  @Test(expected = QuotaExceededException.class, timeout = 30000)
   public void testCloneSnapshotQuotaExceed() throws Exception {
     String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
     NamespaceDescriptor nspDesc =

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
index fdc6c92..e4eff84 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
@@ -989,7 +989,7 @@ public class TestDefaultMemStore {
     edge.setCurrentTimeMillis(1234);
     WALFactory wFactory = new WALFactory(conf, null, "1234");
     HRegion meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
-        conf, FSTableDescriptors.createMetaTableDescriptor(conf),
+        conf, HTableDescriptor.metaTableDescriptor(conf),
         wFactory.getMetaWAL(HRegionInfo.FIRST_META_REGIONINFO.
             getEncodedNameAsBytes()));
     HRegionInfo hri = new HRegionInfo(TableName.valueOf("testShouldFlushMeta"),

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
index 0a8dbc4..09532cf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
@@ -826,11 +826,10 @@ public class TestStoreFile extends HBaseTestCase {
 
     // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname.
     Path storedir = new Path(new Path(testDir, "7e0102"), Bytes.toString(family));
-    Path dir = new Path(storedir, "1234567890");
     HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
     // Make a store file and write data to it.
     StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
-            .withOutputDir(dir)
+            .withOutputDir(storedir)
             .withFileContext(meta)
             .build();
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index dfd00b3..958278d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -55,9 +55,10 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.mob.MobUtils;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
@@ -69,7 +70,6 @@ import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSVisitor;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.MD5Hash;
@@ -497,8 +497,8 @@ public final class SnapshotTestingUtils {
         this.desc = desc;
         this.tableRegions = tableRegions;
         this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
-        new FSTableDescriptors(conf)
-          .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
+        LegacyTableDescriptor.createTableDescriptor(
+            fs, snapshotDir, htd, false);
       }
 
       public HTableDescriptor getTableDescriptor() {
@@ -690,7 +690,7 @@ public final class SnapshotTestingUtils {
     private RegionData[] createTable(final HTableDescriptor htd, final int nregions)
         throws IOException {
       Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
-      new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
+      LegacyTableDescriptor.createTableDescriptor(fs, tableDir, htd, false);
 
       assertTrue(nregions % 2 == 0);
       RegionData[] regions = new RegionData[nregions];

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
index bdc09fe..27d6dde 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.TableExistsException;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.junit.Test;
@@ -62,12 +63,12 @@ public class TestFSTableDescriptors {
 
   @Test (expected=IllegalArgumentException.class)
   public void testRegexAgainstOldStyleTableInfo() {
-    Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX);
-    int i = FSTableDescriptors.getTableInfoSequenceId(p);
+    Path p = new Path("/tmp", LegacyTableDescriptor.TABLEINFO_FILE_PREFIX);
+    int i = LegacyTableDescriptor.getTableInfoSequenceId(p);
     assertEquals(0, i);
     // Assert it won't eat garbage -- that it fails
     p = new Path("/tmp", "abc");
-    FSTableDescriptors.getTableInfoSequenceId(p);
+    LegacyTableDescriptor.getTableInfoSequenceId(p);
   }
 
   @Test
@@ -96,23 +97,28 @@ public class TestFSTableDescriptors {
     HTableDescriptor htd = new HTableDescriptor(
         TableName.valueOf("testSequenceidAdvancesOnTableInfo"));
     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
+    Path tableDir = FSUtils.getTableDir(testdir, htd.getTableName());
     FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
-    Path p0 = fstd.updateTableDescriptor(htd);
-    int i0 = FSTableDescriptors.getTableInfoSequenceId(p0);
-    Path p1 = fstd.updateTableDescriptor(htd);
+    fstd.updateTableDescriptor(htd);
+    Path p0 = LegacyTableDescriptor.getTableInfoPath(fs, tableDir).getPath();
+    int i0 = LegacyTableDescriptor.getTableInfoSequenceId(p0);
+    fstd.updateTableDescriptor(htd);
+    Path p1 = LegacyTableDescriptor.getTableInfoPath(fs, tableDir).getPath();
     // Assert we cleaned up the old file.
     assertTrue(!fs.exists(p0));
-    int i1 = FSTableDescriptors.getTableInfoSequenceId(p1);
+    int i1 = LegacyTableDescriptor.getTableInfoSequenceId(p1);
     assertTrue(i1 == i0 + 1);
-    Path p2 = fstd.updateTableDescriptor(htd);
+    fstd.updateTableDescriptor(htd);
+    Path p2 = LegacyTableDescriptor.getTableInfoPath(fs, tableDir).getPath();
     // Assert we cleaned up the old file.
     assertTrue(!fs.exists(p1));
-    int i2 = FSTableDescriptors.getTableInfoSequenceId(p2);
+    int i2 = LegacyTableDescriptor.getTableInfoSequenceId(p2);
     assertTrue(i2 == i1 + 1);
-    Path p3 = fstd.updateTableDescriptor(htd);
+    fstd.updateTableDescriptor(htd);
+    Path p3 = LegacyTableDescriptor.getTableInfoPath(fs, tableDir).getPath();
     // Assert we cleaned up the old file.
     assertTrue(!fs.exists(p2));
-    int i3 = FSTableDescriptors.getTableInfoSequenceId(p3);
+    int i3 = LegacyTableDescriptor.getTableInfoSequenceId(p3);
     assertTrue(i3 == i2 + 1);
     HTableDescriptor descriptor = fstd.get(htd.getTableName());
     assertEquals(descriptor, htd);
@@ -123,29 +129,29 @@ public class TestFSTableDescriptors {
     Path p0 = assertWriteAndReadSequenceId(0);
     // Assert p0 has format we expect.
     StringBuilder sb = new StringBuilder();
-    for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
+    for (int i = 0; i < LegacyTableDescriptor.WIDTH_OF_SEQUENCE_ID; i++) {
       sb.append("0");
     }
-    assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(),
+    assertEquals(LegacyTableDescriptor.TABLEINFO_FILE_PREFIX + "." + sb.toString(),
       p0.getName());
     // Check a few more.
     Path p2 = assertWriteAndReadSequenceId(2);
     Path p10000 = assertWriteAndReadSequenceId(10000);
     // Get a .tablinfo that has no sequenceid suffix.
-    Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_FILE_PREFIX);
+    Path p = new Path(p0.getParent(), LegacyTableDescriptor.TABLEINFO_FILE_PREFIX);
     FileStatus fs = new FileStatus(0, false, 0, 0, 0, p);
     FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0);
     FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2);
     FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000);
-    Comparator<FileStatus> comparator = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
+    Comparator<FileStatus> comparator = LegacyTableDescriptor.TABLEINFO_FILESTATUS_COMPARATOR;
     assertTrue(comparator.compare(fs, fs0) > 0);
     assertTrue(comparator.compare(fs0, fs2) > 0);
     assertTrue(comparator.compare(fs2, fs10000) > 0);
   }
 
   private Path assertWriteAndReadSequenceId(final int i) {
-    Path p = new Path("/tmp", FSTableDescriptors.getTableInfoFileName(i));
-    int ii = FSTableDescriptors.getTableInfoSequenceId(p);
+    Path p = new Path("/tmp", LegacyTableDescriptor.getTableInfoFileName(i));
+    int ii = LegacyTableDescriptor.getTableInfoSequenceId(p);
     assertEquals(i, ii);
     return p;
   }
@@ -171,7 +177,7 @@ public class TestFSTableDescriptors {
     FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
     fstd.createTableDescriptor(htd);
     HTableDescriptor td2 =
-      FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName());
+      LegacyTableDescriptor.getTableDescriptorFromFs(fs, rootdir, htd.getTableName());
     assertTrue(htd.equals(td2));
   }
 
@@ -181,20 +187,17 @@ public class TestFSTableDescriptors {
     Path rootdir = UTIL.getDataTestDir(name);
     FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
-    Path descriptorFile = fstd.updateTableDescriptor(htd);
-    try (FSDataOutputStream out = fs.create(descriptorFile, true)) {
-      out.write(htd.toByteArray());
-    }
+    Path tableDir = FSUtils.getTableDir(rootdir, htd.getTableName());
+    fstd.updateTableDescriptor(htd);
+    Path descriptorFile = LegacyTableDescriptor.getTableInfoPath(fs, tableDir).getPath();
+    FSUtils.writeFully(fs, descriptorFile, htd.toByteArray(), true);
     FSTableDescriptors fstd2 = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
-    HTableDescriptor td2 = fstd2.get(htd.getTableName());
+    HTableDescriptor td2 = fstd2.getDescriptor(htd.getTableName());
     assertEquals(htd, td2);
-    FileStatus descriptorFile2 =
-        FSTableDescriptors.getTableInfoPath(fs, fstd2.getTableDir(htd.getTableName()));
+    FileStatus descriptorFile2 = LegacyTableDescriptor.getTableInfoPath(fs, tableDir);
     byte[] buffer = htd.toByteArray();
-    try (FSDataInputStream in = fs.open(descriptorFile2.getPath())) {
-      in.readFully(buffer);
-    }
-    HTableDescriptor td3 = HTableDescriptor.parseFrom(buffer);
+    FSUtils.readFully(fs, descriptorFile2.getPath(), buffer);
+    TableDescriptor td3 = TableDescriptor.parseFrom(buffer);
     assertEquals(htd, td3);
   }
 
@@ -372,7 +375,7 @@ public class TestFSTableDescriptors {
   public void testTableInfoFileStatusComparator() {
     FileStatus bare =
       new FileStatus(0, false, 0, 0, -1,
-        new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
+        new Path("/tmp", LegacyTableDescriptor.TABLEINFO_FILE_PREFIX));
     FileStatus future =
       new FileStatus(0, false, 0, 0, -1,
         new Path("/tmp/tablinfo." + System.currentTimeMillis()));
@@ -382,7 +385,7 @@ public class TestFSTableDescriptors {
     FileStatus [] alist = {bare, future, farFuture};
     FileStatus [] blist = {bare, farFuture, future};
     FileStatus [] clist = {farFuture, bare, future};
-    Comparator<FileStatus> c = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
+    Comparator<FileStatus> c = LegacyTableDescriptor.TABLEINFO_FILESTATUS_COMPARATOR;
     Arrays.sort(alist, c);
     Arrays.sort(blist, c);
     Arrays.sort(clist, c);
@@ -419,12 +422,12 @@ public class TestFSTableDescriptors {
     assertFalse(fstd.createTableDescriptor(htd));
     htd.setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue"));
     assertTrue(fstd.createTableDescriptor(htd)); //this will re-create
-    Path tableDir = fstd.getTableDir(htd.getTableName());
-    Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR);
+    Path tableDir = FSUtils.getTableDir(testdir, htd.getTableName());
+    Path tmpTableDir = new Path(tableDir, LegacyTableDescriptor.TMP_DIR);
     FileStatus[] statuses = fs.listStatus(tmpTableDir);
     assertTrue(statuses.length == 0);
 
-    assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
+    assertEquals(htd, LegacyTableDescriptor.getTableDescriptorFromFs(fs, tableDir));
   }
 
   private static class FSTableDescriptorsTest extends FSTableDescriptors {

http://git-wip-us.apache.org/repos/asf/hbase/blob/419800f8/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
index e03a0d5..e15609a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableDescriptor;
 import org.apache.hadoop.hbase.MetaTableAccessor;
 import org.apache.hadoop.hbase.MiniHBaseCluster;
 import org.apache.hadoop.hbase.ServerName;
@@ -44,6 +45,7 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
 import org.apache.hadoop.hbase.io.hfile.TestHFile;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.RegionState;
@@ -230,7 +232,7 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
       Path hbaseTableDir = FSUtils.getTableDir(
           FSUtils.getRootDir(conf), table);
       fs = hbaseTableDir.getFileSystem(conf);
-      FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
+      FileStatus status = LegacyTableDescriptor.getTableInfoPath(fs, hbaseTableDir);
       tableinfo = status.getPath();
       fs.rename(tableinfo, new Path("/.tableinfo"));
 
@@ -242,7 +244,7 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
       hbck = doFsck(conf, true);
       assertNoErrors(hbck);
       status = null;
-      status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
+      status = LegacyTableDescriptor.getTableInfoPath(fs, hbaseTableDir);
       assertNotNull(status);
 
       HTableDescriptor htd = admin.getTableDescriptor(table);
@@ -256,7 +258,7 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
       htd = admin.getTableDescriptor(table); // warms up cached htd on master
       hbck = doFsck(conf, true);
       assertNoErrors(hbck);
-      status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
+      status = LegacyTableDescriptor.getTableInfoPath(fs, hbaseTableDir);
       assertNotNull(status);
       htd = admin.getTableDescriptor(table);
       assertEquals(htd.getValue("NOT_DEFAULT"), "true");
@@ -420,8 +422,8 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
       htdDisabled.addFamily(new HColumnDescriptor(FAM));
 
       // Write the .tableinfo
-      FSTableDescriptors fstd = new FSTableDescriptors(conf);
-      fstd.createTableDescriptor(htdDisabled);
+      cluster.getMaster().getMasterFileSystem().createTableDescriptor(
+        new TableDescriptor(htdDisabled), true);
       List<HRegionInfo> disabledRegions =
           TEST_UTIL.createMultiRegionsInMeta(conf, htdDisabled, SPLIT_KEYS);