You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2016/09/26 18:50:06 UTC
[1/3] hbase git commit: HBASE-14439 Moved from MasterFileSystem to
MasterStorage abstract class for storage abstraction layer.
Repository: hbase
Updated Branches:
refs/heads/hbase-14439 2986c9716 -> dea247ec4
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index a852547..d51b62a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -145,7 +145,7 @@ public class TestFlushSnapshotFromClient {
SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
LOG.debug("FS state before snapshot:");
- UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
// take a snapshot of the enabled table
String snapshotString = "offlineTableSnapshot";
@@ -160,7 +160,7 @@ public class TestFlushSnapshotFromClient {
// make sure its a valid snapshot
LOG.debug("FS state after snapshot:");
- UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
SnapshotTestingUtils.confirmSnapshotValid(UTIL,
ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);
@@ -181,7 +181,7 @@ public class TestFlushSnapshotFromClient {
UTIL.flush(TABLE_NAME);
LOG.debug("FS state before snapshot:");
- UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
// take a snapshot of the enabled table
String snapshotString = "skipFlushTableSnapshot";
@@ -196,7 +196,7 @@ public class TestFlushSnapshotFromClient {
// make sure its a valid snapshot
LOG.debug("FS state after snapshot:");
- UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
SnapshotTestingUtils.confirmSnapshotValid(UTIL,
ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);
@@ -220,7 +220,7 @@ public class TestFlushSnapshotFromClient {
SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
LOG.debug("FS state before snapshot:");
- UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
// take a snapshot of the enabled table
String snapshotString = "offlineTableSnapshot";
@@ -239,7 +239,7 @@ public class TestFlushSnapshotFromClient {
// make sure its a valid snapshot
LOG.debug("FS state after snapshot:");
- UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
SnapshotTestingUtils.confirmSnapshotValid(UTIL,
ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);
@@ -289,7 +289,7 @@ public class TestFlushSnapshotFromClient {
HMaster master = UTIL.getMiniHBaseCluster().getMaster();
SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
LOG.info(" === Async Snapshot Completed ===");
- UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
// make sure we get the snapshot
SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
@@ -401,8 +401,8 @@ public class TestFlushSnapshotFromClient {
SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
String snapshotName = "flushSnapshotCreateListDestroy";
- FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
- Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
+ Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM),
snapshotName, rootDir, fs, true);
}
@@ -492,7 +492,7 @@ public class TestFlushSnapshotFromClient {
}
// dump for debugging
- UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
List<SnapshotDescription> taken = admin.listSnapshots();
int takenSize = taken.size();
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
index bf26c69..a32c747 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
@@ -207,7 +207,7 @@ public class TestRestoreFlushSnapshotFromClient {
// Helpers
// ==========================================================================
private void logFSTree() throws IOException {
- UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ UTIL.getMiniHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
}
protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
index e15609a..dd7f18e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
@@ -422,7 +422,7 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
htdDisabled.addFamily(new HColumnDescriptor(FAM));
// Write the .tableinfo
- cluster.getMaster().getMasterFileSystem().createTableDescriptor(
+ cluster.getMaster().getMasterStorage().createTableDescriptor(
new TableDescriptor(htdDisabled), true);
List<HRegionInfo> disabledRegions =
TEST_UTIL.createMultiRegionsInMeta(conf, htdDisabled, SPLIT_KEYS);
[2/3] hbase git commit: HBASE-14439 Moved from MasterFileSystem to
MasterStorage abstract class for storage abstraction layer.
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
index d37159b..dbcb7c7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
@@ -31,12 +31,10 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.RegionLocator;
-import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.BulkReOpen;
import org.apache.hadoop.hbase.util.Bytes;
@@ -62,7 +60,7 @@ public final class MasterDDLOperationHelper {
List<HRegionInfo> regionInfoList,
final byte[] familyName,
boolean hasMob) throws IOException {
- final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+ final MasterStorage ms = env.getMasterServices().getMasterStorage();
if (LOG.isDebugEnabled()) {
LOG.debug("Removing family=" + Bytes.toString(familyName) + " from table=" + tableName);
}
@@ -71,7 +69,7 @@ public final class MasterDDLOperationHelper {
}
for (HRegionInfo hri : regionInfoList) {
// Delete the family directory in FS for all the regions one by one
- mfs.deleteFamilyFromFS(hri, familyName, hasMob);
+ ms.deleteFamilyFromStorage(hri, familyName, hasMob);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
index feb9228..9c373bd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
@@ -41,7 +41,8 @@ import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.master.MetricsSnapshot;
import org.apache.hadoop.hbase.master.RegionStates;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
@@ -333,12 +334,12 @@ public class RestoreSnapshotProcedure
if (!getTableName().isSystemTable()) {
// Table already exist. Check and update the region quota for this table namespace.
- final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+ final MasterStorage ms = env.getMasterServices().getMasterStorage();
SnapshotManifest manifest = SnapshotManifest.open(
env.getMasterConfiguration(),
- mfs.getFileSystem(),
- SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()),
- snapshot);
+ ms.getFileSystem(),
+ SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, ((LegacyPathIdentifier) ms
+ .getRootContainer()).path), snapshot);
int snapshotRegionCount = manifest.getRegionManifestsMap().size();
int tableRegionCount =
ProcedureSyncWait.getMasterQuotaManager(env).getRegionCountOfTable(tableName);
@@ -365,9 +366,9 @@ public class RestoreSnapshotProcedure
* @throws IOException
**/
private void restoreSnapshot(final MasterProcedureEnv env) throws IOException {
- MasterFileSystem fileSystemManager = env.getMasterServices().getMasterFileSystem();
+ MasterStorage fileSystemManager = env.getMasterServices().getMasterStorage();
FileSystem fs = fileSystemManager.getFileSystem();
- Path rootDir = fileSystemManager.getRootDir();
+ Path rootDir = ((LegacyPathIdentifier) fileSystemManager.getRootContainer()).path;
final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher();
LOG.info("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot));
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
index 40f5845..6e14f47 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
@@ -90,7 +90,7 @@ public final class MasterSnapshotVerifier {
* @param rootDir root directory of the hbase installation.
*/
public MasterSnapshotVerifier(MasterServices services, SnapshotDescription snapshot, Path rootDir) {
- this.fs = services.getMasterFileSystem().getFileSystem();
+ this.fs = services.getMasterStorage().getFileSystem();
this.services = services;
this.snapshot = snapshot;
this.rootDir = rootDir;
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index 3a5b974..a2c0ea0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -27,10 +27,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -50,8 +47,9 @@ import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.executor.ExecutorService;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.ipc.RpcServer;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.MetricsMaster;
@@ -88,7 +86,6 @@ import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.KeyLocker;
-import org.apache.hadoop.hbase.wal.WAL;
import org.apache.zookeeper.KeeperException;
/**
@@ -188,8 +185,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
throws IOException, UnsupportedOperationException {
this.master = master;
- this.rootDir = master.getMasterFileSystem().getRootDir();
- checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
+ this.rootDir = ((LegacyPathIdentifier) master.getMasterStorage().getRootContainer()).path;
+ checkSnapshotSupport(master.getConfiguration(), master.getMasterStorage());
this.coordinator = coordinator;
this.executorService = pool;
@@ -214,7 +211,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
// first create the snapshot root path and check to see if it exists
- FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ FileSystem fs = master.getMasterStorage().getFileSystem();
if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
// if there are no snapshots, return an empty list
@@ -274,8 +271,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
void resetTempDir() throws IOException {
// cleanup any existing snapshots.
Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
- if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) {
- if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
+ if (master.getMasterStorage().getFileSystem().exists(tmpdir)) {
+ if (!master.getMasterStorage().getFileSystem().delete(tmpdir, true)) {
LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
}
}
@@ -295,7 +292,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
String snapshotName = snapshot.getName();
// first create the snapshot description and check to see if it exists
- FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ FileSystem fs = master.getMasterStorage().getFileSystem();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
// Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
// just the "name" and it does not contains the "real" snapshot information
@@ -428,7 +425,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
*/
private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
throws HBaseSnapshotException {
- FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ FileSystem fs = master.getMasterStorage().getFileSystem();
Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
TableName snapshotTable =
TableName.valueOf(snapshot.getTable());
@@ -524,7 +521,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
// cleanup the working directory by trying to delete it from the fs.
Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
try {
- if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
+ if (!this.master.getMasterStorage().getFileSystem().delete(workingDir, true)) {
LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
ClientSnapshotDescriptionUtils.toString(snapshot));
}
@@ -662,7 +659,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
try {
final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
- FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ FileSystem fs = master.getMasterStorage().getFileSystem();
// check to see if the snapshot already exists
return fs.exists(snapshotDir);
} catch (IllegalArgumentException iae) {
@@ -765,7 +762,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
SnapshotDescription reqSnapshot,
final long nonceGroup,
final long nonce) throws IOException {
- FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ FileSystem fs = master.getMasterStorage().getFileSystem();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
// check if the snapshot exists
@@ -1045,12 +1042,12 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
* starting the master if there're snapshots present but the cleaners needed are missing.
* Otherwise we can end up with snapshot data loss.
* @param conf The {@link Configuration} object to use
- * @param mfs The MasterFileSystem to use
+ * @param ms The MasterFileSystem to use
* @throws IOException in case of file-system operation failure
* @throws UnsupportedOperationException in case cleaners are missing and
* there're snapshot in the system
*/
- private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
+ private void checkSnapshotSupport(final Configuration conf, final MasterStorage ms)
throws IOException, UnsupportedOperationException {
// Verify if snapshot is disabled by the user
String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
@@ -1067,8 +1064,9 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
if (cleaners != null) Collections.addAll(logCleaners, cleaners);
// check if an older version of snapshot directory was present
- Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
- FileSystem fs = mfs.getFileSystem();
+ Path oldSnapshotDir = new Path(((LegacyPathIdentifier) ms.getRootContainer()).path, HConstants
+ .OLD_SNAPSHOT_DIR_NAME);
+ FileSystem fs = ms.getFileSystem();
List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
if (ss != null && !ss.isEmpty()) {
LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
@@ -1109,7 +1107,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
// otherwise we end up with snapshot data loss.
if (!snapshotEnabled) {
LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
- Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
+ Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(((LegacyPathIdentifier) ms
+ .getRootContainer()).path);
if (fs.exists(snapshotDir)) {
FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
@@ -1126,8 +1125,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
IOException, UnsupportedOperationException {
this.master = master;
- this.rootDir = master.getMasterFileSystem().getRootDir();
- checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
+ this.rootDir = ((LegacyPathIdentifier) master.getMasterStorage().getRootContainer()).path;
+ checkSnapshotSupport(master.getConfiguration(), master.getMasterStorage());
// get the configuration for the coordinator
Configuration conf = master.getConfiguration();
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index a0e5b93..503f346 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.executor.EventType;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.MetricsSnapshot;
import org.apache.hadoop.hbase.master.SnapshotSentinel;
@@ -107,8 +108,8 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
this.snapshotManager = snapshotManager;
this.snapshotTable = TableName.valueOf(snapshot.getTable());
this.conf = this.master.getConfiguration();
- this.fs = this.master.getMasterFileSystem().getFileSystem();
- this.rootDir = this.master.getMasterFileSystem().getRootDir();
+ this.fs = this.master.getMasterStorage().getFileSystem();
+ this.rootDir = ((LegacyPathIdentifier) this.master.getMasterStorage().getRootContainer()).path;
this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
this.workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
this.monitor = new ForeignExceptionDispatcher(snapshot.getName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index ebbe5df..7f03435 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -31,15 +31,14 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Coprocessor;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableInfoMissingException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
/**
* Implementation of {@link TableDescriptors} that reads descriptors from the
@@ -63,7 +62,7 @@ import org.apache.hadoop.hbase.fs.MasterFileSystem;
public class FSTableDescriptors implements TableDescriptors {
private static final Log LOG = LogFactory.getLog(FSTableDescriptors.class);
- private final MasterFileSystem mfs;
+ private final MasterStorage<? extends StorageIdentifier> ms;
private final boolean fsreadonly;
private volatile boolean usecache;
@@ -103,17 +102,18 @@ public class FSTableDescriptors implements TableDescriptors {
*/
public FSTableDescriptors(final Configuration conf, final FileSystem fs,
final Path rootdir, final boolean fsreadonly, final boolean usecache) throws IOException {
- this(MasterFileSystem.open(conf, fs, rootdir, false), fsreadonly, usecache);
+ this(MasterStorage.open(conf, rootdir, false), fsreadonly, usecache);
}
- private FSTableDescriptors(final MasterFileSystem mfs, boolean fsreadonly, boolean usecache)
+ private FSTableDescriptors(final MasterStorage<? extends StorageIdentifier> ms, boolean
+ fsreadonly, boolean usecache)
throws IOException {
super();
- this.mfs = mfs;
+ this.ms = ms;
this.fsreadonly = fsreadonly;
this.usecache = usecache;
- this.metaTableDescriptor = HTableDescriptor.metaTableDescriptor(mfs.getConfiguration());
+ this.metaTableDescriptor = HTableDescriptor.metaTableDescriptor(ms.getConfiguration());
}
@Override
@@ -164,7 +164,7 @@ public class FSTableDescriptors implements TableDescriptors {
}
HTableDescriptor tdmt = null;
try {
- tdmt = mfs.getTableDescriptor(tablename);
+ tdmt = ms.getTableDescriptor(tablename);
} catch (NullPointerException e) {
LOG.debug("Exception during readTableDecriptor. Current table name = " + tablename, e);
} catch (TableInfoMissingException e) {
@@ -198,7 +198,7 @@ public class FSTableDescriptors implements TableDescriptors {
LOG.debug("Fetching table descriptors from the filesystem.");
boolean allvisited = true;
- for (TableName table: mfs.getTables()) {
+ for (TableName table: ms.getTables()) {
HTableDescriptor htd = null;
try {
htd = get(table);
@@ -239,7 +239,7 @@ public class FSTableDescriptors implements TableDescriptors {
@Override
public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
Map<String, HTableDescriptor> htds = new TreeMap<String, HTableDescriptor>();
- for (TableName table: mfs.getTables(name)) {
+ for (TableName table: ms.getTables(name)) {
HTableDescriptor htd = null;
try {
htd = get(table);
@@ -284,7 +284,7 @@ public class FSTableDescriptors implements TableDescriptors {
if (fsreadonly) {
throw new NotImplementedException("Cannot remove a table descriptor - in read only mode");
}
- mfs.deleteTable(tablename); // for test only??
+ ms.deleteTable(tablename); // for test only??
HTableDescriptor descriptor = this.cache.remove(tablename);
return descriptor;
}
@@ -299,7 +299,7 @@ public class FSTableDescriptors implements TableDescriptors {
if (fsreadonly) {
throw new NotImplementedException("Cannot update a table descriptor - in read only mode");
}
- mfs.updateTableDescriptor(td);
+ ms.updateTableDescriptor(td);
if (usecache) {
this.cache.put(td.getTableName(), td);
}
@@ -328,6 +328,6 @@ public class FSTableDescriptors implements TableDescriptors {
}
// forceCreation???
- return mfs.createTableDescriptor(htd, forceCreation);
+ return ms.createTableDescriptor(htd, forceCreation);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 662f20d..0eacc2b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -40,13 +40,8 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Vector;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.FutureTask;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
@@ -73,13 +68,12 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.fs.RegionStorage;
-import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.security.AccessDeniedException;
-import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.FSProtos;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -93,7 +87,6 @@ import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
-import com.google.common.base.Throwables;
import com.google.common.collect.Iterators;
import com.google.common.primitives.Ints;
@@ -1088,12 +1081,12 @@ public abstract class FSUtils {
int cfCountTotal = 0;
int cfFragTotal = 0;
- MasterFileSystem mfs = master.getMasterFileSystem();
- for (TableName table: mfs.getTables()) {
+ MasterStorage<? extends StorageIdentifier> ms = master.getMasterStorage();
+ for (TableName table: ms.getTables()) {
int cfCount = 0;
int cfFrag = 0;
- for (HRegionInfo hri: mfs.getRegions(table)) {
- RegionStorage rfs = mfs.getRegionStorage(hri);
+ for (HRegionInfo hri: ms.getRegions(table)) {
+ RegionStorage rfs = ms.getRegionStorage(hri);
final Collection<String> families = rfs.getFamilies();
for (String family: families) {
cfCount++;
@@ -1782,14 +1775,14 @@ public abstract class FSUtils {
throws IOException {
long startTime = EnvironmentEdgeManager.currentTime();
- MasterFileSystem mfs = MasterFileSystem.open(conf, false);
+ MasterStorage<? extends StorageIdentifier> ms = MasterStorage.open(conf, false);
Collection<HRegionInfo> hris;
if (desiredTable != null) {
- hris = mfs.getRegions(TableName.valueOf(desiredTable));
+ hris = ms.getRegions(TableName.valueOf(desiredTable));
} else {
hris = new ArrayList<HRegionInfo>();
- for (TableName tableName: mfs.getTables()) {
- hris.addAll(mfs.getRegions(tableName));
+ for (TableName tableName: ms.getTables()) {
+ hris.addAll(ms.getRegions(tableName));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 8a14cca..45c9989 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -110,7 +110,7 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.fs.RegionStorage;
import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
import org.apache.hadoop.hbase.fs.StorageIdentifier;
@@ -118,7 +118,6 @@ import org.apache.hadoop.hbase.fs.legacy.LegacyLayout;
import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.fs.legacy.LegacyRegionStorage;
import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.master.RegionState;
@@ -130,7 +129,6 @@ import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
import org.apache.hadoop.hbase.security.AccessDeniedException;
import org.apache.hadoop.hbase.security.UserProvider;
-import org.apache.hadoop.hbase.util.MetaUtils;
import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
@@ -240,7 +238,7 @@ public class HBaseFsck extends Configured implements Closeable {
// successful
private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
- private final MasterFileSystem mfs;
+ private final MasterStorage ms;
/***********
* Options
@@ -361,7 +359,7 @@ public class HBaseFsck extends Configured implements Closeable {
// Disable usage of meta replicas in hbck
getConf().setBoolean(HConstants.USE_META_REPLICAS, false);
- mfs = MasterFileSystem.open(getConf(), false);
+ ms = MasterStorage.open(getConf(), false);
errors = getErrorReporter(getConf());
this.executor = exec;
@@ -1095,14 +1093,14 @@ public class HBaseFsck extends Configured implements Closeable {
private void offlineReferenceFileRepair() throws IOException {
clearState();
LOG.info("Validating mapping using HDFS state");
- mfs.visitStoreFiles(new StoreFileVisitor() {
+ ms.visitStoreFiles(new StoreFileVisitor() {
@Override
public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
throws IOException {
if (errors != null) errors.progress();
if (!storeFile.isReference()) return;
- FileSystem fs = mfs.getFileSystem();
+ FileSystem fs = ms.getFileSystem();
Path path = storeFile.getPath();
Path referredToFile = StoreFileInfo.getReferredToFile(path);
if (fs.exists(referredToFile)) return; // good, expected
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java
index 4abf350..c3effc1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptorDefaultVersions.java
@@ -26,10 +26,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.AfterClass;
import org.junit.Before;
@@ -158,7 +157,7 @@ public class TestHColumnDescriptorDefaultVersions {
verifyHColumnDescriptor(expected, hcds, tableName, families);
// Verify descriptor from HDFS
- MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+ MasterStorage mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterStorage();
Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
HTableDescriptor td = LegacyTableDescriptor.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir);
hcds = td.getColumnFamilies();
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
index 2dca6b1..baaa14b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
@@ -230,7 +230,7 @@ public class TestNamespace {
TEST_UTIL.waitTableAvailable(desc.getTableName().getName(), 10000);
FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
assertTrue(fs.exists(
- new Path(master.getMasterFileSystem().getRootDir(),
+ new Path(master.getMasterStorage().getRootDir(),
new Path(HConstants.BASE_NAMESPACE_DIR,
new Path(nsName, desc.getTableName().getQualifierAsString())))));
assertEquals(1, admin.listTables().length);
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index f5f380f..9b4206c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -26,7 +26,6 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CategoryBasedTimeout;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -34,7 +33,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
@@ -44,7 +43,6 @@ import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -304,7 +302,7 @@ public class TestRestoreSnapshotFromClient {
}
private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
- MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+ MasterStorage mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterStorage();
final Set<String> families = new HashSet<String>();
mfs.visitStoreFiles(tableName, new StoreFileVisitor() {
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
index 2d4b4c9..565da24 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
import org.junit.After;
@@ -121,8 +120,8 @@ public class TestSnapshotCloneIndependence {
@Before
public void setup() throws Exception {
- fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
- rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ fs = UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
+ rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
admin = UTIL.getHBaseAdmin();
originalTableName = TableName.valueOf("test" + testName.getMethodName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
index fbd347a..702b80a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
@@ -221,10 +221,10 @@ public class TestSnapshotFromClient {
SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
// make sure its a valid snapshot
- FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
- Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
+ Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
LOG.debug("FS state after snapshot:");
- UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
SnapshotTestingUtils.confirmSnapshotValid(
ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM,
@@ -291,10 +291,10 @@ public class TestSnapshotFromClient {
SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
// make sure its a valid snapshot
- FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
- Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
+ Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
LOG.debug("FS state after snapshot:");
- UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ UTIL.getHBaseCluster().getMaster().getMasterStorage().logStorageState(LOG);
List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
List<byte[]> nonEmptyCfs = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java
index 1a774e5..e043290 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java
@@ -98,8 +98,8 @@ public class TestSnapshotMetadata {
setupConf(UTIL.getConfiguration());
UTIL.startMiniCluster(NUM_RS);
- fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
- rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ fs = UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
+ rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
index 0f0baff..cf08221 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableSnapshotScanner.java
@@ -58,7 +58,7 @@ public class TestTableSnapshotScanner {
public void setupCluster() throws Exception {
setupConf(UTIL.getConfiguration());
UTIL.startMiniCluster(NUM_REGION_SERVERS, true);
- rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
fs = rootDir.getFileSystem(UTIL.getConfiguration());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
index 3180c50..8b869a5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
@@ -280,7 +280,7 @@ public class TestBlockReorder {
"/" + targetRs.getServerName().toString()).toUri().getPath();
DistributedFileSystem mdfs = (DistributedFileSystem)
- hbm.getMaster().getMasterFileSystem().getFileSystem();
+ hbm.getMaster().getMasterStorage().getFileSystem();
int nbTest = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
index 314d6b3..c3c9597 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.mapreduce;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CategoryBasedTimeout;
@@ -63,7 +62,7 @@ public abstract class TableSnapshotInputFormatTestBase {
public void setupCluster() throws Exception {
setupConf(UTIL.getConfiguration());
UTIL.startMiniCluster(NUM_REGION_SERVERS, true);
- rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ rootDir = UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
fs = rootDir.getFileSystem(UTIL.getConfiguration());
}
@@ -127,7 +126,7 @@ public abstract class TableSnapshotInputFormatTestBase {
testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir);
- UTIL.getHBaseCluster().getMaster().getMasterFileSystem().visitStoreFiles(tableName,
+ UTIL.getHBaseCluster().getMaster().getMasterStorage().visitStoreFiles(tableName,
new StoreFileVisitor() {
@Override
public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
index c17d408..d109907 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
@@ -108,7 +108,7 @@ public class TestWALPlayer {
// replay the WAL, map table 1 to table 2
WAL log = cluster.getRegionServer(0).getWAL(null);
log.rollWriter();
- String walInputDir = new Path(cluster.getMaster().getMasterFileSystem()
+ String walInputDir = new Path(cluster.getMaster().getMasterStorage()
.getRootDir(), HConstants.HREGION_LOGDIR_NAME).toString();
Configuration configuration= TEST_UTIL.getConfiguration();
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index 56a8522..ee232d5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -103,7 +103,7 @@ public class MockNoopMasterServices implements MasterServices, Server {
}
@Override
- public MasterFileSystem getMasterFileSystem() {
+ public MasterFileSystem getMasterStorage() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 69ba38f..5e2053f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails;
import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
@@ -90,7 +90,7 @@ public class TestCatalogJanitor {
*/
class MockMasterServices extends MockNoopMasterServices {
private final ClusterConnection connection;
- private final MasterFileSystem mfs;
+ private final MasterStorage mfs;
private final AssignmentManager asm;
MockMasterServices(final HBaseTestingUtility htu) throws IOException {
@@ -133,7 +133,7 @@ public class TestCatalogJanitor {
FSUtils.setRootDir(getConfiguration(), rootdir);
Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
- this.mfs = new MasterFileSystem(this);
+ this.mfs = new MasterStorage(this);
this.asm = Mockito.mock(AssignmentManager.class);
}
@@ -143,7 +143,7 @@ public class TestCatalogJanitor {
}
@Override
- public MasterFileSystem getMasterFileSystem() {
+ public MasterStorage getMasterStorage() {
return this.mfs;
}
@@ -237,7 +237,7 @@ public class TestCatalogJanitor {
// remove the parent.
Result r = createResult(parent, splita, splitb);
// Add a reference under splitA directory so we don't clear out the parent.
- Path rootdir = services.getMasterFileSystem().getRootDir();
+ Path rootdir = services.getMasterStorage().getRootDir();
Path tabledir =
FSUtils.getTableDir(rootdir, htd.getTableName());
Path storedir = HStore.getStoreHomedir(tabledir, splita,
@@ -246,7 +246,7 @@ public class TestCatalogJanitor {
long now = System.currentTimeMillis();
// Reference name has this format: StoreFile#REF_NAME_PARSER
Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
- FileSystem fs = services.getMasterFileSystem().getFileSystem();
+ FileSystem fs = services.getMasterStorage().getFileSystem();
Path path = ref.write(fs, p);
assertTrue(fs.exists(path));
assertFalse(janitor.cleanParent(parent, r));
@@ -573,7 +573,7 @@ public class TestCatalogJanitor {
// remove the parent.
Result parentMetaRow = createResult(parent, splita, splitb);
FileSystem fs = FileSystem.get(htu.getConfiguration());
- Path rootdir = services.getMasterFileSystem().getRootDir();
+ Path rootdir = services.getMasterStorage().getRootDir();
// have to set the root directory since we use it in HFileDisposer to figure out to get to the
// archive directory. Otherwise, it just seems to pick the first root directory it can find (so
// the single test passes, but when the full suite is run, things get borked).
@@ -656,7 +656,7 @@ public class TestCatalogJanitor {
FileSystem fs = FileSystem.get(htu.getConfiguration());
- Path rootdir = services.getMasterFileSystem().getRootDir();
+ Path rootdir = services.getMasterStorage().getRootDir();
// have to set the root directory since we use it in HFileDisposer to figure out to get to the
// archive directory. Otherwise, it just seems to pick the first root directory it can find (so
// the single test passes, but when the full suite is run, things get borked).
@@ -701,7 +701,7 @@ public class TestCatalogJanitor {
private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
throws IOException {
// get the existing store files
- FileSystem fs = services.getMasterFileSystem().getFileSystem();
+ FileSystem fs = services.getMasterStorage().getFileSystem();
fs.mkdirs(storedir);
// create the store files in the parent
for (int i = 0; i < count; i++) {
@@ -741,7 +741,7 @@ public class TestCatalogJanitor {
final HTableDescriptor htd, final HRegionInfo parent,
final HRegionInfo daughter, final byte [] midkey, final boolean top)
throws IOException {
- Path rootdir = services.getMasterFileSystem().getRootDir();
+ Path rootdir = services.getMasterStorage().getRootDir();
Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
Path storedir = HStore.getStoreHomedir(tabledir, daughter,
htd.getColumnFamilies()[0].getName());
@@ -750,7 +750,7 @@ public class TestCatalogJanitor {
long now = System.currentTimeMillis();
// Reference name has this format: StoreFile#REF_NAME_PARSER
Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
- FileSystem fs = services.getMasterFileSystem().getFileSystem();
+ FileSystem fs = services.getMasterStorage().getFileSystem();
ref.write(fs, p);
return p;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 002438a..1583f33 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -101,7 +101,6 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
-import org.apache.hadoop.hbase.wal.FSHLogProvider;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.hadoop.hbase.wal.WALSplitter;
@@ -217,7 +216,7 @@ public class TestDistributedLogSplitting {
// turn off load balancing to prevent regions from moving around otherwise
// they will consume recovered.edits
master.balanceSwitch(false);
- FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ FileSystem fs = master.getMasterStorage().getFileSystem();
List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
@@ -867,7 +866,7 @@ public class TestDistributedLogSplitting {
});
int count = 0;
- FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ FileSystem fs = master.getMasterStorage().getFileSystem();
Path rootdir = FSUtils.getRootDir(conf);
Path tdir = FSUtils.getTableDir(rootdir, TableName.valueOf("disableTable"));
for (HRegionInfo hri : regions) {
@@ -1005,7 +1004,7 @@ public class TestDistributedLogSplitting {
startCluster(3);
final int NUM_LOG_LINES = 10000;
final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager();
- FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ FileSystem fs = master.getMasterStorage().getFileSystem();
final List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
HRegionServer hrs = findRSToKill(false, "table");
@@ -1124,7 +1123,7 @@ public class TestDistributedLogSplitting {
LOG.info("testDelayedDeleteOnFailure");
startCluster(1);
final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager();
- final FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ final FileSystem fs = master.getMasterStorage().getFileSystem();
final Path logDir = new Path(FSUtils.getRootDir(conf), "x");
fs.mkdirs(logDir);
ExecutorService executor = null;
@@ -1442,7 +1441,7 @@ public class TestDistributedLogSplitting {
final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
Table ht = installTable(zkw, "table", "family", 10);
try {
- FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ FileSystem fs = master.getMasterStorage().getFileSystem();
Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), TableName.valueOf("table"));
List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir);
long newSeqId = WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0), 1L, 1000L);
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
index 48e143d..3a692d9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
@@ -26,8 +26,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -58,7 +57,7 @@ public class TestMasterFileSystem {
@Test
public void testFsUriSetProperly() throws Exception {
HMaster master = UTIL.getMiniHBaseCluster().getMaster();
- MasterFileSystem fs = master.getMasterFileSystem();
+ MasterStorage fs = master.getMasterStorage();
Path masterRoot = FSUtils.getRootDir(fs.getConfiguration());
Path rootDir = FSUtils.getRootDir(fs.getFileSystem().getConf());
// make sure the fs and the found root dir have the same scheme
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
index 650171d..6725422 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
@@ -102,7 +102,7 @@ public class TestSnapshotFromMaster {
UTIL.startMiniCluster(NUM_RS);
fs = UTIL.getDFSCluster().getFileSystem();
master = UTIL.getMiniHBaseCluster().getMaster();
- rootDir = master.getMasterFileSystem().getRootDir();
+ rootDir = master.getMasterStorage().getRootDir();
archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index 3ea193b..d60a221 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -89,8 +89,8 @@ public class MasterProcedureTestingUtility {
public static void validateTableCreation(final HMaster master, final TableName tableName,
final HRegionInfo[] regions, boolean hasFamilyDirs, String... family) throws IOException {
// check filesystem
- final FileSystem fs = master.getMasterFileSystem().getFileSystem();
- final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
+ final FileSystem fs = master.getMasterStorage().getFileSystem();
+ final Path tableDir = FSUtils.getTableDir(master.getMasterStorage().getRootDir(), tableName);
assertTrue(fs.exists(tableDir));
FSUtils.logFileSystemState(fs, tableDir, LOG);
List<Path> allRegionDirs = FSUtils.getRegionDirs(fs, tableDir);
@@ -132,8 +132,8 @@ public class MasterProcedureTestingUtility {
public static void validateTableDeletion(
final HMaster master, final TableName tableName) throws IOException {
// check filesystem
- final FileSystem fs = master.getMasterFileSystem().getFileSystem();
- final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
+ final FileSystem fs = master.getMasterStorage().getFileSystem();
+ final Path tableDir = FSUtils.getTableDir(master.getMasterStorage().getRootDir(), tableName);
assertFalse(fs.exists(tableDir));
// check meta
@@ -208,7 +208,7 @@ public class MasterProcedureTestingUtility {
assertFalse(htd.hasFamily(deletedFamily.getBytes()));
// verify fs
- master.getMasterFileSystem().visitStoreFiles(tableName, new StoreFileVisitor() {
+ master.getMasterStorage().visitStoreFiles(tableName, new StoreFileVisitor() {
@Override
public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
index 9dd5065..91e8f2d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
@@ -122,7 +122,7 @@ public class TestMasterFailoverWithProcedures {
Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration();
Mockito.doReturn(true).when(backupMaster3).isActiveMaster();
final WALProcedureStore backupStore3 = new WALProcedureStore(firstMaster.getConfiguration(),
- firstMaster.getMasterFileSystem().getFileSystem(),
+ firstMaster.getMasterStorage().getFileSystem(),
((WALProcedureStore)masterStore).getLogDir(),
new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3));
// Abort Latch for the test store
@@ -202,7 +202,7 @@ public class TestMasterFailoverWithProcedures {
Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration();
Mockito.doReturn(true).when(backupMaster3).isActiveMaster();
final WALProcedureStore procStore2 = new WALProcedureStore(firstMaster.getConfiguration(),
- firstMaster.getMasterFileSystem().getFileSystem(),
+ firstMaster.getMasterStorage().getFileSystem(),
((WALProcedureStore)procStore).getLogDir(),
new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3));
@@ -504,6 +504,6 @@ public class TestMasterFailoverWithProcedures {
}
private Path getRootDir() {
- return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ return UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java
index c143b9a..78e6e7d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTableDescriptorModificationFromClient.java
@@ -32,12 +32,11 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.InvalidFamilyOperationException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -273,7 +272,7 @@ public class TestTableDescriptorModificationFromClient {
verifyTableDescriptor(htd, tableName, families);
// Verify descriptor from HDFS
- MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+ MasterStorage mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterStorage();
Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
HTableDescriptor td =
LegacyTableDescriptor.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir);
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
index 035b17b..817a2d2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.testclassification.MasterTests;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.MetricsMaster;
@@ -54,7 +54,7 @@ public class TestSnapshotManager {
MetricsMaster metrics = Mockito.mock(MetricsMaster.class);
ProcedureCoordinator coordinator = Mockito.mock(ProcedureCoordinator.class);
ExecutorService pool = Mockito.mock(ExecutorService.class);
- MasterFileSystem mfs = Mockito.mock(MasterFileSystem.class);
+ MasterStorage mfs = Mockito.mock(MasterStorage.class);
FileSystem fs;
{
try {
@@ -72,7 +72,7 @@ public class TestSnapshotManager {
throws IOException, KeeperException {
Mockito.reset(services);
Mockito.when(services.getConfiguration()).thenReturn(conf);
- Mockito.when(services.getMasterFileSystem()).thenReturn(mfs);
+ Mockito.when(services.getMasterStorage()).thenReturn(mfs);
Mockito.when(mfs.getFileSystem()).thenReturn(fs);
Mockito.when(mfs.getRootDir()).thenReturn(UTIL.getDataTestDir());
return new SnapshotManager(services, metrics, coordinator, pool);
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
index 49a290c..aa1c7ef 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
@@ -63,7 +63,7 @@ import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.RegionState;
@@ -81,7 +81,6 @@ import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.zookeeper.KeeperException;
import org.junit.After;
import org.junit.AfterClass;
@@ -189,7 +188,7 @@ public class TestNamespaceAuditor {
@Test
public void testValidQuotas() throws Exception {
boolean exceptionCaught = false;
- MasterFileSystem mfs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem();
+ MasterStorage mfs = UTIL.getHBaseCluster().getMaster().getMasterStorage();
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(prefix + "vq1")
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
index 6c851de..97238cf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
@@ -60,7 +60,7 @@ public class TestCompactSplitThread {
setupConf(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(NUM_RS);
fs = TEST_UTIL.getDFSCluster().getFileSystem();
- rootDir = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ rootDir = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterStorage().getRootDir();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
index 140b038..331ef7b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
@@ -35,15 +35,12 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.fs.FsContext;
+import org.apache.hadoop.hbase.fs.StorageContext;
import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.TestTableName;
import org.junit.After;
@@ -106,8 +103,8 @@ public class TestCorruptedRegionStoreFile {
// get the store file paths
storeFiles.clear();
- UTIL.getHBaseCluster().getMaster().getMasterFileSystem().visitStoreFiles(
- FsContext.DATA, tableName, new StoreFileVisitor() {
+ UTIL.getHBaseCluster().getMaster().getMasterStorage().visitStoreFiles(
+ StorageContext.DATA, tableName, new StoreFileVisitor() {
public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
throws IOException {
storeFiles.add(storeFile.getPath());
@@ -190,11 +187,11 @@ public class TestCorruptedRegionStoreFile {
// Helpers
// ==========================================================================
private FileSystem getFileSystem() {
- return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ return UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
}
private Path getRootDir() {
- return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ return UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
}
private void evictHFileCache(final Path hfile) throws Exception {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
index ee3d7d6..f824517 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
@@ -239,8 +239,8 @@ public class TestRegionMergeTransactionOnCluster {
PairOfSameType<HRegionInfo> p = MetaTableAccessor.getMergeRegions(mergedRegionResult);
HRegionInfo regionA = p.getFirst();
HRegionInfo regionB = p.getSecond();
- FileSystem fs = MASTER.getMasterFileSystem().getFileSystem();
- Path rootDir = MASTER.getMasterFileSystem().getRootDir();
+ FileSystem fs = MASTER.getMasterStorage().getFileSystem();
+ Path rootDir = MASTER.getMasterStorage().getRootDir();
Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
Path regionAdir = new Path(tabledir, regionA.getEncodedName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerRetriableFailure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerRetriableFailure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerRetriableFailure.java
index c89860c..508b5dc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerRetriableFailure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerRetriableFailure.java
@@ -120,11 +120,11 @@ public class TestScannerRetriableFailure {
// Helpers
// ==========================================================================
private FileSystem getFileSystem() {
- return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ return UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
}
private Path getRootDir() {
- return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ return UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
}
public void loadTable(final Table table, int numRows) throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index c980ae6..165acd0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -31,7 +31,6 @@ import java.io.InterruptedIOException;
import java.util.Collection;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -39,7 +38,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.CoordinatedStateManager;
@@ -57,7 +55,6 @@ import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.UnknownRegionException;
-import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
@@ -93,8 +90,6 @@ import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.HBaseFsck;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.hadoop.hbase.util.PairOfSameType;
import org.apache.hadoop.hbase.util.Threads;
@@ -769,7 +764,7 @@ public class TestSplitTransactionOnCluster {
try {
// Precondition: we created a table with no data, no store files.
printOutRegions(regionServer, "Initial regions: ");
- cluster.getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ cluster.getMaster().getMasterStorage().logStorageState(LOG);
List<StoreFileInfo> storefiles = getStoreFiles(tableName);
assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
@@ -793,7 +788,7 @@ public class TestSplitTransactionOnCluster {
assertTrue(daughters.size() == 2);
// check dirs
- cluster.getMaster().getMasterFileSystem().logFileSystemState(LOG);
+ cluster.getMaster().getMasterStorage().logStorageState(LOG);
List<StoreFileInfo> storefilesAfter = getStoreFiles(tableName);
assertEquals("Expected nothing but found " + storefilesAfter.toString(),
storefilesAfter.size(), 0);
@@ -949,7 +944,7 @@ public class TestSplitTransactionOnCluster {
SplitTransactionImpl st = new SplitTransactionImpl(regions.get(0), Bytes.toBytes("r3"));
st.prepare();
st.stepsBeforePONR(regionServer, regionServer, false);
- assertEquals(3, cluster.getMaster().getMasterFileSystem().getRegions(desc.getTableName()).size());
+ assertEquals(3, cluster.getMaster().getMasterStorage().getRegions(desc.getTableName()).size());
cluster.startRegionServer();
regionServer.kill();
// Before we check deadServerInProgress, we should ensure server is dead at master side.
@@ -965,7 +960,7 @@ public class TestSplitTransactionOnCluster {
AssignmentManager am = cluster.getMaster().getAssignmentManager();
assertEquals(am.getRegionStates().getRegionsInTransition().toString(), 0, am
.getRegionStates().getRegionsInTransition().size());
- assertEquals(1, cluster.getMaster().getMasterFileSystem().getRegions(desc.getTableName()).size());
+ assertEquals(1, cluster.getMaster().getMasterStorage().getRegions(desc.getTableName()).size());
} finally {
TESTING_UTIL.deleteTable(table);
}
@@ -1364,7 +1359,7 @@ public class TestSplitTransactionOnCluster {
private List<StoreFileInfo> getStoreFiles(TableName table) throws IOException {
final ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>();
- cluster.getMaster().getMasterFileSystem().visitStoreFiles(table, new StoreFileVisitor() {
+ cluster.getMaster().getMasterStorage().visitStoreFiles(table, new StoreFileVisitor() {
@Override
public void storeFile(HRegionInfo region, String family, StoreFileInfo storeFile)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index d8ceafb..d824d70 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -55,7 +55,7 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.fs.RegionStorage;
import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor;
import org.apache.hadoop.hbase.io.HFileLink;
@@ -162,7 +162,7 @@ public final class SnapshotTestingUtils {
public static void confirmSnapshotValid(HBaseTestingUtility testUtil,
HBaseProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
throws IOException {
- MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem();
+ MasterStorage mfs = testUtil.getHBaseCluster().getMaster().getMasterStorage();
confirmSnapshotValid(snapshotDescriptor, tableName, family,
mfs.getRootDir(), testUtil.getHBaseAdmin(), mfs.getFileSystem());
}
@@ -424,7 +424,7 @@ public final class SnapshotTestingUtils {
*/
public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
throws IOException {
- final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
+ final MasterStorage mfs = util.getHBaseCluster().getMaster().getMasterStorage();
final FileSystem fs = mfs.getFileSystem();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
@@ -854,7 +854,7 @@ public final class SnapshotTestingUtils {
public static void deleteArchiveDirectory(final HBaseTestingUtility util)
throws IOException {
// Ensure the archiver to be empty
- MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+ MasterStorage mfs = util.getMiniHBaseCluster().getMaster().getMasterStorage();
Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
mfs.getFileSystem().delete(archiveDir, true);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index ee0d0e2..2af6f6e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -231,8 +231,8 @@ public class TestExportSnapshot {
public void testSnapshotWithRefsExportFileSystemState() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
- Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
- FileSystem fs = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
+ Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
+ FileSystem fs = TEST_UTIL.getHBaseCluster().getMaster().getMasterStorage().getFileSystem();
SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("tableWithRefsV1",
@@ -346,7 +346,7 @@ public class TestExportSnapshot {
conf.setInt("mapreduce.map.maxattempts", 3);
}
// Export Snapshot
- Path sourceDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ Path sourceDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
int res = ExportSnapshot.innerMain(conf, new String[] {
"-snapshot", Bytes.toString(snapshotName),
"-copy-from", sourceDir.toString(),
@@ -427,7 +427,7 @@ public class TestExportSnapshot {
}
private Path getHdfsDestinationDir() {
- Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterStorage().getRootDir();
Path path = new Path(new Path(rootDir, "export-test"), "export-" + System.currentTimeMillis());
LOG.info("HDFS export destination path: " + path);
return path;
[3/3] hbase git commit: HBASE-14439 Moved from MasterFileSystem to
MasterStorage abstract class for storage abstraction layer.
Posted by bu...@apache.org.
HBASE-14439 Moved from MasterFileSystem to MasterStorage abstract class for storage abstraction layer.
Code compiles with 'mvn -DskipTests clean compile'.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dea247ec
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dea247ec
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dea247ec
Branch: refs/heads/hbase-14439
Commit: dea247ec44c3bef6a21a1e4cc8e9307198b09ba8
Parents: 2986c97
Author: Umesh Agashe <ua...@cloudera.com>
Authored: Fri Sep 23 16:40:39 2016 -0700
Committer: Sean Busbey <bu...@apache.org>
Committed: Mon Sep 26 11:50:56 2016 -0500
----------------------------------------------------------------------
.../SplitLogManagerCoordination.java | 4 +-
.../org/apache/hadoop/hbase/fs/FsContext.java | 28 --
.../hadoop/hbase/fs/MasterFileSystem.java | 346 -------------------
.../apache/hadoop/hbase/fs/MasterStorage.java | 270 +++++++++++++++
.../apache/hadoop/hbase/fs/RegionStorage.java | 62 +---
.../apache/hadoop/hbase/fs/StorageContext.java | 28 ++
.../hbase/fs/legacy/HFileArchiveUtil.java | 4 +-
.../hbase/fs/legacy/LegacyMasterFileSystem.java | 151 ++++++--
.../hadoop/hbase/master/CatalogJanitor.java | 10 +-
.../hbase/master/ClusterStatusPublisher.java | 2 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 29 +-
.../hadoop/hbase/master/MasterServices.java | 6 +-
.../hadoop/hbase/master/MasterWalManager.java | 22 +-
.../hadoop/hbase/master/RegionStates.java | 6 +-
.../procedure/CloneSnapshotProcedure.java | 31 +-
.../procedure/CreateNamespaceProcedure.java | 3 +-
.../master/procedure/CreateTableProcedure.java | 20 +-
.../procedure/DeleteNamespaceProcedure.java | 8 +-
.../master/procedure/DeleteTableProcedure.java | 25 +-
.../procedure/MasterDDLOperationHelper.java | 8 +-
.../procedure/RestoreSnapshotProcedure.java | 15 +-
.../master/snapshot/MasterSnapshotVerifier.java | 2 +-
.../hbase/master/snapshot/SnapshotManager.java | 43 ++-
.../master/snapshot/TakeSnapshotHandler.java | 5 +-
.../hadoop/hbase/util/FSTableDescriptors.java | 28 +-
.../org/apache/hadoop/hbase/util/FSUtils.java | 27 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 12 +-
.../TestHColumnDescriptorDefaultVersions.java | 5 +-
.../org/apache/hadoop/hbase/TestNamespace.java | 2 +-
.../client/TestRestoreSnapshotFromClient.java | 6 +-
.../client/TestSnapshotCloneIndependence.java | 5 +-
.../hbase/client/TestSnapshotFromClient.java | 12 +-
.../hbase/client/TestSnapshotMetadata.java | 4 +-
.../hbase/client/TestTableSnapshotScanner.java | 2 +-
.../hadoop/hbase/fs/TestBlockReorder.java | 2 +-
.../TableSnapshotInputFormatTestBase.java | 5 +-
.../hadoop/hbase/mapreduce/TestWALPlayer.java | 2 +-
.../hbase/master/MockNoopMasterServices.java | 2 +-
.../hadoop/hbase/master/TestCatalogJanitor.java | 22 +-
.../master/TestDistributedLogSplitting.java | 11 +-
.../hbase/master/TestMasterFileSystem.java | 5 +-
.../master/cleaner/TestSnapshotFromMaster.java | 2 +-
.../MasterProcedureTestingUtility.java | 10 +-
.../TestMasterFailoverWithProcedures.java | 6 +-
...stTableDescriptorModificationFromClient.java | 5 +-
.../master/snapshot/TestSnapshotManager.java | 6 +-
.../hbase/namespace/TestNamespaceAuditor.java | 5 +-
.../regionserver/TestCompactSplitThread.java | 2 +-
.../TestCorruptedRegionStoreFile.java | 13 +-
.../TestRegionMergeTransactionOnCluster.java | 4 +-
.../TestScannerRetriableFailure.java | 4 +-
.../TestSplitTransactionOnCluster.java | 15 +-
.../hbase/snapshot/SnapshotTestingUtils.java | 8 +-
.../hbase/snapshot/TestExportSnapshot.java | 8 +-
.../snapshot/TestFlushSnapshotFromClient.java | 20 +-
.../TestRestoreFlushSnapshotFromClient.java | 2 +-
.../hadoop/hbase/util/TestHBaseFsckOneRS.java | 2 +-
57 files changed, 658 insertions(+), 734 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
index 3afce96..3561330 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
@@ -42,11 +42,11 @@ import com.google.common.annotations.VisibleForTesting;
* <P>
* Methods required for task life circle: <BR>
* {@link #markRegionsRecovering(ServerName, Set)} mark regions for log replaying. Used by
- * {@link org.apache.hadoop.hbase.fs.MasterFileSystem} <BR>
+ * {@link org.apache.hadoop.hbase.fs.MasterStorage} <BR>
* {@link #removeRecoveringRegions(Set, Boolean)} make regions cleanup that previous were marked as
* recovering. Called after all tasks processed <BR>
* {@link #removeStaleRecoveringRegions(Set)} remove stale recovering. called by
- * {@link org.apache.hadoop.hbase.fs.MasterFileSystem} after Active Master is initialized <BR>
+ * {@link org.apache.hadoop.hbase.fs.MasterStorage} after Active Master is initialized <BR>
* {@link #getLastRecoveryTime()} required for garbage collector and should indicate when the last
* recovery has been made<BR>
* {@link #checkTaskStillAvailable(String)} Check that task is still there <BR>
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java
deleted file mode 100644
index ced095d..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.fs;
-
-public enum FsContext {
- TEMP,
- DATA,
- ARCHIVE,
- SNAPSHOT,
- SIDELINE,
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
deleted file mode 100644
index 4590a91..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.fs;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.ClusterId;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.legacy.LegacyMasterFileSystem;
-import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-
-@InterfaceAudience.Private
-public abstract class MasterFileSystem {
- private static Log LOG = LogFactory.getLog(MasterFileSystem.class);
-
- // Persisted unique cluster ID
- private ClusterId clusterId;
-
- private Configuration conf;
- private FileSystem fs;
- private Path rootDir;
-
- protected MasterFileSystem(Configuration conf, FileSystem fs, Path rootDir) {
- this.rootDir = rootDir;
- this.conf = conf;
- this.fs = fs;
- }
-
- public Configuration getConfiguration() { return conf; }
- public FileSystem getFileSystem() { return fs; }
- public Path getRootDir() { return rootDir; }
-
- // ==========================================================================
- // PUBLIC Interfaces - Visitors
- // ==========================================================================
- public interface NamespaceVisitor {
- void visitNamespace(String namespace) throws IOException;
- }
-
- public interface TableVisitor {
- void visitTable(TableName tableName) throws IOException;
- }
-
- public interface RegionVisitor {
- void visitRegion(HRegionInfo regionInfo) throws IOException;
- }
-
- // ==========================================================================
- // PUBLIC Methods - Namespace related
- // ==========================================================================
- public abstract void createNamespace(NamespaceDescriptor nsDescriptor) throws IOException;
- public abstract void deleteNamespace(String namespaceName) throws IOException;
- public abstract Collection<String> getNamespaces(FsContext ctx) throws IOException;
-
- public Collection<String> getNamespaces() throws IOException {
- return getNamespaces(FsContext.DATA);
- }
- // should return or get a NamespaceDescriptor? how is that different from HTD?
-
- // ==========================================================================
- // PUBLIC Methods - Table Descriptor related
- // ==========================================================================
- public HTableDescriptor getTableDescriptor(TableName tableName)
- throws IOException {
- return getTableDescriptor(FsContext.DATA, tableName);
- }
-
- public boolean createTableDescriptor(HTableDescriptor tableDesc, boolean force)
- throws IOException {
- return createTableDescriptor(FsContext.DATA, tableDesc, force);
- }
-
- public void updateTableDescriptor(HTableDescriptor tableDesc) throws IOException {
- updateTableDescriptor(FsContext.DATA, tableDesc);
- }
-
- public abstract HTableDescriptor getTableDescriptor(FsContext ctx, TableName tableName)
- throws IOException;
- public abstract boolean createTableDescriptor(FsContext ctx, HTableDescriptor tableDesc,
- boolean force) throws IOException;
- public abstract void updateTableDescriptor(FsContext ctx, HTableDescriptor tableDesc)
- throws IOException;
-
- // ==========================================================================
- // PUBLIC Methods - Table related
- // ==========================================================================
- public void deleteTable(TableName tableName) throws IOException {
- deleteTable(FsContext.DATA, tableName);
- }
-
- public Collection<TableName> getTables(String namespace) throws IOException {
- return getTables(FsContext.DATA, namespace);
- }
-
- public abstract void deleteTable(FsContext ctx, TableName tableName) throws IOException;
-
- public abstract Collection<TableName> getTables(FsContext ctx, String namespace)
- throws IOException;
-
- public Collection<TableName> getTables() throws IOException {
- ArrayList<TableName> tables = new ArrayList<TableName>();
- for (String ns: getNamespaces()) {
- tables.addAll(getTables(ns));
- }
- return tables;
- }
-
- // ==========================================================================
- // PUBLIC Methods - Table Region related
- // ==========================================================================
- public void deleteRegion(HRegionInfo regionInfo) throws IOException {
- RegionStorage.destroy(conf, regionInfo);
- }
-
- public Collection<HRegionInfo> getRegions(TableName tableName) throws IOException {
- return getRegions(FsContext.DATA, tableName);
- }
-
- public abstract Collection<HRegionInfo> getRegions(FsContext ctx, TableName tableName)
- throws IOException;
-
- // TODO: Move in HRegionStorage
- public void deleteFamilyFromFS(HRegionInfo regionInfo, byte[] familyName, boolean hasMob)
- throws IOException {
- getRegionStorage(regionInfo).deleteFamily(Bytes.toString(familyName), hasMob);
- }
-
- public RegionStorage getRegionStorage(HRegionInfo regionInfo) throws IOException {
- return RegionStorage.open(conf, regionInfo, false);
- }
-
- // ==========================================================================
- // PUBLIC Methods - visitors
- // ==========================================================================
- public void visitStoreFiles(StoreFileVisitor visitor)
- throws IOException {
- visitStoreFiles(FsContext.DATA, visitor);
- }
-
- public void visitStoreFiles(String namespace, StoreFileVisitor visitor)
- throws IOException {
- visitStoreFiles(FsContext.DATA, namespace, visitor);
- }
-
- public void visitStoreFiles(TableName table, StoreFileVisitor visitor)
- throws IOException {
- visitStoreFiles(FsContext.DATA, table, visitor);
- }
-
- public void visitStoreFiles(FsContext ctx, StoreFileVisitor visitor)
- throws IOException {
- for (String namespace: getNamespaces()) {
- visitStoreFiles(ctx, namespace, visitor);
- }
- }
-
- public void visitStoreFiles(FsContext ctx, String namespace, StoreFileVisitor visitor)
- throws IOException {
- for (TableName tableName: getTables(namespace)) {
- visitStoreFiles(ctx, tableName, visitor);
- }
- }
-
- public void visitStoreFiles(FsContext ctx, TableName table, StoreFileVisitor visitor)
- throws IOException {
- for (HRegionInfo hri: getRegions(ctx, table)) {
- RegionStorage.open(conf, hri, false).visitStoreFiles(visitor);
- }
- }
-
- // ==========================================================================
- // PUBLIC Methods - bootstrap
- // ==========================================================================
- public abstract Path getTempDir();
-
- public void logFileSystemState(Log log) throws IOException {
- FSUtils.logFileSystemState(getFileSystem(), getRootDir(), LOG);
- }
-
- /**
- * @return The unique identifier generated for this cluster
- */
- public ClusterId getClusterId() {
- return clusterId;
- }
-
- protected void bootstrap() throws IOException {
- // check if the root directory exists
- createInitialLayout(getRootDir(), conf, this.fs);
-
- // check if temp directory exists and clean it
- startupCleanup();
- }
-
- protected abstract void bootstrapMeta() throws IOException;
- protected abstract void startupCleanup() throws IOException;
-
- /**
- * Create initial layout in filesystem.
- * <ol>
- * <li>Check if the meta region exists and is readable, if not create it.
- * Create hbase.version and the hbase:meta directory if not one.
- * </li>
- * <li>Create a log archive directory for RS to put archived logs</li>
- * </ol>
- * Idempotent.
- */
- private void createInitialLayout(final Path rd, final Configuration c, final FileSystem fs)
- throws IOException {
- // If FS is in safe mode wait till out of it.
- FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
-
- boolean isSecurityEnabled = "kerberos".equalsIgnoreCase(c.get("hbase.security.authentication"));
- FsPermission rootDirPerms = new FsPermission(c.get("hbase.rootdir.perms", "700"));
-
- // Filesystem is good. Go ahead and check for hbase.rootdir.
- try {
- if (!fs.exists(rd)) {
- if (isSecurityEnabled) {
- fs.mkdirs(rd, rootDirPerms);
- } else {
- fs.mkdirs(rd);
- }
- // DFS leaves safe mode with 0 DNs when there are 0 blocks.
- // We used to handle this by checking the current DN count and waiting until
- // it is nonzero. With security, the check for datanode count doesn't work --
- // it is a privileged op. So instead we adopt the strategy of the jobtracker
- // and simply retry file creation during bootstrap indefinitely. As soon as
- // there is one datanode it will succeed. Permission problems should have
- // already been caught by mkdirs above.
- FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
- 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
- HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
- } else {
- if (!fs.isDirectory(rd)) {
- throw new IllegalArgumentException(rd.toString() + " is not a directory");
- }
- if (isSecurityEnabled && !rootDirPerms.equals(fs.getFileStatus(rd).getPermission())) {
- // check whether the permission match
- LOG.warn("Found rootdir permissions NOT matching expected \"hbase.rootdir.perms\" for "
- + "rootdir=" + rd.toString() + " permissions=" + fs.getFileStatus(rd).getPermission()
- + " and \"hbase.rootdir.perms\" configured as "
- + c.get("hbase.rootdir.perms", "700") + ". Automatically setting the permissions. You"
- + " can change the permissions by setting \"hbase.rootdir.perms\" in hbase-site.xml "
- + "and restarting the master");
- fs.setPermission(rd, rootDirPerms);
- }
- // as above
- FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
- 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
- HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
- }
- } catch (DeserializationException de) {
- LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
- IOException ioe = new IOException();
- ioe.initCause(de);
- throw ioe;
- } catch (IllegalArgumentException iae) {
- LOG.fatal("Please fix invalid configuration for "
- + HConstants.HBASE_DIR + " " + rd.toString(), iae);
- throw iae;
- }
- // Make sure cluster ID exists
- if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
- HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
- FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
- }
- clusterId = FSUtils.getClusterId(fs, rd);
-
- // Make sure the meta region exists!
- bootstrapMeta();
- }
-
- // ==========================================================================
- // PUBLIC
- // ==========================================================================
- public static MasterFileSystem open(Configuration conf, boolean bootstrap)
- throws IOException {
- return open(conf, FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), bootstrap);
- }
-
- public static MasterFileSystem open(Configuration conf, FileSystem fs,
- Path rootDir, boolean bootstrap) throws IOException {
- // Cover both bases, the old way of setting default fs and the new.
- // We're supposed to run on 0.20 and 0.21 anyways.
- fs = rootDir.getFileSystem(conf);
- FSUtils.setFsDefault(conf, new Path(fs.getUri()));
- // make sure the fs has the same conf
- fs.setConf(conf);
-
- MasterFileSystem mfs = getInstance(conf, fs, rootDir);
- if (bootstrap) {
- mfs.bootstrap();
- }
- HFileSystem.addLocationsOrderInterceptor(conf);
- return mfs;
- }
-
- private static MasterFileSystem getInstance(Configuration conf, final FileSystem fs,
- Path rootDir) throws IOException {
- String fsType = conf.get("hbase.fs.layout.type", "legacy").toLowerCase();
- switch (fsType) {
- case "legacy":
- return new LegacyMasterFileSystem(conf, fs, rootDir);
- default:
- throw new IOException("Invalid filesystem type " + fsType);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java
new file mode 100644
index 0000000..f1d06d3
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java
@@ -0,0 +1,270 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.ClusterId;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.legacy.LegacyMasterFileSystem;
+import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+@InterfaceAudience.Private
+public abstract class MasterStorage<IDENTIFIER extends StorageIdentifier> {
+ private static Log LOG = LogFactory.getLog(MasterStorage.class);
+
+ // Persisted unique cluster ID
+ private ClusterId clusterId;
+
+ private Configuration conf;
+ private FileSystem fs; // TODO: definitely remove
+ private IDENTIFIER rootContainer;
+
+ protected MasterStorage(Configuration conf, FileSystem fs, IDENTIFIER rootContainer) {
+ this.rootContainer = rootContainer;
+ this.conf = conf;
+ this.fs = fs;
+ }
+
+ public Configuration getConfiguration() { return conf; }
+ public FileSystem getFileSystem() { return fs; } // TODO: definitely remove
+ public IDENTIFIER getRootContainer() { return rootContainer; }
+
+ // ==========================================================================
+ // PUBLIC Interfaces - Visitors
+ // ==========================================================================
+ public interface NamespaceVisitor {
+ void visitNamespace(String namespace) throws IOException;
+ }
+
+ public interface TableVisitor {
+ void visitTable(TableName tableName) throws IOException;
+ }
+
+ public interface RegionVisitor {
+ void visitRegion(HRegionInfo regionInfo) throws IOException;
+ }
+
+ // ==========================================================================
+ // PUBLIC Methods - Namespace related
+ // ==========================================================================
+ public abstract void createNamespace(NamespaceDescriptor nsDescriptor) throws IOException;
+ public abstract void deleteNamespace(String namespaceName) throws IOException;
+ public abstract Collection<String> getNamespaces(StorageContext ctx) throws IOException;
+
+ public Collection<String> getNamespaces() throws IOException {
+ return getNamespaces(StorageContext.DATA);
+ }
+ // should return or get a NamespaceDescriptor? how is that different from HTD?
+
+ // ==========================================================================
+ // PUBLIC Methods - Table Descriptor related
+ // ==========================================================================
+ public HTableDescriptor getTableDescriptor(TableName tableName)
+ throws IOException {
+ return getTableDescriptor(StorageContext.DATA, tableName);
+ }
+
+ public boolean createTableDescriptor(HTableDescriptor tableDesc, boolean force)
+ throws IOException {
+ return createTableDescriptor(StorageContext.DATA, tableDesc, force);
+ }
+
+ public void updateTableDescriptor(HTableDescriptor tableDesc) throws IOException {
+ updateTableDescriptor(StorageContext.DATA, tableDesc);
+ }
+
+ public abstract HTableDescriptor getTableDescriptor(StorageContext ctx, TableName tableName)
+ throws IOException;
+ public abstract boolean createTableDescriptor(StorageContext ctx, HTableDescriptor tableDesc,
+ boolean force) throws IOException;
+ public abstract void updateTableDescriptor(StorageContext ctx, HTableDescriptor tableDesc)
+ throws IOException;
+
+ // ==========================================================================
+ // PUBLIC Methods - Table related
+ // ==========================================================================
+ public void deleteTable(TableName tableName) throws IOException {
+ deleteTable(StorageContext.DATA, tableName);
+ }
+
+ public Collection<TableName> getTables(String namespace) throws IOException {
+ return getTables(StorageContext.DATA, namespace);
+ }
+
+ public abstract void deleteTable(StorageContext ctx, TableName tableName) throws IOException;
+
+ public abstract Collection<TableName> getTables(StorageContext ctx, String namespace)
+ throws IOException;
+
+ public Collection<TableName> getTables() throws IOException {
+ ArrayList<TableName> tables = new ArrayList<TableName>();
+ for (String ns: getNamespaces()) {
+ tables.addAll(getTables(ns));
+ }
+ return tables;
+ }
+
+ // ==========================================================================
+ // PUBLIC Methods - Table Region related
+ // ==========================================================================
+ public void deleteRegion(HRegionInfo regionInfo) throws IOException {
+ RegionStorage.destroy(conf, regionInfo);
+ }
+
+ public Collection<HRegionInfo> getRegions(TableName tableName) throws IOException {
+ return getRegions(StorageContext.DATA, tableName);
+ }
+
+ public abstract Collection<HRegionInfo> getRegions(StorageContext ctx, TableName tableName)
+ throws IOException;
+
+ // TODO: Move in HRegionStorage
+ public void deleteFamilyFromStorage(HRegionInfo regionInfo, byte[] familyName, boolean hasMob)
+ throws IOException {
+ getRegionStorage(regionInfo).deleteFamily(Bytes.toString(familyName), hasMob);
+ }
+
+ public RegionStorage getRegionStorage(HRegionInfo regionInfo) throws IOException {
+ return RegionStorage.open(conf, regionInfo, false);
+ }
+
+ // ==========================================================================
+ // PUBLIC Methods - visitors
+ // ==========================================================================
+ public void visitStoreFiles(StoreFileVisitor visitor)
+ throws IOException {
+ visitStoreFiles(StorageContext.DATA, visitor);
+ }
+
+ public void visitStoreFiles(String namespace, StoreFileVisitor visitor)
+ throws IOException {
+ visitStoreFiles(StorageContext.DATA, namespace, visitor);
+ }
+
+ public void visitStoreFiles(TableName table, StoreFileVisitor visitor)
+ throws IOException {
+ visitStoreFiles(StorageContext.DATA, table, visitor);
+ }
+
+ public void visitStoreFiles(StorageContext ctx, StoreFileVisitor visitor)
+ throws IOException {
+ for (String namespace: getNamespaces()) {
+ visitStoreFiles(ctx, namespace, visitor);
+ }
+ }
+
+ public void visitStoreFiles(StorageContext ctx, String namespace, StoreFileVisitor visitor)
+ throws IOException {
+ for (TableName tableName: getTables(namespace)) {
+ visitStoreFiles(ctx, tableName, visitor);
+ }
+ }
+
+ public void visitStoreFiles(StorageContext ctx, TableName table, StoreFileVisitor visitor)
+ throws IOException {
+ for (HRegionInfo hri: getRegions(ctx, table)) {
+ RegionStorage.open(conf, hri, false).visitStoreFiles(visitor);
+ }
+ }
+
+ // ==========================================================================
+ // PUBLIC Methods - bootstrap
+ // ==========================================================================
+ public abstract IDENTIFIER getTempContainer();
+
+ public abstract void logStorageState(Log log) throws IOException;
+
+ /**
+ * @return The unique identifier generated for this cluster
+ */
+ public ClusterId getClusterId() {
+ return clusterId;
+ }
+
+ /**
+ * Bootstrap MasterStorage
+ * @throws IOException
+ */
+ protected void bootstrap() throws IOException {
+ // Initialize
+ clusterId = startup();
+
+ // Make sure the meta region exists!
+ bootstrapMeta();
+
+ // check if temp directory exists and clean it
+ startupCleanup();
+ }
+
+ protected abstract ClusterId startup() throws IOException;
+ protected abstract void bootstrapMeta() throws IOException;
+ protected abstract void startupCleanup() throws IOException;
+
+ // ==========================================================================
+ // PUBLIC
+ // ==========================================================================
+ public static MasterStorage open(Configuration conf, boolean bootstrap)
+ throws IOException {
+ return open(conf, FSUtils.getRootDir(conf), bootstrap);
+ }
+
+ public static MasterStorage open(Configuration conf, Path rootDir, boolean bootstrap)
+ throws IOException {
+ // Cover both bases, the old way of setting default fs and the new.
+ // We're supposed to run on 0.20 and 0.21 anyways.
+ FileSystem fs = rootDir.getFileSystem(conf);
+ FSUtils.setFsDefault(conf, new Path(fs.getUri()));
+ // make sure the fs has the same conf
+ fs.setConf(conf);
+
+ MasterStorage ms = getInstance(conf, fs, rootDir);
+ if (bootstrap) {
+ ms.bootstrap();
+ }
+ HFileSystem.addLocationsOrderInterceptor(conf);
+ return ms;
+ }
+
+ private static MasterStorage getInstance(Configuration conf, final FileSystem fs,
+ Path rootDir) throws IOException {
+ String storageType = conf.get("hbase.storage.type", "legacy").toLowerCase();
+ switch (storageType) {
+ case "legacy":
+ return new LegacyMasterFileSystem(conf, fs, new LegacyPathIdentifier(rootDir));
+ default:
+ throw new IOException("Invalid filesystem type " + storageType);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
index 143abad..d15d1ee 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/RegionStorage.java
@@ -20,20 +20,15 @@
package org.apache.hadoop.hbase.fs;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.ClusterId;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.fs.legacy.LegacyRegionStorage;
@@ -42,63 +37,14 @@ import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import java.util.Map;
-import java.util.UUID;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.FSUtilsWithRetries;
-import org.apache.hadoop.hbase.fs.FsContext;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.MetaUtils;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.fs.HFileSystem;
+
import org.apache.hadoop.hbase.regionserver.*;
-import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSHDFSUtils;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
@InterfaceAudience.Private
public abstract class RegionStorage<IDENTIFIER extends StorageIdentifier> {
@@ -331,12 +277,12 @@ public abstract class RegionStorage<IDENTIFIER extends StorageIdentifier> {
private static RegionStorage getInstance(Configuration conf, FileSystem fs,
StorageIdentifier rootContainer, StorageIdentifier regionContainer) throws IOException {
- String fsType = conf.get("hbase.fs.storage.type", "legacy").toLowerCase();
- switch (fsType) {
+ String storageType = conf.get("hbase.storage.type", "legacy").toLowerCase();
+ switch (storageType) {
case "legacy":
return new LegacyRegionStorage(conf, fs, (LegacyPathIdentifier)rootContainer, (LegacyPathIdentifier)regionContainer);
default:
- throw new IOException("Invalid filesystem type " + fsType);
+ throw new IOException("Invalid filesystem type " + storageType);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/StorageContext.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/StorageContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/StorageContext.java
new file mode 100644
index 0000000..cc324a9
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/StorageContext.java
@@ -0,0 +1,28 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+public enum StorageContext {
+ TEMP,
+ DATA,
+ ARCHIVE,
+ SNAPSHOT,
+ SIDELINE,
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
index 2fbaa92..e422861 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/HFileArchiveUtil.java
@@ -133,7 +133,7 @@ public class HFileArchiveUtil {
/**
* Get the full path to the archive directory on the configured
- * {@link org.apache.hadoop.hbase.fs.MasterFileSystem}
+ * {@link org.apache.hadoop.hbase.fs.MasterStorage}
* @param conf to look for archive directory name and root directory. Cannot be null. Notes for
* testing: requires a FileSystem root directory to be specified.
* @return the full {@link Path} to the archive directory, as defined by the configuration
@@ -145,7 +145,7 @@ public class HFileArchiveUtil {
/**
* Get the full path to the archive directory on the configured
- * {@link org.apache.hadoop.hbase.fs.MasterFileSystem}
+ * {@link org.apache.hadoop.hbase.fs.MasterStorage}
* @param rootdir {@link Path} to the root directory where hbase files are stored (for building
* the archive path)
* @return the full {@link Path} to the archive directory, as defined by the configuration
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
index 33b9eb4..2db2154 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
@@ -32,15 +32,13 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.fs.FsContext;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.StorageContext;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.MetaUtils;
@@ -48,7 +46,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.backup.HFileArchiver;
@InterfaceAudience.Private
-public class LegacyMasterFileSystem extends MasterFileSystem {
+public class LegacyMasterFileSystem extends MasterStorage<LegacyPathIdentifier> {
private static final Log LOG = LogFactory.getLog(LegacyMasterFileSystem.class);
private final Path sidelineDir;
@@ -59,16 +57,16 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
private final Path dataDir;
private final Path tmpDir;
- public LegacyMasterFileSystem(Configuration conf, FileSystem fs, Path rootDir) {
+ public LegacyMasterFileSystem(Configuration conf, FileSystem fs, LegacyPathIdentifier rootDir) {
super(conf, fs, rootDir);
// base directories
- this.sidelineDir = LegacyLayout.getSidelineDir(rootDir);
- this.snapshotDir = LegacyLayout.getSnapshotDir(rootDir);
- this.archiveDir = LegacyLayout.getArchiveDir(rootDir);
+ this.sidelineDir = LegacyLayout.getSidelineDir(rootDir.path);
+ this.snapshotDir = LegacyLayout.getSnapshotDir(rootDir.path);
+ this.archiveDir = LegacyLayout.getArchiveDir(rootDir.path);
this.archiveDataDir = LegacyLayout.getDataDir(this.archiveDir);
- this.dataDir = LegacyLayout.getDataDir(rootDir);
- this.tmpDir = LegacyLayout.getTempDir(rootDir);
+ this.dataDir = LegacyLayout.getDataDir(rootDir.path);
+ this.tmpDir = LegacyLayout.getTempDir(rootDir.path);
this.tmpDataDir = LegacyLayout.getDataDir(this.tmpDir);
}
@@ -76,12 +74,12 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
// PUBLIC Methods - Namespace related
// ==========================================================================
public void createNamespace(NamespaceDescriptor nsDescriptor) throws IOException {
- getFileSystem().mkdirs(getNamespaceDir(FsContext.DATA, nsDescriptor.getName()));
+ getFileSystem().mkdirs(getNamespaceDir(StorageContext.DATA, nsDescriptor.getName()));
}
public void deleteNamespace(String namespaceName) throws IOException {
FileSystem fs = getFileSystem();
- Path nsDir = getNamespaceDir(FsContext.DATA, namespaceName);
+ Path nsDir = getNamespaceDir(StorageContext.DATA, namespaceName);
try {
for (FileStatus status : fs.listStatus(nsDir)) {
@@ -98,7 +96,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
}
}
- public Collection<String> getNamespaces(FsContext ctx) throws IOException {
+ public Collection<String> getNamespaces(StorageContext ctx) throws IOException {
FileStatus[] stats = FSUtils.listStatus(getFileSystem(), getNamespaceDir(ctx));
if (stats == null) return Collections.emptyList();
@@ -115,20 +113,20 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
// PUBLIC Methods - Table Descriptor related
// ==========================================================================s
@Override
- public boolean createTableDescriptor(FsContext ctx, HTableDescriptor tableDesc, boolean force)
+ public boolean createTableDescriptor(StorageContext ctx, HTableDescriptor tableDesc, boolean force)
throws IOException {
return LegacyTableDescriptor.createTableDescriptor(getFileSystem(),
getTableDir(ctx, tableDesc.getTableName()), tableDesc, force);
}
@Override
- public void updateTableDescriptor(FsContext ctx, HTableDescriptor tableDesc) throws IOException {
+ public void updateTableDescriptor(StorageContext ctx, HTableDescriptor tableDesc) throws IOException {
LegacyTableDescriptor.updateTableDescriptor(getFileSystem(),
getTableDir(ctx, tableDesc.getTableName()), tableDesc);
}
@Override
- public HTableDescriptor getTableDescriptor(FsContext ctx, TableName tableName)
+ public HTableDescriptor getTableDescriptor(StorageContext ctx, TableName tableName)
throws IOException {
return LegacyTableDescriptor.getTableDescriptorFromFs(
getFileSystem(), getTableDir(ctx, tableName));
@@ -138,7 +136,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
// PUBLIC Methods - Table related
// ==========================================================================
@Override
- public void deleteTable(FsContext ctx, TableName tableName) throws IOException {
+ public void deleteTable(StorageContext ctx, TableName tableName) throws IOException {
Path tableDir = getTableDir(ctx, tableName);
if (!FSUtils.deleteDirectory(getFileSystem(), tableDir)) {
throw new IOException("Failed delete of " + tableName);
@@ -146,7 +144,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
}
@Override
- public Collection<TableName> getTables(FsContext ctx, String namespace)
+ public Collection<TableName> getTables(StorageContext ctx, String namespace)
throws IOException {
FileStatus[] stats = FSUtils.listStatus(getFileSystem(),
getNamespaceDir(ctx, namespace), new FSUtils.UserTableDirFilter(getFileSystem()));
@@ -163,7 +161,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
// PUBLIC Methods - Table Regions related
// ==========================================================================
@Override
- public Collection<HRegionInfo> getRegions(FsContext ctx, TableName tableName)
+ public Collection<HRegionInfo> getRegions(StorageContext ctx, TableName tableName)
throws IOException {
FileStatus[] stats = FSUtils.listStatus(getFileSystem(),
getTableDir(ctx, tableName), new FSUtils.RegionDirFilter(getFileSystem()));
@@ -188,10 +186,96 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
// ==========================================================================
// PROTECTED Methods - Bootstrap
// ==========================================================================
+
+ /**
+ * Create initial layout in filesystem.
+ * <ol>
+ * <li>Check if the meta region exists and is readable, if not create it.
+ * Create hbase.version and the hbase:meta directory if not one.
+ * </li>
+ * <li>Create a log archive directory for RS to put archived logs</li>
+ * </ol>
+ * Idempotent.
+ * @throws IOException
+ */
+ @Override
+ protected ClusterId startup() throws IOException {
+ Configuration c = getConfiguration();
+ Path rc = ((LegacyPathIdentifier)getRootContainer()).path;
+ FileSystem fs = getFileSystem();
+
+ // If FS is in safe mode wait till out of it.
+ FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
+
+ boolean isSecurityEnabled = "kerberos".equalsIgnoreCase(c.get("hbase.security.authentication"));
+ FsPermission rootDirPerms = new FsPermission(c.get("hbase.rootdir.perms", "700"));
+
+ // Filesystem is good. Go ahead and check for hbase.rootdir.
+ try {
+ if (!fs.exists(rc)) {
+ if (isSecurityEnabled) {
+ fs.mkdirs(rc, rootDirPerms);
+ } else {
+ fs.mkdirs(rc);
+ }
+ // DFS leaves safe mode with 0 DNs when there are 0 blocks.
+ // We used to handle this by checking the current DN count and waiting until
+ // it is nonzero. With security, the check for datanode count doesn't work --
+ // it is a privileged op. So instead we adopt the strategy of the jobtracker
+ // and simply retry file creation during bootstrap indefinitely. As soon as
+ // there is one datanode it will succeed. Permission problems should have
+ // already been caught by mkdirs above.
+ FSUtils.setVersion(fs, rc, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+ 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
+ HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
+ } else {
+ if (!fs.isDirectory(rc)) {
+ throw new IllegalArgumentException(rc.toString() + " is not a directory");
+ }
+ if (isSecurityEnabled && !rootDirPerms.equals(fs.getFileStatus(rc).getPermission())) {
+ // check whether the permission match
+ LOG.warn("Found rootdir permissions NOT matching expected \"hbase.rootdir.perms\" for "
+ + "rootdir=" + rc.toString() + " permissions=" + fs.getFileStatus(rc).getPermission()
+ + " and \"hbase.rootdir.perms\" configured as "
+ + c.get("hbase.rootdir.perms", "700") + ". Automatically setting the permissions. You"
+ + " can change the permissions by setting \"hbase.rootdir.perms\" in hbase-site.xml "
+ + "and restarting the master");
+ fs.setPermission(rc, rootDirPerms);
+ }
+ // as above
+ FSUtils.checkVersion(fs, rc, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+ 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
+ HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
+ }
+ } catch (DeserializationException de) {
+ LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
+ IOException ioe = new IOException();
+ ioe.initCause(de);
+ throw ioe;
+ } catch (IllegalArgumentException iae) {
+ LOG.fatal("Please fix invalid configuration for "
+ + HConstants.HBASE_DIR + " " + rc.toString(), iae);
+ throw iae;
+ }
+ // Make sure cluster ID exists
+ if (!FSUtils.checkClusterIdExists(fs, rc, c.getInt(
+ HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
+ FSUtils.setClusterId(fs, rc, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10
+ * 1000));
+ }
+ return FSUtils.getClusterId(fs, rc);
+ }
+
+ @Override
+ public void logStorageState(Log log) throws IOException {
+ FSUtils.logFileSystemState(getFileSystem(), ((LegacyPathIdentifier)getRootContainer()).path,
+ LOG);
+ }
+
@Override
protected void bootstrapMeta() throws IOException {
// TODO ask RegionStorage
- if (!FSUtils.metaRegionExists(getFileSystem(), getRootDir())) {
+ if (!FSUtils.metaRegionExists(getFileSystem(), getRootContainer().path)) {
bootstrapMeta(getConfiguration());
}
@@ -225,7 +309,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
@Override
protected void startupCleanup() throws IOException {
- checkTempDir(getTempDir(), getConfiguration(), getFileSystem());
+ checkTempDir(getTempContainer().path, getConfiguration(), getFileSystem());
}
/**
@@ -240,7 +324,7 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
// if not the cleaner will take care of them.
for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
- HFileArchiver.archiveRegion(fs, getRootDir(), tabledir, regiondir);
+ HFileArchiver.archiveRegion(fs, getRootContainer().path, tabledir, regiondir);
}
}
if (!fs.delete(tmpdir, true)) {
@@ -257,27 +341,28 @@ public class LegacyMasterFileSystem extends MasterFileSystem {
// ==========================================================================
// PROTECTED Methods - Path
// ==========================================================================
- protected Path getNamespaceDir(FsContext ctx) {
+ protected Path getNamespaceDir(StorageContext ctx) {
return getBaseDirFromContext(ctx);
}
- protected Path getNamespaceDir(FsContext ctx, String namespace) {
+ protected Path getNamespaceDir(StorageContext ctx, String namespace) {
return LegacyLayout.getNamespaceDir(getBaseDirFromContext(ctx), namespace);
}
- protected Path getTableDir(FsContext ctx, TableName table) {
+ protected Path getTableDir(StorageContext ctx, TableName table) {
return LegacyLayout.getTableDir(getBaseDirFromContext(ctx), table);
}
- protected Path getRegionDir(FsContext ctx, TableName table, HRegionInfo hri) {
+ protected Path getRegionDir(StorageContext ctx, TableName table, HRegionInfo hri) {
return LegacyLayout.getRegionDir(getTableDir(ctx, table), hri);
}
- public Path getTempDir() {
- return tmpDir;
+ @Override
+ public LegacyPathIdentifier getTempContainer() {
+ return new LegacyPathIdentifier(tmpDir);
}
- protected Path getBaseDirFromContext(FsContext ctx) {
+ protected Path getBaseDirFromContext(StorageContext ctx) {
switch (ctx) {
case TEMP: return tmpDataDir;
case DATA: return dataDir;
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index 1546f58..801cebd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.fs.RegionStorage;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
@@ -207,7 +208,7 @@ public class CatalogJanitor extends ScheduledChore {
+ regionB.getRegionNameAsString()
+ " from fs because merged region no longer holds references");
// TODO update HFileArchiver to use RegionStorage
- FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
+ FileSystem fs = this.services.getMasterStorage().getFileSystem();
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
@@ -351,7 +352,7 @@ public class CatalogJanitor extends ScheduledChore {
if (hasNoReferences(a) && hasNoReferences(b)) {
LOG.debug("Deleting region " + parent.getRegionNameAsString() +
" because daughter splits no longer hold references");
- FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
+ FileSystem fs = this.services.getMasterStorage().getFileSystem();
if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
MetaTableAccessor.deleteRegion(this.connection, parent);
@@ -386,8 +387,9 @@ public class CatalogJanitor extends ScheduledChore {
return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
}
- FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
- Path rootdir = this.services.getMasterFileSystem().getRootDir();
+ FileSystem fs = this.services.getMasterStorage().getFileSystem();
+ Path rootdir = ((LegacyPathIdentifier) this.services.getMasterStorage().getRootContainer())
+ .path;
Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());
Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
index 194e023..41c25e9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
@@ -163,7 +163,7 @@ public class ClusterStatusPublisher extends ScheduledChore {
// This could be extended in the future, for example if we want to send stuff like the
// hbase:meta server name.
ClusterStatus cs = new ClusterStatus(VersionInfo.getVersion(),
- master.getMasterFileSystem().getClusterId().toString(),
+ master.getMasterStorage().getClusterId().toString(),
null,
sns,
master.getServerName(),
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c9056e3..9181579 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -89,8 +89,9 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.exceptions.MergeRegionException;
import org.apache.hadoop.hbase.executor.ExecutorType;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
@@ -119,7 +120,6 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler.Procedu
import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure;
import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
-import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.mob.MobConstants;
@@ -142,8 +142,6 @@ import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy;
import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesZKImpl;
import org.apache.hadoop.hbase.replication.master.TableCFsUpdater;
import org.apache.hadoop.hbase.replication.regionserver.Replication;
import org.apache.hadoop.hbase.security.UserProvider;
@@ -164,7 +162,6 @@ import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
import org.apache.hadoop.hbase.zookeeper.MasterMaintenanceModeTracker;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker;
import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
import org.apache.hadoop.hbase.zookeeper.SplitOrMergeTracker;
@@ -275,8 +272,8 @@ public class HMaster extends HRegionServer implements MasterServices {
// Metrics for the HMaster
final MetricsMaster metricsMaster;
- // file system manager for the master FS operations
- private MasterFileSystem fileSystemManager;
+ // storage manager for the master storage operations
+ private MasterStorage storageManager;
private MasterWalManager walManager;
// server manager to deal with region server info
@@ -682,7 +679,7 @@ public class HMaster extends HRegionServer implements MasterServices {
this.masterActiveTime = System.currentTimeMillis();
// TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
- this.fileSystemManager = MasterFileSystem.open(conf, true);
+ this.storageManager = MasterStorage.open(conf, true);
this.walManager = new MasterWalManager(this);
// enable table descriptors cache
@@ -698,7 +695,7 @@ public class HMaster extends HRegionServer implements MasterServices {
// publish cluster ID
status.setStatus("Publishing Cluster ID in ZooKeeper");
- ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
+ ZKClusterId.setClusterId(this.zooKeeper, storageManager.getClusterId());
this.initLatch.countDown();
this.serverManager = createServerManager(this);
@@ -922,8 +919,8 @@ public class HMaster extends HRegionServer implements MasterServices {
}
@Override
- public MasterFileSystem getMasterFileSystem() {
- return this.fileSystemManager;
+ public MasterStorage getMasterStorage() {
+ return this.storageManager;
}
@Override
@@ -976,7 +973,7 @@ public class HMaster extends HRegionServer implements MasterServices {
Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
Map<String, Object> params = new HashMap<String, Object>();
params.put(MASTER, this);
- this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()
+ this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterStorage()
.getFileSystem(), archiveDir, params);
getChoreService().scheduleChore(hfileCleaner);
serviceStarted = true;
@@ -1030,10 +1027,10 @@ public class HMaster extends HRegionServer implements MasterServices {
private void startProcedureExecutor() throws IOException {
final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
- final Path logDir = new Path(fileSystemManager.getRootDir(),
+ final Path logDir = new Path(((LegacyPathIdentifier) storageManager.getRootContainer()).path,
MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR);
- procedureStore = new WALProcedureStore(conf, fileSystemManager.getFileSystem(), logDir,
+ procedureStore = new WALProcedureStore(conf, storageManager.getFileSystem(), logDir,
new MasterProcedureEnv.WALStoreLeaseRecovery(this));
procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore,
@@ -2133,8 +2130,8 @@ public class HMaster extends HRegionServer implements MasterServices {
}});
}
- String clusterId = fileSystemManager != null ?
- fileSystemManager.getClusterId().toString() : null;
+ String clusterId = storageManager != null ?
+ storageManager.getClusterId().toString() : null;
Set<RegionState> regionsInTransition = assignmentManager != null ?
assignmentManager.getRegionStates().getRegionsInTransition() : null;
String[] coprocessors = cpHost != null ? getMasterCoprocessors() : null;
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index 3dfe4a6..2a6704d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
@@ -68,9 +68,9 @@ public interface MasterServices extends Server {
AssignmentManager getAssignmentManager();
/**
- * @return Master's filesystem {@link MasterFileSystem} utility class.
+ * @return Master's filesystem {@link MasterStorage} utility class.
*/
- MasterFileSystem getMasterFileSystem();
+ MasterStorage getMasterStorage();
/**
* @return Master's WALs {@link MasterWalManager} utility class.
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
index 58b646b..b2fb9d9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -79,7 +81,7 @@ public class MasterWalManager {
// The Path to the old logs dir
private final Path oldLogDir;
- private final Path rootDir;
+ private final StorageIdentifier rootContainer;
// create the split log lock
private final Lock splitLogLock = new ReentrantLock();
@@ -90,20 +92,21 @@ public class MasterWalManager {
private volatile boolean fsOk = true;
public MasterWalManager(MasterServices services) throws IOException {
- this(services.getConfiguration(), services.getMasterFileSystem().getFileSystem(),
- services.getMasterFileSystem().getRootDir(), services);
+ this(services.getConfiguration(), services.getMasterStorage().getFileSystem(),
+ services.getMasterStorage().getRootContainer(), services);
}
- public MasterWalManager(Configuration conf, FileSystem fs, Path rootDir, MasterServices services)
- throws IOException {
+ public MasterWalManager(Configuration conf, FileSystem fs, StorageIdentifier rootContainer,
+ MasterServices services) throws IOException {
this.fs = fs;
this.conf = conf;
- this.rootDir = rootDir;
+ this.rootContainer = rootContainer;
this.services = services;
this.splitLogManager = new SplitLogManager(services, conf);
this.distributedLogReplay = this.splitLogManager.isLogReplaying();
- this.oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ this.oldLogDir = new Path(((LegacyPathIdentifier) rootContainer).path, HConstants
+ .HREGION_OLDLOGDIR_NAME);
// Make sure the region servers can archive their old logs
if (!this.fs.exists(oldLogDir)) {
@@ -161,7 +164,8 @@ public class MasterWalManager {
WALSplitter.SPLIT_SKIP_ERRORS_DEFAULT);
Set<ServerName> serverNames = new HashSet<ServerName>();
- Path logsDirPath = new Path(this.rootDir, HConstants.HREGION_LOGDIR_NAME);
+ Path logsDirPath = new Path(((LegacyPathIdentifier) this.rootContainer).path, HConstants
+ .HREGION_LOGDIR_NAME);
do {
if (services.isStopped()) {
@@ -259,7 +263,7 @@ public class MasterWalManager {
}
try {
for (ServerName serverName : serverNames) {
- Path logDir = new Path(this.rootDir,
+ Path logDir = new Path(((LegacyPathIdentifier) this.rootContainer).path,
AbstractFSWALProvider.getWALDirectoryName(serverName.toString()));
Path splitDir = logDir.suffix(AbstractFSWALProvider.SPLITTING_EXT);
// Rename the directory so a rogue RS doesn't create more WALs
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index e875f30..8b70685 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -20,13 +20,10 @@ package org.apache.hadoop.hbase.master;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -52,7 +49,6 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
/**
@@ -679,7 +675,7 @@ public class RegionStates {
// This is a cleanup task. Not critical.
if (MetaTableAccessor.getRegion(server.getConnection(), hri.getEncodedNameAsBytes()) == null) {
regionOffline(hri);
- server.getMasterFileSystem().deleteRegion(hri);
+ server.getMasterStorage().deleteRegion(hri);
}
} catch (IOException e) {
LOG.warn("Got exception while deleting " + hri + " directories from file system.", e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
index f52e321..f3cabbd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.master.procedure;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -41,8 +40,10 @@ import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.fs.FsContext;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.StorageContext;
+import org.apache.hadoop.hbase.fs.MasterStorage;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.MetricsSnapshot;
import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions;
@@ -52,7 +53,6 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CloneSnapshotState;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -308,13 +308,13 @@ public class CloneSnapshotProcedure
throws IOException, InterruptedException {
if (!getTableName().isSystemTable()) {
// Check and update namespace quota
- final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+ final MasterStorage ms = env.getMasterServices().getMasterStorage();
SnapshotManifest manifest = SnapshotManifest.open(
env.getMasterConfiguration(),
- mfs.getFileSystem(),
- SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()),
- snapshot);
+ ms.getFileSystem(),
+ SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, ((LegacyPathIdentifier) ms
+ .getRootContainer()).path), snapshot);
ProcedureSyncWait.getMasterQuotaManager(env)
.checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size());
@@ -358,9 +358,9 @@ public class CloneSnapshotProcedure
final Path tableRootDir, final TableName tableName,
final List<HRegionInfo> newRegions) throws IOException {
- final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
- final FileSystem fs = mfs.getFileSystem();
- final Path rootDir = mfs.getRootDir();
+ final MasterStorage ms = env.getMasterServices().getMasterStorage();
+ final FileSystem fs = ms.getFileSystem();
+ final StorageIdentifier rootContainer = ms.getRootContainer();
final Configuration conf = env.getMasterConfiguration();
final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher();
@@ -368,7 +368,8 @@ public class CloneSnapshotProcedure
try {
// 1. Execute the on-disk Clone
- Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot,
+ ((LegacyPathIdentifier) rootContainer).path);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
conf, fs, manifest, hTableDescriptor, tableRootDir, monitorException, monitorStatus);
@@ -413,14 +414,14 @@ public class CloneSnapshotProcedure
final HTableDescriptor hTableDescriptor,
List<HRegionInfo> newRegions,
final CreateHdfsRegions hdfsRegionHandler) throws IOException {
- final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
- final Path tempdir = mfs.getTempDir();
+ final MasterStorage ms = env.getMasterServices().getMasterStorage();
+ final Path tempdir = ((LegacyPathIdentifier)ms.getTempContainer()).path;
// 1. Create Table Descriptor
// using a copy of descriptor, table will be created enabling first
HTableDescriptor underConstruction = new HTableDescriptor(hTableDescriptor);
final Path tempTableDir = FSUtils.getTableDir(tempdir, hTableDescriptor.getTableName());
- mfs.createTableDescriptor(FsContext.TEMP, underConstruction, false);
+ ms.createTableDescriptor(StorageContext.TEMP, underConstruction, false);
// 2. Create Regions
newRegions = hdfsRegionHandler.createHdfsRegions(
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
index 7562a04..0b4dc5d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.hbase.master.TableNamespaceManager;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CreateNamespaceState;
-import org.apache.hadoop.hbase.util.FSUtils;
/**
* The procedure to create a new namespace.
@@ -202,7 +201,7 @@ public class CreateNamespaceProcedure
protected static void createDirectory(
final MasterProcedureEnv env,
final NamespaceDescriptor nsDescriptor) throws IOException {
- env.getMasterServices().getMasterFileSystem().createNamespace(nsDescriptor);
+ env.getMasterServices().getMasterStorage().createNamespace(nsDescriptor);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
index 196cb16..e44192d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.master.procedure;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
@@ -38,11 +37,11 @@ import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.fs.FsContext;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.StorageContext;
+import org.apache.hadoop.hbase.fs.MasterStorage;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
-import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
@@ -294,14 +293,14 @@ public class CreateTableProcedure
protected static List<HRegionInfo> createFsLayout(final MasterProcedureEnv env,
final HTableDescriptor hTableDescriptor, List<HRegionInfo> newRegions,
final CreateHdfsRegions hdfsRegionHandler) throws IOException {
- final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
- final Path tempdir = mfs.getTempDir();
+ final MasterStorage ms = env.getMasterServices().getMasterStorage();
+ final Path tempdir = ((LegacyPathIdentifier) ms.getTempContainer()).path;
// 1. Create Table Descriptor
// using a copy of descriptor, table will be created enabling first
HTableDescriptor underConstruction = new HTableDescriptor(hTableDescriptor);
final Path tempTableDir = FSUtils.getTableDir(tempdir, hTableDescriptor.getTableName());
- mfs.createTableDescriptor(FsContext.TEMP, underConstruction, false);
+ ms.createTableDescriptor(StorageContext.TEMP, underConstruction, false);
// 2. Create Regions
newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir,
@@ -317,9 +316,10 @@ public class CreateTableProcedure
final MasterProcedureEnv env,
final HTableDescriptor hTableDescriptor,
final Path tempTableDir) throws IOException {
- final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
- final Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), hTableDescriptor.getTableName());
- FileSystem fs = mfs.getFileSystem();
+ final MasterStorage ms = env.getMasterServices().getMasterStorage();
+ final Path tableDir = FSUtils.getTableDir(((LegacyPathIdentifier)ms.getRootContainer()).path,
+ hTableDescriptor.getTableName());
+ FileSystem fs = ms.getFileSystem();
if (!fs.delete(tableDir, true) && fs.exists(tableDir)) {
throw new IOException("Couldn't delete " + tableDir);
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
index c6ca374..2069a6a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
@@ -25,20 +25,14 @@ import java.io.OutputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NamespaceNotFoundException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.constraint.ConstraintException;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
import org.apache.hadoop.hbase.master.TableNamespaceManager;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.DeleteNamespaceState;
-import org.apache.hadoop.hbase.util.FSUtils;
/**
* The procedure to remove a namespace.
@@ -275,7 +269,7 @@ public class DeleteNamespaceProcedure
protected static void deleteDirectory(
final MasterProcedureEnv env,
final String namespaceName) throws IOException {
- env.getMasterServices().getMasterFileSystem().deleteNamespace(namespaceName);
+ env.getMasterServices().getMasterStorage().deleteNamespace(namespaceName);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea247ec/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index 63a4f0f..a22e124 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.master.procedure;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
@@ -43,8 +42,9 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterStorage;
import org.apache.hadoop.hbase.exceptions.HBaseException;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.mob.MobConstants;
@@ -262,11 +262,12 @@ public class DeleteTableProcedure
protected static void deleteFromFs(final MasterProcedureEnv env,
final TableName tableName, final List<HRegionInfo> regions,
final boolean archive) throws IOException {
- final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
- final FileSystem fs = mfs.getFileSystem();
- final Path tempdir = mfs.getTempDir();
+ final MasterStorage ms = env.getMasterServices().getMasterStorage();
+ final FileSystem fs = ms.getFileSystem();
+ final Path tempdir = ((LegacyPathIdentifier)ms.getTempContainer()).path;
- final Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
+ final Path tableDir = FSUtils.getTableDir(((LegacyPathIdentifier)ms.getRootContainer()).path,
+ tableName);
final Path tempTableDir = FSUtils.getTableDir(tempdir, tableName);
if (fs.exists(tableDir)) {
@@ -290,7 +291,8 @@ public class DeleteTableProcedure
if (files != null && files.length > 0) {
for (int i = 0; i < files.length; ++i) {
if (!files[i].isDir()) continue;
- HFileArchiver.archiveRegion(fs, mfs.getRootDir(), tempTableDir, files[i].getPath());
+ HFileArchiver.archiveRegion(fs, ((LegacyPathIdentifier) ms.getRootContainer()).path,
+ tempTableDir, files[i].getPath());
}
}
fs.delete(tempdir, true);
@@ -303,19 +305,20 @@ public class DeleteTableProcedure
if (archive) {
for (HRegionInfo hri : regions) {
LOG.debug("Archiving region " + hri.getRegionNameAsString() + " from FS");
- HFileArchiver.archiveRegion(fs, mfs.getRootDir(),
+ HFileArchiver.archiveRegion(fs, ((LegacyPathIdentifier) ms.getRootContainer()).path,
tempTableDir, HRegion.getRegionDir(tempTableDir, hri.getEncodedName()));
}
LOG.debug("Table '" + tableName + "' archived!");
}
// Archive mob data
- Path mobTableDir = FSUtils.getTableDir(new Path(mfs.getRootDir(), MobConstants.MOB_DIR_NAME),
- tableName);
+ Path mobTableDir = FSUtils.getTableDir(new Path(((LegacyPathIdentifier) ms.getRootContainer())
+ .path, MobConstants.MOB_DIR_NAME), tableName);
Path regionDir =
new Path(mobTableDir, MobUtils.getMobRegionInfo(tableName).getEncodedName());
if (fs.exists(regionDir)) {
- HFileArchiver.archiveRegion(fs, mfs.getRootDir(), mobTableDir, regionDir);
+ HFileArchiver.archiveRegion(fs, ((LegacyPathIdentifier) ms.getRootContainer()).path,
+ mobTableDir, regionDir);
}
// Delete table directory from FS (temp directory)