You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2016/09/22 16:34:38 UTC
[50/50] [abbrv] hbase git commit: HBASE-14439 Move fs stuff out pt1 -
master fs + table desc
HBASE-14439 Move fs stuff out pt1 - master fs + table desc
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6796171f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6796171f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6796171f
Branch: refs/heads/hbase-14439
Commit: 6796171fc43c551ab4aed46b95840cc4c72208a4
Parents: 1384c9a
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Tue Oct 6 16:02:20 2015 -0700
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Sep 22 11:19:52 2016 -0500
----------------------------------------------------------------------
.../apache/hadoop/hbase/HTableDescriptor.java | 63 +++
.../SplitLogManagerCoordination.java | 4 +-
.../org/apache/hadoop/hbase/fs/FsContext.java | 28 +
.../hadoop/hbase/fs/MasterFileSystem.java | 320 ++++++++++++
.../hadoop/hbase/fs/legacy/LegacyLayout.java | 69 +++
.../hbase/fs/legacy/LegacyMasterFileSystem.java | 262 ++++++++++
.../hbase/fs/legacy/LegacyTableDescriptor.java | 399 +++++++++++++++
.../org/apache/hadoop/hbase/master/HMaster.java | 4 +-
.../hadoop/hbase/master/MasterFileSystem.java | 324 ------------
.../hadoop/hbase/master/MasterServices.java | 1 +
.../hadoop/hbase/master/MasterWalManager.java | 2 +-
.../procedure/CloneSnapshotProcedure.java | 6 +-
.../procedure/CreateNamespaceProcedure.java | 5 +-
.../master/procedure/CreateTableProcedure.java | 10 +-
.../procedure/DeleteNamespaceProcedure.java | 20 +-
.../master/procedure/DeleteTableProcedure.java | 2 +-
.../procedure/MasterDDLOperationHelper.java | 2 +-
.../procedure/RestoreSnapshotProcedure.java | 2 +-
.../hbase/master/snapshot/SnapshotManager.java | 2 +-
.../hbase/regionserver/CompactionTool.java | 10 +-
.../hadoop/hbase/snapshot/SnapshotManifest.java | 7 +-
.../hadoop/hbase/util/FSTableDescriptors.java | 510 ++-----------------
.../org/apache/hadoop/hbase/util/FSUtils.java | 33 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 10 +-
.../hadoop/hbase/util/HFileArchiveUtil.java | 4 +-
.../org/apache/hadoop/hbase/util/HMerge.java | 3 +-
.../org/apache/hadoop/hbase/util/Merge.java | 3 +-
.../org/apache/hadoop/hbase/util/MetaUtils.java | 16 +-
.../TestHColumnDescriptorDefaultVersions.java | 5 +-
.../client/TestRestoreSnapshotFromClient.java | 2 +-
.../hbase/client/TestSnapshotFromClient.java | 7 +-
.../hadoop/hbase/master/TestCatalogJanitor.java | 1 +
.../hbase/master/TestMasterFileSystem.java | 1 +
...stTableDescriptorModificationFromClient.java | 7 +-
.../master/snapshot/TestSnapshotManager.java | 2 +-
.../hbase/namespace/TestNamespaceAuditor.java | 18 +-
.../hbase/regionserver/TestDefaultMemStore.java | 2 +-
.../hbase/regionserver/TestStoreFile.java | 3 +-
.../hbase/snapshot/SnapshotTestingUtils.java | 10 +-
.../hbase/util/TestFSTableDescriptors.java | 71 +--
.../hadoop/hbase/util/TestHBaseFsckOneRS.java | 12 +-
41 files changed, 1330 insertions(+), 932 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 9abdf42..867dc0d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -34,6 +34,7 @@ import java.util.regex.Matcher;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
@@ -42,6 +43,7 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema;
+import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
@@ -1567,4 +1569,65 @@ public class HTableDescriptor implements Comparable<HTableDescriptor> {
public void removeConfiguration(final String key) {
configuration.remove(key);
}
+
+ @InterfaceAudience.Private
+ public static HTableDescriptor metaTableDescriptor(final Configuration conf)
+ throws IOException {
+ HTableDescriptor metaDescriptor = new HTableDescriptor(
+ TableName.META_TABLE_NAME,
+ new HColumnDescriptor[] {
+ new HColumnDescriptor(HConstants.CATALOG_FAMILY)
+ .setMaxVersions(conf.getInt(HConstants.HBASE_META_VERSIONS,
+ HConstants.DEFAULT_HBASE_META_VERSIONS))
+ .setInMemory(true)
+ .setBlocksize(conf.getInt(HConstants.HBASE_META_BLOCK_SIZE,
+ HConstants.DEFAULT_HBASE_META_BLOCK_SIZE))
+ .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+ // Disable blooms for meta. Needs work. Seems to mess w/ getClosestOrBefore.
+ .setBloomFilterType(BloomType.NONE)
+ // Enable cache of data blocks in L1 if more than one caching tier deployed:
+ // e.g. if using CombinedBlockCache (BucketCache).
+ .setCacheDataInL1(true),
+ new HColumnDescriptor(HConstants.REPLICATION_BARRIER_FAMILY)
+ .setMaxVersions(conf.getInt(HConstants.HBASE_META_VERSIONS,
+ HConstants.DEFAULT_HBASE_META_VERSIONS))
+ .setInMemory(true)
+ .setBlocksize(conf.getInt(HConstants.HBASE_META_BLOCK_SIZE,
+ HConstants.DEFAULT_HBASE_META_BLOCK_SIZE))
+ .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+ // Disable blooms for meta. Needs work. Seems to mess w/ getClosestOrBefore.
+ .setBloomFilterType(BloomType.NONE)
+ // Enable cache of data blocks in L1 if more than one caching tier deployed:
+ // e.g. if using CombinedBlockCache (BucketCache).
+ .setCacheDataInL1(true),
+ new HColumnDescriptor(HConstants.REPLICATION_POSITION_FAMILY)
+ .setMaxVersions(conf.getInt(HConstants.HBASE_META_VERSIONS,
+ HConstants.DEFAULT_HBASE_META_VERSIONS))
+ .setInMemory(true)
+ .setBlocksize(conf.getInt(HConstants.HBASE_META_BLOCK_SIZE,
+ HConstants.DEFAULT_HBASE_META_BLOCK_SIZE))
+ .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+ // Disable blooms for meta. Needs work. Seems to mess w/ getClosestOrBefore.
+ .setBloomFilterType(BloomType.NONE)
+ // Enable cache of data blocks in L1 if more than one caching tier deployed:
+ // e.g. if using CombinedBlockCache (BucketCache).
+ .setCacheDataInL1(true),
+ new HColumnDescriptor(HConstants.TABLE_FAMILY)
+ // Ten is arbitrary number. Keep versions to help debugging.
+ .setMaxVersions(10)
+ .setInMemory(true)
+ .setBlocksize(8 * 1024)
+ .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+ // Disable blooms for meta. Needs work. Seems to mess w/ getClosestOrBefore.
+ .setBloomFilterType(BloomType.NONE)
+ // Enable cache of data blocks in L1 if more than one caching tier deployed:
+ // e.g. if using CombinedBlockCache (BucketCache).
+ .setCacheDataInL1(true)
+ }) {
+ };
+ metaDescriptor.addCoprocessor(
+ "org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint",
+ null, Coprocessor.PRIORITY_SYSTEM, null);
+ return metaDescriptor;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
index 3b1bbb7..3afce96 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/SplitLogManagerCoordination.java
@@ -42,11 +42,11 @@ import com.google.common.annotations.VisibleForTesting;
* <P>
* Methods required for task life circle: <BR>
* {@link #markRegionsRecovering(ServerName, Set)} mark regions for log replaying. Used by
- * {@link org.apache.hadoop.hbase.master.MasterFileSystem} <BR>
+ * {@link org.apache.hadoop.hbase.fs.MasterFileSystem} <BR>
* {@link #removeRecoveringRegions(Set, Boolean)} make regions cleanup that previous were marked as
* recovering. Called after all tasks processed <BR>
* {@link #removeStaleRecoveringRegions(Set)} remove stale recovering. called by
- * {@link org.apache.hadoop.hbase.master.MasterFileSystem} after Active Master is initialized <BR>
+ * {@link org.apache.hadoop.hbase.fs.MasterFileSystem} after Active Master is initialized <BR>
* {@link #getLastRecoveryTime()} required for garbage collector and should indicate when the last
* recovery has been made<BR>
* {@link #checkTaskStillAvailable(String)} Check that task is still there <BR>
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java
new file mode 100644
index 0000000..ced095d
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/FsContext.java
@@ -0,0 +1,28 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+public enum FsContext {
+ TEMP,
+ DATA,
+ ARCHIVE,
+ SNAPSHOT,
+ SIDELINE,
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
new file mode 100644
index 0000000..2a58b9e
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterFileSystem.java
@@ -0,0 +1,320 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.ClusterId;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.legacy.LegacyMasterFileSystem;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+
+@InterfaceAudience.Private
+public abstract class MasterFileSystem {
+ private static Log LOG = LogFactory.getLog(MasterFileSystem.class);
+
+ // Persisted unique cluster ID
+ private ClusterId clusterId;
+
+ private Configuration conf;
+ private FileSystem fs;
+ private Path rootDir;
+
+ protected MasterFileSystem(Configuration conf, FileSystem fs, Path rootDir) {
+ this.rootDir = rootDir;
+ this.conf = conf;
+ this.fs = fs;
+ }
+
+ public Configuration getConfiguration() { return conf; }
+ public FileSystem getFileSystem() { return fs; }
+ public Path getRootDir() { return rootDir; }
+
+ // ==========================================================================
+ // PUBLIC Interfaces - Visitors
+ // ==========================================================================
+ public interface NamespaceVisitor {
+ void visitNamespace(String namespace) throws IOException;
+ }
+
+ public interface TableVisitor {
+ void visitTable(TableName tableName) throws IOException;
+ }
+
+ public interface RegionVisitor {
+ void visitRegion(HRegionInfo regionInfo) throws IOException;
+ }
+
+ // ==========================================================================
+ // PUBLIC Methods - Namespace related
+ // ==========================================================================
+ public abstract void createNamespace(NamespaceDescriptor nsDescriptor) throws IOException;
+ public abstract void deleteNamespace(String namespaceName) throws IOException;
+ public abstract Collection<String> getNamespaces(FsContext ctx) throws IOException;
+
+ public Collection<String> getNamespaces() throws IOException {
+ return getNamespaces(FsContext.DATA);
+ }
+ // should return or get a NamespaceDescriptor? how is that different from HTD?
+
+ // ==========================================================================
+ // PUBLIC Methods - Table Descriptor related
+ // ==========================================================================
+ public HTableDescriptor getTableDescriptor(TableName tableName)
+ throws IOException {
+ return getTableDescriptor(FsContext.DATA, tableName);
+ }
+
+ public boolean createTableDescriptor(HTableDescriptor tableDesc, boolean force)
+ throws IOException {
+ return createTableDescriptor(FsContext.DATA, tableDesc, force);
+ }
+
+ public void updateTableDescriptor(HTableDescriptor tableDesc) throws IOException {
+ updateTableDescriptor(FsContext.DATA, tableDesc);
+ }
+
+ public abstract HTableDescriptor getTableDescriptor(FsContext ctx, TableName tableName)
+ throws IOException;
+ public abstract boolean createTableDescriptor(FsContext ctx, HTableDescriptor tableDesc,
+ boolean force) throws IOException;
+ public abstract void updateTableDescriptor(FsContext ctx, HTableDescriptor tableDesc)
+ throws IOException;
+
+ // ==========================================================================
+ // PUBLIC Methods - Table related
+ // ==========================================================================
+ public void deleteTable(TableName tableName) throws IOException {
+ deleteTable(FsContext.DATA, tableName);
+ }
+
+ public Collection<TableName> getTables(String namespace) throws IOException {
+ return getTables(FsContext.DATA, namespace);
+ }
+
+ public abstract void deleteTable(FsContext ctx, TableName tableName) throws IOException;
+
+ public abstract Collection<TableName> getTables(FsContext ctx, String namespace)
+ throws IOException;
+
+ public Collection<TableName> getTables() throws IOException {
+ ArrayList<TableName> tables = new ArrayList<TableName>();
+ for (String ns: getNamespaces()) {
+ tables.addAll(getTables(ns));
+ }
+ return tables;
+ }
+
+ // ==========================================================================
+ // PUBLIC Methods - bootstrap
+ // ==========================================================================
+ public abstract Path getTempDir();
+
+ public void logFileSystemState(Log log) throws IOException {
+ FSUtils.logFileSystemState(getFileSystem(), getRootDir(), LOG);
+ }
+
+ /**
+ * @return The unique identifier generated for this cluster
+ */
+ public ClusterId getClusterId() {
+ return clusterId;
+ }
+
+ protected void bootstrap() throws IOException {
+ // check if the root directory exists
+ createInitialLayout(getRootDir(), conf, this.fs);
+
+ // check if temp directory exists and clean it
+ startupCleanup();
+ }
+
+ protected abstract void bootstrapMeta() throws IOException;
+ protected abstract void startupCleanup() throws IOException;
+
+ /**
+ * Create initial layout in filesystem.
+ * <ol>
+ * <li>Check if the meta region exists and is readable, if not create it.
+ * Create hbase.version and the hbase:meta directory if not one.
+ * </li>
+ * <li>Create a log archive directory for RS to put archived logs</li>
+ * </ol>
+ * Idempotent.
+ */
+ private void createInitialLayout(final Path rd, final Configuration c, final FileSystem fs)
+ throws IOException {
+ // If FS is in safe mode wait till out of it.
+ FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
+
+ boolean isSecurityEnabled = "kerberos".equalsIgnoreCase(c.get("hbase.security.authentication"));
+ FsPermission rootDirPerms = new FsPermission(c.get("hbase.rootdir.perms", "700"));
+
+ // Filesystem is good. Go ahead and check for hbase.rootdir.
+ try {
+ if (!fs.exists(rd)) {
+ if (isSecurityEnabled) {
+ fs.mkdirs(rd, rootDirPerms);
+ } else {
+ fs.mkdirs(rd);
+ }
+ // DFS leaves safe mode with 0 DNs when there are 0 blocks.
+ // We used to handle this by checking the current DN count and waiting until
+ // it is nonzero. With security, the check for datanode count doesn't work --
+ // it is a privileged op. So instead we adopt the strategy of the jobtracker
+ // and simply retry file creation during bootstrap indefinitely. As soon as
+ // there is one datanode it will succeed. Permission problems should have
+ // already been caught by mkdirs above.
+ FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+ 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
+ HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
+ } else {
+ if (!fs.isDirectory(rd)) {
+ throw new IllegalArgumentException(rd.toString() + " is not a directory");
+ }
+ if (isSecurityEnabled && !rootDirPerms.equals(fs.getFileStatus(rd).getPermission())) {
+ // check whether the permission match
+ LOG.warn("Found rootdir permissions NOT matching expected \"hbase.rootdir.perms\" for "
+ + "rootdir=" + rd.toString() + " permissions=" + fs.getFileStatus(rd).getPermission()
+ + " and \"hbase.rootdir.perms\" configured as "
+ + c.get("hbase.rootdir.perms", "700") + ". Automatically setting the permissions. You"
+ + " can change the permissions by setting \"hbase.rootdir.perms\" in hbase-site.xml "
+ + "and restarting the master");
+ fs.setPermission(rd, rootDirPerms);
+ }
+ // as above
+ FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
+ 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
+ HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
+ }
+ } catch (DeserializationException de) {
+ LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
+ IOException ioe = new IOException();
+ ioe.initCause(de);
+ throw ioe;
+ } catch (IllegalArgumentException iae) {
+ LOG.fatal("Please fix invalid configuration for "
+ + HConstants.HBASE_DIR + " " + rd.toString(), iae);
+ throw iae;
+ }
+ // Make sure cluster ID exists
+ if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
+ HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
+ FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
+ }
+ clusterId = FSUtils.getClusterId(fs, rd);
+
+ // Make sure the meta region exists!
+ bootstrapMeta();
+ }
+
+ // TODO: Move in HRegionFileSystem
+ public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName, boolean hasMob)
+ throws IOException {
+ // archive family store files
+ Path tableDir = FSUtils.getTableDir(getRootDir(), region.getTable());
+ HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
+
+ // delete the family folder
+ Path familyDir = new Path(tableDir,
+ new Path(region.getEncodedName(), Bytes.toString(familyName)));
+ if (fs.delete(familyDir, true) == false) {
+ if (fs.exists(familyDir)) {
+ throw new IOException("Could not delete family "
+ + Bytes.toString(familyName) + " from FileSystem for region "
+ + region.getRegionNameAsString() + "(" + region.getEncodedName()
+ + ")");
+ }
+ }
+
+ // archive and delete mob files
+ if (hasMob) {
+ Path mobTableDir =
+ FSUtils.getTableDir(new Path(getRootDir(), MobConstants.MOB_DIR_NAME), region.getTable());
+ HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(region.getTable());
+ Path mobFamilyDir =
+ new Path(mobTableDir,
+ new Path(mobRegionInfo.getEncodedName(), Bytes.toString(familyName)));
+ // archive mob family store files
+ MobUtils.archiveMobStoreFiles(conf, fs, mobRegionInfo, mobFamilyDir, familyName);
+
+ if (!fs.delete(mobFamilyDir, true)) {
+ throw new IOException("Could not delete mob store files for family "
+ + Bytes.toString(familyName) + " from FileSystem region "
+ + mobRegionInfo.getRegionNameAsString() + "(" + mobRegionInfo.getEncodedName() + ")");
+ }
+ }
+ }
+
+ // ==========================================================================
+ // PUBLIC
+ // ==========================================================================
+ public static MasterFileSystem open(Configuration conf, boolean bootstrap)
+ throws IOException {
+ return open(conf, FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), bootstrap);
+ }
+
+ public static MasterFileSystem open(Configuration conf, FileSystem fs,
+ Path rootDir, boolean bootstrap) throws IOException {
+ // Cover both bases, the old way of setting default fs and the new.
+ // We're supposed to run on 0.20 and 0.21 anyways.
+ fs = rootDir.getFileSystem(conf);
+ FSUtils.setFsDefault(conf, new Path(fs.getUri()));
+ // make sure the fs has the same conf
+ fs.setConf(conf);
+
+ MasterFileSystem mfs = getInstance(conf, fs, rootDir);
+ if (bootstrap) {
+ mfs.bootstrap();
+ }
+ HFileSystem.addLocationsOrderInterceptor(conf);
+ return mfs;
+ }
+
+ private static MasterFileSystem getInstance(Configuration conf, final FileSystem fs,
+ Path rootDir) throws IOException {
+ String fsType = conf.get("hbase.fs.layout.type", "legacy").toLowerCase();
+ switch (fsType) {
+ case "legacy":
+ return new LegacyMasterFileSystem(conf, fs, rootDir);
+ default:
+ throw new IOException("Invalid filesystem type " + fsType);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
new file mode 100644
index 0000000..3af40fa
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java
@@ -0,0 +1,69 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs.legacy;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+
+public final class LegacyLayout {
+ /** Name of the region info file that resides just under the region directory. */
+ public final static String REGION_INFO_FILE = ".regioninfo";
+
+ private LegacyLayout() {}
+
+ public static Path getDataDir(final Path rootDir) {
+ return new Path(rootDir, HConstants.BASE_NAMESPACE_DIR);
+ }
+
+ public static Path getSidelineDir(Path rootDir) {
+ return new Path(rootDir, HConstants.HBCK_SIDELINEDIR_NAME);
+ }
+
+ public static Path getSnapshotDir(Path rootDir) {
+ return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
+ }
+
+ public static Path getSnapshotDir(Path baseSnapshotDir, String snapshotName) {
+ return new Path(baseSnapshotDir, snapshotName);
+ }
+
+ public static Path getArchiveDir(Path rootDir) {
+ return new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+ }
+
+ public static Path getTempDir(Path rootDir) {
+ return new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY);
+ }
+
+ public static Path getNamespaceDir(Path baseDataDir, String namespace) {
+ return new Path(baseDataDir, namespace);
+ }
+
+ public static Path getTableDir(Path baseDataDir, TableName table) {
+ Path nsDir = getNamespaceDir(baseDataDir, table.getNamespaceAsString());
+ return new Path(nsDir, table.getQualifierAsString());
+ }
+
+ public static Path getRegionDir(Path baseDataDir, TableName table, HRegionInfo hri) {
+ return new Path(getTableDir(baseDataDir, table), hri.getEncodedName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
new file mode 100644
index 0000000..f330b6e
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterFileSystem.java
@@ -0,0 +1,262 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs.legacy;
+
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.FsContext;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.MetaUtils;
+
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+
+@InterfaceAudience.Private
+public class LegacyMasterFileSystem extends MasterFileSystem {
+ private static final Log LOG = LogFactory.getLog(LegacyMasterFileSystem.class);
+
+ private final Path sidelineDir;
+ private final Path snapshotDir;
+ private final Path archiveDataDir;
+ private final Path archiveDir;
+ private final Path tmpDataDir;
+ private final Path dataDir;
+ private final Path tmpDir;
+
+ public LegacyMasterFileSystem(Configuration conf, FileSystem fs, Path rootDir) {
+ super(conf, fs, rootDir);
+
+ // base directories
+ this.sidelineDir = LegacyLayout.getSidelineDir(rootDir);
+ this.snapshotDir = LegacyLayout.getSnapshotDir(rootDir);
+ this.archiveDir = LegacyLayout.getArchiveDir(rootDir);
+ this.archiveDataDir = LegacyLayout.getDataDir(this.archiveDir);
+ this.dataDir = LegacyLayout.getDataDir(rootDir);
+ this.tmpDir = LegacyLayout.getTempDir(rootDir);
+ this.tmpDataDir = LegacyLayout.getDataDir(this.tmpDir);
+ }
+
+ // ==========================================================================
+ // PUBLIC Methods - Namespace related
+ // ==========================================================================
+ public void createNamespace(NamespaceDescriptor nsDescriptor) throws IOException {
+ getFileSystem().mkdirs(getNamespaceDir(FsContext.DATA, nsDescriptor.getName()));
+ }
+
+ public void deleteNamespace(String namespaceName) throws IOException {
+ FileSystem fs = getFileSystem();
+ Path nsDir = getNamespaceDir(FsContext.DATA, namespaceName);
+
+ try {
+ for (FileStatus status : fs.listStatus(nsDir)) {
+ if (!HConstants.HBASE_NON_TABLE_DIRS.contains(status.getPath().getName())) {
+ throw new IOException("Namespace directory contains table dir: " + status.getPath());
+ }
+ }
+ if (!fs.delete(nsDir, true)) {
+ throw new IOException("Failed to remove namespace: " + namespaceName);
+ }
+ } catch (FileNotFoundException e) {
+ // File already deleted, continue
+ LOG.debug("deleteDirectory throws exception: " + e);
+ }
+ }
+
+ public Collection<String> getNamespaces(FsContext ctx) throws IOException {
+ FileStatus[] stats = FSUtils.listStatus(getFileSystem(), getNamespaceDir(ctx));
+ if (stats == null) return Collections.emptyList();
+
+ ArrayList<String> namespaces = new ArrayList<String>(stats.length);
+ for (int i = 0; i < stats.length; ++i) {
+ namespaces.add(stats[i].getPath().getName());
+ }
+ return namespaces;
+ }
+
+ // should return or get a NamespaceDescriptor? how is that different from HTD?
+
+ // ==========================================================================
+ // PUBLIC Methods - Table Descriptor related
+ // ==========================================================================s
+ @Override
+ public boolean createTableDescriptor(FsContext ctx, HTableDescriptor tableDesc, boolean force)
+ throws IOException {
+ return LegacyTableDescriptor.createTableDescriptor(getFileSystem(),
+ getTableDir(ctx, tableDesc.getTableName()), tableDesc, force);
+ }
+
+ @Override
+ public void updateTableDescriptor(FsContext ctx, HTableDescriptor tableDesc) throws IOException {
+ LegacyTableDescriptor.updateTableDescriptor(getFileSystem(),
+ getTableDir(ctx, tableDesc.getTableName()), tableDesc);
+ }
+
+ @Override
+ public HTableDescriptor getTableDescriptor(FsContext ctx, TableName tableName)
+ throws IOException {
+ return LegacyTableDescriptor.getTableDescriptorFromFs(
+ getFileSystem(), getTableDir(ctx, tableName));
+ }
+
+ // ==========================================================================
+ // PUBLIC Methods - Table related
+ // ==========================================================================
+ @Override
+ public void deleteTable(FsContext ctx, TableName tableName) throws IOException {
+ Path tableDir = getTableDir(ctx, tableName);
+ if (!FSUtils.deleteDirectory(getFileSystem(), tableDir)) {
+ throw new IOException("Failed delete of " + tableName);
+ }
+ }
+
+ @Override
+ public Collection<TableName> getTables(FsContext ctx, String namespace)
+ throws IOException {
+ FileStatus[] stats = FSUtils.listStatus(getFileSystem(),
+ getNamespaceDir(ctx, namespace), new FSUtils.UserTableDirFilter(getFileSystem()));
+ if (stats == null) return Collections.emptyList();
+
+ ArrayList<TableName> tables = new ArrayList<TableName>(stats.length);
+ for (int i = 0; i < stats.length; ++i) {
+ tables.add(TableName.valueOf(namespace, stats[i].getPath().getName()));
+ }
+ return tables;
+ }
+
+ // ==========================================================================
+ // PROTECTED Methods - Bootstrap
+ // ==========================================================================
+ @Override
+ protected void bootstrapMeta() throws IOException {
+ if (!FSUtils.metaRegionExists(getFileSystem(), getRootDir())) {
+ bootstrapMeta(getRootDir(), getConfiguration());
+ }
+
+ // Create tableinfo-s for hbase:meta if not already there.
+ // assume, created table descriptor is for enabling table
+ // meta table is a system table, so descriptors are predefined,
+ // we should get them from registry.
+ createTableDescriptor(HTableDescriptor.metaTableDescriptor(getConfiguration()), false);
+ }
+
+ private static void bootstrapMeta(final Path rd, final Configuration c) throws IOException {
+ LOG.info("BOOTSTRAP: creating hbase:meta region");
+ try {
+ // Bootstrapping, make sure blockcache is off. Else, one will be
+ // created here in bootstrap and it'll need to be cleaned up. Better to
+ // not make it in first place. Turn off block caching for bootstrap.
+ // Enable after.
+ HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
+ HTableDescriptor metaDescriptor = HTableDescriptor.metaTableDescriptor(c);
+ MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, false);
+ HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor, null);
+ MetaUtils.setInfoFamilyCachingForMeta(metaDescriptor, true);
+ meta.close();
+ } catch (IOException e) {
+ e = e instanceof RemoteException ?
+ ((RemoteException)e).unwrapRemoteException() : e;
+ LOG.error("bootstrap", e);
+ throw e;
+ }
+ }
+
+ @Override
+ protected void startupCleanup() throws IOException {
+ checkTempDir(getTempDir(), getConfiguration(), getFileSystem());
+ }
+
+ /**
+ * Make sure the hbase temp directory exists and is empty.
+ * NOTE that this method is only executed once just after the master becomes the active one.
+ */
+ private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
+ throws IOException {
+ // If the temp directory exists, clear the content (left over, from the previous run)
+ if (fs.exists(tmpdir)) {
+ // Archive table in temp, maybe left over from failed deletion,
+ // if not the cleaner will take care of them.
+ for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
+ for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
+ HFileArchiver.archiveRegion(fs, getRootDir(), tabledir, regiondir);
+ }
+ }
+ if (!fs.delete(tmpdir, true)) {
+ throw new IOException("Unable to clean the temp directory: " + tmpdir);
+ }
+ }
+
+ // Create the temp directory
+ if (!fs.mkdirs(tmpdir)) {
+ throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
+ }
+ }
+
+ // ==========================================================================
+ // PROTECTED Methods - Path
+ // ==========================================================================
+ protected Path getNamespaceDir(FsContext ctx) {
+ return getBaseDirFromContext(ctx);
+ }
+
+ protected Path getNamespaceDir(FsContext ctx, String namespace) {
+ return LegacyLayout.getNamespaceDir(getBaseDirFromContext(ctx), namespace);
+ }
+
+ protected Path getTableDir(FsContext ctx, TableName table) {
+ return LegacyLayout.getTableDir(getBaseDirFromContext(ctx), table);
+ }
+
+ protected Path getRegionDir(FsContext ctx, TableName table, HRegionInfo hri) {
+ return LegacyLayout.getRegionDir(getBaseDirFromContext(ctx), table, hri);
+ }
+
+ public Path getTempDir() {
+ return tmpDir;
+ }
+
+ protected Path getBaseDirFromContext(FsContext ctx) {
+ switch (ctx) {
+ case TEMP: return tmpDataDir;
+ case DATA: return dataDir;
+ case ARCHIVE: return archiveDataDir;
+ case SNAPSHOT: return snapshotDir;
+ case SIDELINE: return sidelineDir;
+ default: throw new RuntimeException("Invalid context: " + ctx);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyTableDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyTableDescriptor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyTableDescriptor.java
new file mode 100644
index 0000000..e4d5bc1
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyTableDescriptor.java
@@ -0,0 +1,399 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.fs.legacy;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableDescriptors;
+import org.apache.hadoop.hbase.TableInfoMissingException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.util.*;
+
+/**
+ * Implementation of {@link TableDescriptors} that reads descriptors from the
+ * passed filesystem. It expects descriptors to be in a file in the
+ * {@link #TABLEINFO_DIR} subdir of the table's directory in FS. Can be read-only
+ * -- i.e. does not modify the filesystem or can be read and write.
+ *
+ * <p>Also has utility for keeping up the table descriptors tableinfo file.
+ * The table schema file is kept in the {@link #TABLEINFO_DIR} subdir
+ * of the table directory in the filesystem.
+ * It has a {@link #TABLEINFO_FILE_PREFIX} and then a suffix that is the
+ * edit sequenceid: e.g. <code>.tableinfo.0000000003</code>. This sequenceid
+ * is always increasing. It starts at zero. The table schema file with the
+ * highest sequenceid has the most recent schema edit. Usually there is one file
+ * only, the most recent but there may be short periods where there are more
+ * than one file. Old files are eventually cleaned. Presumption is that there
+ * will not be lots of concurrent clients making table schema edits. If so,
+ * the below needs a bit of a reworking and perhaps some supporting api in hdfs.
+ */
+@InterfaceAudience.Private
+public final class LegacyTableDescriptor {
+ private static final Log LOG = LogFactory.getLog(LegacyTableDescriptor.class);
+
+ /** The file name prefix used to store HTD in HDFS */
+ public static final String TABLEINFO_FILE_PREFIX = ".tableinfo";
+ public static final String TABLEINFO_DIR = ".tabledesc";
+ public static final String TMP_DIR = ".tmp";
+
+ private LegacyTableDescriptor() {}
+
+ /**
+ * Find the most current table info file for the table located in the given table directory.
+ *
+ * Looks within the {@link #TABLEINFO_DIR} subdirectory of the given directory for any table info
+ * files and takes the 'current' one - meaning the one with the highest sequence number if present
+ * or no sequence number at all if none exist (for backward compatibility from before there
+ * were sequence numbers).
+ *
+ * @return The file status of the current table info file or null if it does not exist
+ * @throws IOException
+ */
+ public static FileStatus getTableInfoPath(FileSystem fs, Path tableDir) throws IOException {
+ return getTableInfoPath(fs, tableDir, false);
+ }
+
+ /**
+ * Find the most current table info file for the table in the given table directory.
+ *
+ * Looks within the {@link #TABLEINFO_DIR} subdirectory of the given directory for any table info
+ * files and takes the 'current' one - meaning the one with the highest sequence number if
+ * present or no sequence number at all if none exist (for backward compatibility from before
+ * there were sequence numbers).
+ * If there are multiple table info files found and removeOldFiles is true it also deletes the
+ * older files.
+ *
+ * @return The file status of the current table info file or null if none exist
+ * @throws IOException
+ */
+ private static FileStatus getTableInfoPath(FileSystem fs, Path tableDir, boolean removeOldFiles)
+ throws IOException {
+ Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
+ return getCurrentTableInfoStatus(fs, tableInfoDir, removeOldFiles);
+ }
+
+ /**
+ * Find the most current table info file in the given directory
+ *
+ * Looks within the given directory for any table info files
+ * and takes the 'current' one - meaning the one with the highest sequence number if present
+ * or no sequence number at all if none exist (for backward compatibility from before there
+ * were sequence numbers).
+ * If there are multiple possible files found
+ * and the we're not in read only mode it also deletes the older files.
+ *
+ * @return The file status of the current table info file or null if it does not exist
+ * @throws IOException
+ */
+ // only visible for FSTableDescriptorMigrationToSubdir, can be removed with that
+ public static FileStatus getCurrentTableInfoStatus(FileSystem fs, Path dir, boolean removeOldFiles)
+ throws IOException {
+ FileStatus [] status = FSUtils.listStatus(fs, dir, TABLEINFO_PATHFILTER);
+ if (status == null || status.length < 1) return null;
+ FileStatus mostCurrent = null;
+ for (FileStatus file : status) {
+ if (mostCurrent == null || TABLEINFO_FILESTATUS_COMPARATOR.compare(file, mostCurrent) < 0) {
+ mostCurrent = file;
+ }
+ }
+ if (removeOldFiles && status.length > 1) {
+ // Clean away old versions
+ for (FileStatus file : status) {
+ Path path = file.getPath();
+ if (file != mostCurrent) {
+ if (!fs.delete(path, false)) {
+ LOG.warn("Failed cleanup of " + path);
+ } else {
+ LOG.debug("Cleaned up old tableinfo file " + path);
+ }
+ }
+ }
+ }
+ return mostCurrent;
+ }
+
+ /**
+ * Update table descriptor on the file system
+ * @throws IOException Thrown if failed update.
+ * @throws NotImplementedException if in read only mode
+ */
+ public static void updateTableDescriptor(FileSystem fs, Path tableDir, HTableDescriptor td)
+ throws IOException {
+ TableName tableName = td.getTableName();
+ Path p = writeTableDescriptor(fs, td, tableDir, getTableInfoPath(fs, tableDir));
+ if (p == null) throw new IOException("Failed update");
+ LOG.info("Updated tableinfo=" + p);
+ }
+
+ /**
+ * Compare {@link FileStatus} instances by {@link Path#getName()}. Returns in
+ * reverse order.
+ */
+ @VisibleForTesting
+ public static final Comparator<FileStatus> TABLEINFO_FILESTATUS_COMPARATOR =
+ new Comparator<FileStatus>() {
+ @Override
+ public int compare(FileStatus left, FileStatus right) {
+ return right.compareTo(left);
+ }};
+
+ private static final PathFilter TABLEINFO_PATHFILTER = new PathFilter() {
+ @Override
+ public boolean accept(Path p) {
+ // Accept any file that starts with TABLEINFO_NAME
+ return p.getName().startsWith(TABLEINFO_FILE_PREFIX);
+ }};
+
+ /**
+ * Width of the sequenceid that is a suffix on a tableinfo file.
+ */
+ @VisibleForTesting public static final int WIDTH_OF_SEQUENCE_ID = 10;
+
+ /*
+ * @param number Number to use as suffix.
+ * @return Returns zero-prefixed decimal version of passed
+ * number (Does absolute in case number is negative).
+ */
+ private static String formatTableInfoSequenceId(final int number) {
+ byte [] b = new byte[WIDTH_OF_SEQUENCE_ID];
+ int d = Math.abs(number);
+ for (int i = b.length - 1; i >= 0; i--) {
+ b[i] = (byte)((d % 10) + '0');
+ d /= 10;
+ }
+ return Bytes.toString(b);
+ }
+
+ /**
+ * Regex to eat up sequenceid suffix on a .tableinfo file.
+ * Use regex because may encounter oldstyle .tableinfos where there is no
+ * sequenceid on the end.
+ */
+ private static final Pattern TABLEINFO_FILE_REGEX =
+ Pattern.compile(TABLEINFO_FILE_PREFIX + "(\\.([0-9]{" + WIDTH_OF_SEQUENCE_ID + "}))?$");
+
+ /**
+ * @param p Path to a <code>.tableinfo</code> file.
+ * @return The current editid or 0 if none found.
+ */
+ @VisibleForTesting
+ public static int getTableInfoSequenceId(final Path p) {
+ if (p == null) return 0;
+ Matcher m = TABLEINFO_FILE_REGEX.matcher(p.getName());
+ if (!m.matches()) throw new IllegalArgumentException(p.toString());
+ String suffix = m.group(2);
+ if (suffix == null || suffix.length() <= 0) return 0;
+ return Integer.parseInt(m.group(2));
+ }
+
+ /**
+ * @param sequenceid
+ * @return Name of tableinfo file.
+ */
+ @VisibleForTesting
+ public static String getTableInfoFileName(final int sequenceid) {
+ return TABLEINFO_FILE_PREFIX + "." + formatTableInfoSequenceId(sequenceid);
+ }
+
+ /**
+ * Returns the latest table descriptor for the given table directly from the file system
+ * if it exists, bypassing the local cache.
+ * Returns null if it's not found.
+ */
+ public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs,
+ Path hbaseRootDir, TableName tableName) throws IOException {
+ Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName);
+ return readTableDescriptor(fs, tableDir);
+ }
+
+ /**
+ * Returns the latest table descriptor for the table located at the given directory
+ * directly from the file system if it exists.
+ * @throws TableInfoMissingException if there is no descriptor
+ */
+ public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs, Path tableDir)
+ throws IOException {
+ final HTableDescriptor htd = readTableDescriptor(fs, tableDir);
+ if (htd == null) {
+ throw new TableInfoMissingException("No table descriptor file under " + tableDir);
+ }
+ return htd;
+ }
+
+ /**
+ * @return null if not found
+ */
+ private static HTableDescriptor readTableDescriptor(FileSystem fs, Path tableDir)
+ throws IOException {
+ FileStatus status = getTableInfoPath(fs, tableDir, false);
+ if (status == null) {
+ return null;
+ }
+ return readTableDescriptor(fs, status);
+ }
+
+ private static HTableDescriptor readTableDescriptor(FileSystem fs, FileStatus status)
+ throws IOException {
+ int len = Ints.checkedCast(status.getLen());
+ byte[] content = new byte[len];
+ FSUtils.readFully(fs, status.getPath(), content);
+ try {
+ HTableDescriptor td = HTableDescriptor.parseFrom(content);
+ return td;
+ } catch (DeserializationException exception) {
+ throw new IOException("failed to deserialize descriptor.", exception);
+ }
+ }
+
+ /**
+ * Deletes files matching the table info file pattern within the given directory
+ * whose sequenceId is at most the given max sequenceId.
+ */
+ private static void deleteTableDescriptorFiles(FileSystem fs, Path dir, int maxSequenceId)
+ throws IOException {
+ FileStatus [] status = FSUtils.listStatus(fs, dir, TABLEINFO_PATHFILTER);
+ for (FileStatus file : status) {
+ Path path = file.getPath();
+ int sequenceId = getTableInfoSequenceId(path);
+ if (sequenceId <= maxSequenceId) {
+ boolean success = FSUtils.delete(fs, path, false);
+ if (success) {
+ LOG.debug("Deleted table descriptor at " + path);
+ } else {
+ LOG.error("Failed to delete descriptor at " + path);
+ }
+ }
+ }
+ }
+
+ /**
+ * Attempts to write a new table descriptor to the given table's directory.
+ * It first writes it to the .tmp dir then uses an atomic rename to move it into place.
+ * It begins at the currentSequenceId + 1 and tries 10 times to find a new sequence number
+ * not already in use.
+ * Removes the current descriptor file if passed in.
+ *
+ * @return Descriptor file or null if we failed write.
+ */
+ private static Path writeTableDescriptor(final FileSystem fs,
+ final HTableDescriptor htd, final Path tableDir,
+ final FileStatus currentDescriptorFile) throws IOException {
+ // Get temporary dir into which we'll first write a file to avoid half-written file phenomenon.
+ // This directory is never removed to avoid removing it out from under a concurrent writer.
+ Path tmpTableDir = new Path(tableDir, TMP_DIR);
+ Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
+
+ // What is current sequenceid? We read the current sequenceid from
+ // the current file. After we read it, another thread could come in and
+ // compete with us writing out next version of file. The below retries
+ // should help in this case some but its hard to do guarantees in face of
+ // concurrent schema edits.
+ int currentSequenceId = currentDescriptorFile == null ? 0 :
+ getTableInfoSequenceId(currentDescriptorFile.getPath());
+ int newSequenceId = currentSequenceId;
+
+ // Put arbitrary upperbound on how often we retry
+ int retries = 10;
+ int retrymax = currentSequenceId + retries;
+ Path tableInfoDirPath = null;
+ do {
+ newSequenceId += 1;
+ String filename = getTableInfoFileName(newSequenceId);
+ Path tempPath = new Path(tmpTableDir, filename);
+ if (fs.exists(tempPath)) {
+ LOG.debug(tempPath + " exists; retrying up to " + retries + " times");
+ continue;
+ }
+ tableInfoDirPath = new Path(tableInfoDir, filename);
+ try {
+ writeTD(fs, tempPath, htd);
+ fs.mkdirs(tableInfoDirPath.getParent());
+ if (!fs.rename(tempPath, tableInfoDirPath)) {
+ throw new IOException("Failed rename of " + tempPath + " to " + tableInfoDirPath);
+ }
+ LOG.debug("Wrote descriptor into: " + tableInfoDirPath);
+ } catch (IOException ioe) {
+ // Presume clash of names or something; go around again.
+ LOG.debug("Failed write and/or rename; retrying", ioe);
+ if (!FSUtils.deleteDirectory(fs, tempPath)) {
+ LOG.warn("Failed cleanup of " + tempPath);
+ }
+ tableInfoDirPath = null;
+ continue;
+ }
+ break;
+ } while (newSequenceId < retrymax);
+ if (tableInfoDirPath != null) {
+ // if we succeeded, remove old table info files.
+ deleteTableDescriptorFiles(fs, tableInfoDir, newSequenceId - 1);
+ }
+ return tableInfoDirPath;
+ }
+
+ private static void writeTD(final FileSystem fs, final Path p, final HTableDescriptor htd)
+ throws IOException {
+ // We used to write this file out as a serialized HTD Writable followed by two '\n's and then
+ // the toString version of HTD. Now we just write out the pb serialization.
+ FSUtils.writeFully(fs, p, htd.toByteArray(), false);
+ }
+
+ /**
+ * Create a new HTableDescriptor in HDFS in the specified table directory. Happens when we create
+ * a new table or snapshot a table.
+ * @param tableDir table directory under which we should write the file
+ * @param htd description of the table to write
+ * @param forceCreation if <tt>true</tt>,then even if previous table descriptor is present it will
+ * be overwritten
+ * @return <tt>true</tt> if the we successfully created the file, <tt>false</tt> if the file
+ * already exists and we weren't forcing the descriptor creation.
+ * @throws IOException if a filesystem error occurs
+ */
+ public static boolean createTableDescriptor(FileSystem fs, Path tableDir,
+ HTableDescriptor htd, boolean forceCreation) throws IOException {
+ FileStatus status = getTableInfoPath(fs, tableDir);
+ if (status != null) {
+ LOG.debug("Current tableInfoPath = " + status.getPath());
+ if (!forceCreation) {
+ if (fs.exists(status.getPath()) && status.getLen() > 0) {
+ if (readTableDescriptor(fs, status).equals(htd)) {
+ LOG.debug("TableInfo already exists.. Skipping creation");
+ return false;
+ }
+ }
+ }
+ }
+ Path p = writeTableDescriptor(fs, htd, tableDir, status);
+ return p != null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 810f95b..c9056e3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.exceptions.MergeRegionException;
import org.apache.hadoop.hbase.executor.ExecutorType;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
@@ -681,7 +682,7 @@ public class HMaster extends HRegionServer implements MasterServices {
this.masterActiveTime = System.currentTimeMillis();
// TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
- this.fileSystemManager = new MasterFileSystem(this);
+ this.fileSystemManager = MasterFileSystem.open(conf, true);
this.walManager = new MasterWalManager(this);
// enable table descriptors cache
@@ -1024,7 +1025,6 @@ public class HMaster extends HRegionServer implements MasterServices {
if (this.serverManager != null) this.serverManager.stop();
if (this.assignmentManager != null) this.assignmentManager.stop();
if (this.walManager != null) this.walManager.stop();
- if (this.fileSystemManager != null) this.fileSystemManager.stop();
if (this.mpmHost != null) this.mpmHost.stop("server shutting down.");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
deleted file mode 100644
index 0ce7411..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hbase.ClusterId;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.ipc.RemoteException;
-
-/**
- * This class abstracts a bunch of operations the HMaster needs to interact with
- * the underlying file system like creating the initial layout, checking file
- * system status, etc.
- */
-@InterfaceAudience.Private
-public class MasterFileSystem {
- private static final Log LOG = LogFactory.getLog(MasterFileSystem.class);
-
- // HBase configuration
- private final Configuration conf;
- // Persisted unique cluster ID
- private ClusterId clusterId;
- // Keep around for convenience.
- private final FileSystem fs;
- // root hbase directory on the FS
- private final Path rootdir;
- // hbase temp directory used for table construction and deletion
- private final Path tempdir;
-
- private final MasterServices services;
-
- public MasterFileSystem(MasterServices services) throws IOException {
- this.conf = services.getConfiguration();
- this.services = services;
- // Set filesystem to be that of this.rootdir else we get complaints about
- // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
- // default localfs. Presumption is that rootdir is fully-qualified before
- // we get to here with appropriate fs scheme.
- this.rootdir = FSUtils.getRootDir(conf);
- this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
- // Cover both bases, the old way of setting default fs and the new.
- // We're supposed to run on 0.20 and 0.21 anyways.
- this.fs = this.rootdir.getFileSystem(conf);
- FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
- // make sure the fs has the same conf
- fs.setConf(conf);
- // setup the filesystem variable
- createInitialFileSystemLayout();
- HFileSystem.addLocationsOrderInterceptor(conf);
- }
-
- /**
- * Create initial layout in filesystem.
- * <ol>
- * <li>Check if the meta region exists and is readable, if not create it.
- * Create hbase.version and the hbase:meta directory if not one.
- * </li>
- * </ol>
- * Idempotent.
- */
- private void createInitialFileSystemLayout() throws IOException {
- // check if the root directory exists
- checkRootDir(this.rootdir, conf, this.fs);
-
- // check if temp directory exists and clean it
- checkTempDir(this.tempdir, conf, this.fs);
- }
-
- public FileSystem getFileSystem() {
- return this.fs;
- }
-
- public Configuration getConfiguration() {
- return this.conf;
- }
-
- /**
- * @return HBase root dir.
- */
- public Path getRootDir() {
- return this.rootdir;
- }
-
- /**
- * @return HBase temp dir.
- */
- public Path getTempDir() {
- return this.tempdir;
- }
-
- /**
- * @return The unique identifier generated for this cluster
- */
- public ClusterId getClusterId() {
- return clusterId;
- }
-
- /**
- * Get the rootdir. Make sure its wholesome and exists before returning.
- * @param rd
- * @param c
- * @param fs
- * @return hbase.rootdir (after checks for existence and bootstrapping if
- * needed populating the directory with necessary bootup files).
- * @throws IOException
- */
- private Path checkRootDir(final Path rd, final Configuration c, final FileSystem fs)
- throws IOException {
- // If FS is in safe mode wait till out of it.
- FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
-
- boolean isSecurityEnabled = "kerberos".equalsIgnoreCase(c.get("hbase.security.authentication"));
- FsPermission rootDirPerms = new FsPermission(c.get("hbase.rootdir.perms", "700"));
-
- // Filesystem is good. Go ahead and check for hbase.rootdir.
- try {
- if (!fs.exists(rd)) {
- if (isSecurityEnabled) {
- fs.mkdirs(rd, rootDirPerms);
- } else {
- fs.mkdirs(rd);
- }
- // DFS leaves safe mode with 0 DNs when there are 0 blocks.
- // We used to handle this by checking the current DN count and waiting until
- // it is nonzero. With security, the check for datanode count doesn't work --
- // it is a privileged op. So instead we adopt the strategy of the jobtracker
- // and simply retry file creation during bootstrap indefinitely. As soon as
- // there is one datanode it will succeed. Permission problems should have
- // already been caught by mkdirs above.
- FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
- 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
- HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
- } else {
- if (!fs.isDirectory(rd)) {
- throw new IllegalArgumentException(rd.toString() + " is not a directory");
- }
- if (isSecurityEnabled && !rootDirPerms.equals(fs.getFileStatus(rd).getPermission())) {
- // check whether the permission match
- LOG.warn("Found rootdir permissions NOT matching expected \"hbase.rootdir.perms\" for "
- + "rootdir=" + rd.toString() + " permissions=" + fs.getFileStatus(rd).getPermission()
- + " and \"hbase.rootdir.perms\" configured as "
- + c.get("hbase.rootdir.perms", "700") + ". Automatically setting the permissions. You"
- + " can change the permissions by setting \"hbase.rootdir.perms\" in hbase-site.xml "
- + "and restarting the master");
- fs.setPermission(rd, rootDirPerms);
- }
- // as above
- FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
- 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
- HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
- }
- } catch (DeserializationException de) {
- LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
- IOException ioe = new IOException();
- ioe.initCause(de);
- throw ioe;
- } catch (IllegalArgumentException iae) {
- LOG.fatal("Please fix invalid configuration for "
- + HConstants.HBASE_DIR + " " + rd.toString(), iae);
- throw iae;
- }
- // Make sure cluster ID exists
- if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
- HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
- FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
- }
- clusterId = FSUtils.getClusterId(fs, rd);
-
- // Make sure the meta region directory exists!
- if (!FSUtils.metaRegionExists(fs, rd)) {
- bootstrap(rd, c);
- }
-
- // Create tableinfo-s for hbase:meta if not already there.
- // assume, created table descriptor is for enabling table
- // meta table is a system table, so descriptors are predefined,
- // we should get them from registry.
- FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd);
- fsd.createTableDescriptor(fsd.get(TableName.META_TABLE_NAME));
-
- return rd;
- }
-
- /**
- * Make sure the hbase temp directory exists and is empty.
- * NOTE that this method is only executed once just after the master becomes the active one.
- */
- private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
- throws IOException {
- // If the temp directory exists, clear the content (left over, from the previous run)
- if (fs.exists(tmpdir)) {
- // Archive table in temp, maybe left over from failed deletion,
- // if not the cleaner will take care of them.
- for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
- for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
- HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
- }
- }
- if (!fs.delete(tmpdir, true)) {
- throw new IOException("Unable to clean the temp directory: " + tmpdir);
- }
- }
-
- // Create the temp directory
- if (!fs.mkdirs(tmpdir)) {
- throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
- }
- }
-
- private static void bootstrap(final Path rd, final Configuration c)
- throws IOException {
- LOG.info("BOOTSTRAP: creating hbase:meta region");
- try {
- // Bootstrapping, make sure blockcache is off. Else, one will be
- // created here in bootstrap and it'll need to be cleaned up. Better to
- // not make it in first place. Turn off block caching for bootstrap.
- // Enable after.
- HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
- HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
- setInfoFamilyCachingForMeta(metaDescriptor, false);
- HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor, null);
- setInfoFamilyCachingForMeta(metaDescriptor, true);
- meta.close();
- } catch (IOException e) {
- e = e instanceof RemoteException ?
- ((RemoteException)e).unwrapRemoteException() : e;
- LOG.error("bootstrap", e);
- throw e;
- }
- }
-
- /**
- * Enable in memory caching for hbase:meta
- */
- public static void setInfoFamilyCachingForMeta(HTableDescriptor metaDescriptor, final boolean b) {
- for (HColumnDescriptor hcd: metaDescriptor.getColumnFamilies()) {
- if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
- hcd.setBlockCacheEnabled(b);
- hcd.setInMemory(b);
- }
- }
- }
-
- public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName, boolean hasMob)
- throws IOException {
- // archive family store files
- Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
- HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
-
- // delete the family folder
- Path familyDir = new Path(tableDir,
- new Path(region.getEncodedName(), Bytes.toString(familyName)));
- if (fs.delete(familyDir, true) == false) {
- if (fs.exists(familyDir)) {
- throw new IOException("Could not delete family "
- + Bytes.toString(familyName) + " from FileSystem for region "
- + region.getRegionNameAsString() + "(" + region.getEncodedName()
- + ")");
- }
- }
-
- // archive and delete mob files
- if (hasMob) {
- Path mobTableDir =
- FSUtils.getTableDir(new Path(getRootDir(), MobConstants.MOB_DIR_NAME), region.getTable());
- HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(region.getTable());
- Path mobFamilyDir =
- new Path(mobTableDir,
- new Path(mobRegionInfo.getEncodedName(), Bytes.toString(familyName)));
- // archive mob family store files
- MobUtils.archiveMobStoreFiles(conf, fs, mobRegionInfo, mobFamilyDir, familyName);
-
- if (!fs.delete(mobFamilyDir, true)) {
- throw new IOException("Could not delete mob store files for family "
- + Bytes.toString(familyName) + " from FileSystem region "
- + mobRegionInfo.getRegionNameAsString() + "(" + mobRegionInfo.getEncodedName() + ")");
- }
- }
- }
-
- public void stop() {
- }
-
- public void logFileSystemState(Log log) throws IOException {
- FSUtils.logFileSystemState(fs, rootdir, log);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index 670642f..3dfe4a6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
index 4d19e9e..58b646b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
@@ -347,4 +347,4 @@ public class MasterWalManager {
public RecoveryMode getLogRecoveryMode() {
return this.splitLogManager.getRecoveryMode();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
index 9c21040..f52e321 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
@@ -41,8 +41,9 @@ import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.fs.FsContext;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.MetricsSnapshot;
import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
@@ -419,8 +420,7 @@ public class CloneSnapshotProcedure
// using a copy of descriptor, table will be created enabling first
HTableDescriptor underConstruction = new HTableDescriptor(hTableDescriptor);
final Path tempTableDir = FSUtils.getTableDir(tempdir, hTableDescriptor.getTableName());
- ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors()))
- .createTableDescriptorForTableDirectory(tempTableDir, underConstruction, false);
+ mfs.createTableDescriptor(FsContext.TEMP, underConstruction, false);
// 2. Create Regions
newRegions = hdfsRegionHandler.createHdfsRegions(
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
index c350185..7562a04 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
@@ -27,7 +27,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NamespaceExistException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.TableNamespaceManager;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
@@ -203,9 +202,7 @@ public class CreateNamespaceProcedure
protected static void createDirectory(
final MasterProcedureEnv env,
final NamespaceDescriptor nsDescriptor) throws IOException {
- MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
- mfs.getFileSystem().mkdirs(
- FSUtils.getNamespaceDir(mfs.getRootDir(), nsDescriptor.getName()));
+ env.getMasterServices().getMasterFileSystem().createNamespace(nsDescriptor);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
index 0dde420..f0a71b0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
@@ -38,14 +38,16 @@ import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.fs.FsContext;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CreateTableState;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
@@ -297,10 +299,9 @@ public class CreateTableProcedure
// 1. Create Table Descriptor
// using a copy of descriptor, table will be created enabling first
+ HTableDescriptor underConstruction = new HTableDescriptor(hTableDescriptor);
final Path tempTableDir = FSUtils.getTableDir(tempdir, hTableDescriptor.getTableName());
- ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors()))
- .createTableDescriptorForTableDirectory(
- tempTableDir, hTableDescriptor, false);
+ mfs.createTableDescriptor(FsContext.TEMP, underConstruction, false);
// 2. Create Regions
newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir,
@@ -319,6 +320,7 @@ public class CreateTableProcedure
final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
final Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), hTableDescriptor.getTableName());
FileSystem fs = mfs.getFileSystem();
+
if (!fs.delete(tableDir, true) && fs.exists(tableDir)) {
throw new IOException("Couldn't delete " + tableDir);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
index 2b042dd..c6ca374 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NamespaceNotFoundException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.constraint.ConstraintException;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
import org.apache.hadoop.hbase.master.TableNamespaceManager;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
@@ -275,23 +275,7 @@ public class DeleteNamespaceProcedure
protected static void deleteDirectory(
final MasterProcedureEnv env,
final String namespaceName) throws IOException {
- MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
- FileSystem fs = mfs.getFileSystem();
- Path p = FSUtils.getNamespaceDir(mfs.getRootDir(), namespaceName);
-
- try {
- for(FileStatus status : fs.listStatus(p)) {
- if (!HConstants.HBASE_NON_TABLE_DIRS.contains(status.getPath().getName())) {
- throw new IOException("Namespace directory contains table dir: " + status.getPath());
- }
- }
- if (!fs.delete(FSUtils.getNamespaceDir(mfs.getRootDir(), namespaceName), true)) {
- throw new IOException("Failed to remove namespace: " + namespaceName);
- }
- } catch (FileNotFoundException e) {
- // File already deleted, continue
- LOG.debug("deleteDirectory throws exception: " + e);
- }
+ env.getMasterServices().getMasterFileSystem().deleteNamespace(namespaceName);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index f8dd0b6..63a4f0f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -43,10 +43,10 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
import org.apache.hadoop.hbase.exceptions.HBaseException;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
index 8fa4183..d37159b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.BulkReOpen;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.util.Bytes;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
index b27a3b8..feb9228 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
import org.apache.hadoop.hbase.master.MetricsSnapshot;
import org.apache.hadoop.hbase.master.RegionStates;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
http://git-wip-us.apache.org/repos/asf/hbase/blob/6796171f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index abd69ad..3a5b974 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -51,8 +51,8 @@ import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.ipc.RpcServer;
+import org.apache.hadoop.hbase.fs.MasterFileSystem;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.MetricsMaster;
import org.apache.hadoop.hbase.master.SnapshotSentinel;