You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2015/09/17 01:53:55 UTC
[3/3] hbase git commit: Incomplete experiment (it does not compile
and has missing code)
Incomplete experiment (it does not compile and has missing code)
Trying to move everyting fs related to fs/
HMasterFileSystem has a LegacyMasterFs and a HierarchicalMasterFs impl
HRegionFileSystem has a LegacyRegionFs and a HierarchicalRegionFs impl
I still have the old MasterFileSystem around,
because I didn't have time to move it.
but the idea is to move everything that loops on the dirs
or knows about the layout in the MasterFs and RegionFs classes
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e7743d77
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e7743d77
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e7743d77
Branch: refs/heads/hbase-14439
Commit: e7743d779a8bfcd65d874afdb88d273ba9e5c4f2
Parents: f2856e2
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Wed Sep 16 14:12:39 2015 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Wed Sep 16 16:49:25 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/HTableDescriptor.java | 19 -
.../hadoop/hbase/fs/HMasterFileSystem.java | 61 +
.../hadoop/hbase/fs/HRegionFileSystem.java | 275 ++++
.../hbase/fs/HierarchicalMasterFileSystem.java | 49 +
.../hbase/fs/HierarchicalRegionFileSystem.java | 88 ++
.../hadoop/hbase/fs/LegacyMasterFileSystem.java | 152 +++
.../hadoop/hbase/fs/LegacyRegionFileSystem.java | 293 +++++
.../hadoop/hbase/fs/layout/AFsLayout.java | 55 -
.../hbase/fs/layout/AHFileLinkManager.java | 73 --
.../apache/hadoop/hbase/fs/layout/FsLayout.java | 220 ----
.../hbase/fs/layout/HierarchicalFsLayout.java | 171 ---
.../hbase/fs/layout/StandardHBaseFsLayout.java | 115 --
.../org/apache/hadoop/hbase/io/hfile/HFile.java | 29 +-
.../hbase/io/hfile/HFilePrettyPrinter.java | 38 +-
.../hadoop/hbase/master/CatalogJanitor.java | 27 +-
.../hadoop/hbase/master/MasterFileSystem.java | 40 +-
.../hadoop/hbase/master/RegionStates.java | 4 +-
.../hbase/regionserver/CompactionTool.java | 2 +-
.../hadoop/hbase/regionserver/HRegion.java | 98 +-
.../hbase/regionserver/HRegionFileSystem.java | 1216 ------------------
.../regionserver/HRegionFileSystemFactory.java | 30 -
.../hadoop/hbase/regionserver/HStore.java | 6 +-
.../HierarchicalHRegionFileSystem.java | 48 -
.../HierarchicalHRegionFileSystemFactory.java | 31 -
.../RegionDoesNotExistException.java | 42 +
.../RegionMergeTransactionImpl.java | 3 +-
.../regionserver/SplitTransactionImpl.java | 5 +-
.../hbase/regionserver/StoreFileInfo.java | 111 +-
.../hbase/snapshot/RestoreSnapshotHelper.java | 4 +-
.../hadoop/hbase/snapshot/SnapshotManifest.java | 5 +-
.../hbase/snapshot/SnapshotManifestV1.java | 8 +-
.../hadoop/hbase/util/FSRegionScanner.java | 52 +-
.../hadoop/hbase/util/FSTableDescriptors.java | 1 +
.../org/apache/hadoop/hbase/util/FSUtils.java | 368 +-----
.../org/apache/hadoop/hbase/util/FSVisitor.java | 231 ----
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 15 +-
.../hadoop/hbase/util/HFileV1Detector.java | 2 +-
.../hadoop/hbase/util/RegionSplitter.java | 8 +-
.../hbase/util/hbck/HFileCorruptionChecker.java | 5 -
.../apache/hadoop/hbase/util/TestFSVisitor.java | 229 ----
40 files changed, 1187 insertions(+), 3042 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 9145cdc..1722268 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -1368,25 +1368,6 @@ public class HTableDescriptor implements Comparable<HTableDescriptor> {
remove(match);
}
- /**
- * Returns the {@link Path} object representing the table directory under
- * path rootdir
- *
- * Deprecated use FSUtils.getTableDir() instead.
- *
- * @param rootdir qualified path of HBase root directory
- * @param tableName name of table
- * @return {@link Path} for table
- */
- @Deprecated
- public static Path getTableDir(Path rootdir, final byte [] tableName) {
- //This is bad I had to mirror code from FSUTils.getTableDir since
- //there is no module dependency between hbase-client and hbase-server
- TableName name = TableName.valueOf(tableName);
- return new Path(rootdir, new Path(HConstants.BASE_NAMESPACE_DIR,
- new Path(name.getNamespaceAsString(), new Path(name.getQualifierAsString()))));
- }
-
/** Table descriptor for <code>hbase:meta</code> catalog table
* Deprecated, use TableDescriptors#get(TableName.META_TABLE) or
* Admin#getTableDescriptor(TableName.META_TABLE) instead.
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HMasterFileSystem.java
new file mode 100644
index 0000000..8f92cf0
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HMasterFileSystem.java
@@ -0,0 +1,61 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+public abstract class HMasterFileSystem {
+ public abstract Collection<String> getNamespaces() throws IOException;
+ public abstract Collection<TableName> getTables(String namespace) throws IOException;
+ public abstract Collection<HRegionInfo> getRegions(TableName tableName) throws IOException;
+
+ public Collection<TableName> getTables() throws IOException {
+ ArrayList<TableName> tables = new ArrayList<TableName>();
+ for (String ns: getNamespaces()) {
+ tables.addAll(getTables(ns));
+ }
+ return tables;
+ }
+
+ public static HMasterFileSystem open(final Configuration conf) {
+ return open(conf, FSUtils.getRootDir(conf));
+ }
+
+ public static HMasterFileSystem open(Configuration conf, Path rootDir) {
+ String fsType = conf.get("hbase.fs.layout.type").toLowerCase();
+ switch (fsType) {
+ case "legacy":
+ return new LegacyMasterFileSystem(conf, rootDir);
+ case "hierarchical":
+ return new HierarchicalMasterFileSystem(conf, rootDir);
+ default:
+ throw new IOException("Invalid filesystem type " + fsType);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HRegionFileSystem.java
new file mode 100644
index 0000000..966ac06
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HRegionFileSystem.java
@@ -0,0 +1,275 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
+
+@InterfaceAudience.Private
+public abstract class HRegionFileSystem {
+ private static final Log LOG = LogFactory.getLog(HRegionFileSystem.class);
+
+ // blah... compat
+ public enum Type { TABLE, SNAPSHOT, ARCHIVE };
+
+ /** Name of the region info file that resides just under the region directory. */
+ public final static String REGION_INFO_FILE = ".regioninfo";
+
+ // ==========================================================================
+ // PUBLIC methods - add/remove/list store files
+ // ==========================================================================
+ /**
+ * Generate a unique temporary Path. Used in conjuction with commitStoreFile()
+ * to get a safer file creation.
+ * <code>
+ * Path file = fs.createTempName();
+ * ...StoreFile.Writer(file)...
+ * fs.commitStoreFile("family", file);
+ * </code>
+ *
+ * @return Unique {@link Path} of the temporary file
+ */
+ public Path createTempName() {
+ return createTempName(null);
+ }
+
+ /**
+ * Generate a unique temporary Path. Used in conjuction with commitStoreFile()
+ * to get a safer file creation.
+ * <code>
+ * Path file = fs.createTempName();
+ * ...StoreFile.Writer(file)...
+ * fs.commitStoreFile("family", file);
+ * </code>
+ *
+ * @param suffix extra information to append to the generated name
+ * @return Unique {@link Path} of the temporary file
+ */
+ public abstract Path createTempName(final String suffix); // TODO REMOVE THIS
+
+
+ /**
+ * Move the file from a build/temp location to the main family store directory.
+ * @param familyName Family that will gain the file
+ * @param buildPath {@link Path} to the file to commit.
+ * @return The new {@link Path} of the committed file
+ * @throws IOException
+ */
+ public Path commitStoreFile(final String familyName, final Path buildPath) throws IOException {
+ return commitStoreFile(familyName, buildPath, -1, false);
+ }
+
+ public abstract Path commitStoreFile(final String familyName, final Path buildPath,
+ final long seqNum, final boolean generateNewName) throws IOException;
+
+ /**
+ * Moves multiple store files to the relative region's family store directory.
+ * @param storeFiles list of store files divided by family
+ * @throws IOException
+ */
+ public abstract void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles)
+ throws IOException;
+
+ /**
+ * Bulk load: Add a specified store file to the specified family.
+ * If the source file is on the same different file-system is moved from the
+ * source location to the destination location, otherwise is copied over.
+ *
+ * @param familyName Family that will gain the file
+ * @param srcPath {@link Path} to the file to import
+ * @param seqNum Bulk Load sequence number
+ * @return The destination {@link Path} of the bulk loaded file
+ * @throws IOException
+ */
+ public abstract Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
+ throws IOException;
+
+ /**
+ * Archives the specified store file from the specified family.
+ * @param familyName Family that contains the store files
+ * @param filePath {@link Path} to the store file to remove
+ * @throws IOException if the archiving fails
+ */
+ public abstract void removeStoreFile(String familyName, Path filePath)
+ throws IOException;
+
+ /**
+ * Closes and archives the specified store files from the specified family.
+ * @param familyName Family that contains the store files
+ * @param storeFiles set of store files to remove
+ * @throws IOException if the archiving fails
+ */
+ public abstract void removeStoreFiles(String familyName, Collection<StoreFile> storeFiles)
+ throws IOException;
+
+ public abstract Path getStoreFilePath(final String familyName, final String fileName);
+
+ /**
+ * Returns the store files available for the family.
+ * This methods performs the filtering based on the valid store files.
+ * @param familyName Column Family Name
+ * @return a set of {@link StoreFileInfo} for the specified family.
+ */
+ public Collection<StoreFileInfo> getStoreFiles(final byte[] familyName) throws IOException {
+ return getStoreFiles(Bytes.toString(familyName));
+ }
+
+ public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
+ return getStoreFiles(familyName, true);
+ }
+
+ /**
+ * Returns the store files available for the family.
+ * This methods performs the filtering based on the valid store files.
+ * @param familyName Column Family Name
+ * @return a set of {@link StoreFileInfo} for the specified family.
+ */
+ public abstract Collection<StoreFileInfo> getStoreFiles(String familyName, boolean validate)
+ throws IOException;
+
+ /**
+ * Return the store file information of the specified family/file.
+ *
+ * @param familyName Column Family Name
+ * @param fileName File Name
+ * @return The {@link StoreFileInfo} for the specified family/file
+ */
+ public abstract StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
+ throws IOException;
+
+ /**
+ * @return the set of families present on disk
+ * @throws IOException
+ */
+ public abstract Collection<String> getFamilies() throws IOException;
+
+ // ==========================================================================
+ // ??? methods - do we still need this stuff in 2.0?
+ // ==========================================================================
+ public abstract void writeRecoveryCheckPoint() throws IOException;
+ public abstract void cleanup() throws IOException;
+
+ // ???
+ public void logFileSystemState(Log log) {}
+ public HRegionInfo getRegionInfoForFS() { return null; }
+ public FileSystem getFileSystem() { return null; }
+ public Path getRegionDir() { return null; }
+ public Path getStoreDir(String family) { return null; }
+
+ // remove on proc-v2 rewrite... move now in Merge/Split FS interface?
+ public void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException {}
+ public void commitDaughterRegion(HRegionInfo hri) {}
+ public void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException {}
+ public void commitMergedRegion(HRegionInfo hri) {}
+ public void createSplitsDir() throws IOException {}
+ public Path splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f,
+ final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy)
+ throws IOException { return null; }
+ public Path getMergesDir() { return null; }
+ public Path getMergesDir(final HRegionInfo hri) { return null; }
+ public void createMergesDir() throws IOException {}
+ public void assertReferenceFileCountOfSplitsDir(int expectedReferenceFileCount, HRegionInfo daughter) {}
+ public void assertReferenceFileCountOfDaughterDir(int expectedReferenceFileCount, HRegionInfo daughter) {}
+ public void cleanupSplitsDir() throws IOException {}
+ public void cleanupMergesDir() throws IOException {}
+ public Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
+ final StoreFile f, final Path mergedDir)
+ throws IOException { return null; }
+
+ public boolean hasReferences(final String familyName) { return false; }
+ public boolean hasReferences(final HTableDescriptor htd) { return false; }
+
+ public void openFamily(final String family) throws IOException {}
+
+ // ==========================================================================
+ // MAYBE methods
+ // ==========================================================================
+ public abstract HRegionInfo getRegionInfo();
+
+ public TableName getTable() {
+ return getRegionInfo().getTable();
+ }
+
+ // ==========================================================================
+ // PUBLIC methods - create/open/destroy
+ // ==========================================================================
+ public abstract void open(boolean rdonly) throws IOException;
+ public abstract void create() throws IOException;
+ public abstract void destroy() throws IOException;
+
+ // ==========================================================================
+ // PUBLIC Static/Factory methods - create/open/destroy
+ // ==========================================================================
+ public static HRegionFileSystem createSnapshot(final Configuration conf,
+ final HRegionInfo regionInfo) throws IOException {
+ // we need a dir on open, create if we want to reuse...
+ return null;
+ }
+
+ public static HRegionFileSystem create(final Configuration conf,
+ final HRegionInfo regionInfo) throws IOException {
+ HRegionFileSystem rfs = getInstance(conf, regionInfo);
+ rfs.create();
+ return rfs;
+ }
+
+ public static HRegionFileSystem destroy(final Configuration conf,
+ final HRegionInfo regionInfo) throws IOException {
+ HRegionFileSystem rfs = getInstance(conf, regionInfo);
+ rfs.destroy();
+ return rfs;
+ }
+
+ public static HRegionFileSystem open(final Configuration conf,
+ final HRegionInfo regionInfo, boolean readOnly) throws IOException {
+ HRegionFileSystem rfs = getInstance(conf, regionInfo);
+ rfs.open(readOnly);
+ return rfs;
+ }
+
+ private static HRegionFileSystem getInstance(final Configuration conf,
+ final HRegionInfo regionInfo) throws IOException {
+ String fsType = conf.get("hbase.fs.layout.type").toLowerCase();
+ switch (fsType) {
+ case "legacy":
+ return new LegacyRegionFileSystem(conf, regionInfo);
+ case "hierarchical":
+ return new HierarchicalRegionFileSystem(conf, regionInfo);
+ default:
+ throw new IOException("Invalid filesystem type " + fsType);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalMasterFileSystem.java
new file mode 100644
index 0000000..fef4686
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalMasterFileSystem.java
@@ -0,0 +1,49 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+public class HierarchicalMasterFileSystem extends LegacyMasterFileSystem {
+ public HierarchicalMasterFileSystem(Configuration conf, Path rootDir) {
+ super(conf, rootDir);
+ }
+
+ public Collection<HRegionInfo> getRegions(TableName tableName) throws IOException {
+ /*
+ for (...ns bucket..) {
+ for (...rs...) {
+ ...
+ }
+ }
+ */
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalRegionFileSystem.java
new file mode 100644
index 0000000..d6a6a97
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HierarchicalRegionFileSystem.java
@@ -0,0 +1,88 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+
+@InterfaceAudience.Private
+public class HierarchicalRegionFileSystem extends LegacyRegionFileSystem {
+ private static final Log LOG = LogFactory.getLog(HierarchicalRegionFileSystem.class);
+
+ private static final String OLD_REGION_NAME_PADDING = "abcdef1234abcdef1234abcdef1234ab";
+
+ /** Number of characters for DIR name, 4 characters for 16^4 = 65536 buckets. */
+ public static final int HUMONGOUS_DIR_NAME_SIZE = 4;
+
+ public HierarchicalRegionFileSystem(Configuration conf, HRegionInfo hri) {
+ super(conf, hri);
+ }
+
+ // ==========================================================================
+ // PROTECTED FS-Layout Internals
+ // ==========================================================================
+ @Override
+ protected Path getRegionDir(Path baseDir) {
+ return getHumongousRegionDir(baseDir, getTable(), getRegionInfo().getEncodedName());
+ }
+
+ private Path getHumongousRegionDir(Path baseDir, TableName tableName, String name) {
+ if (name.length() != HRegionInfo.MD5_HEX_LENGTH) {
+ String table = tableName.getQualifierAsString();
+ String namespace = tableName.getNamespaceAsString();
+
+ // Meta and old root table use the old encoded name format still
+ if (!namespace.equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
+ throw new IllegalArgumentException("The region with encoded name " + name
+ + " is not a humongous region, cannot get humongous region dir from it.");
+ }
+ if (!table.equals(TableName.META_TABLE_NAME.getQualifierAsString()) &&
+ !table.equals(TableName.OLD_ROOT_TABLE_NAME.getQualifierAsString())) {
+ throw new IllegalArgumentException("The region with encoded name " + name
+ + " is not a humongous region, cannot get humongous region dir from it.");
+ }
+
+ // Add padding to guarantee we will have enough characters
+ return new Path(new Path(baseDir, makeBucketName(name, OLD_REGION_NAME_PADDING)), name);
+ }
+ return new Path(new Path(baseDir, makeBucketName(name, null)), name);
+ }
+
+ private String makeBucketName(String regionName, String padding) {
+ if (padding != null) {
+ regionName = regionName + padding;
+ }
+ return regionName.substring(HRegionInfo.MD5_HEX_LENGTH - HUMONGOUS_DIR_NAME_SIZE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyMasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyMasterFileSystem.java
new file mode 100644
index 0000000..87ee61b
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyMasterFileSystem.java
@@ -0,0 +1,152 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+@InterfaceAudience.Private
+public class LegacyMasterFileSystem extends HMasterFileSystem {
+ private final Path activeNsDir;
+ private final Path rootDir;
+
+ private final FileSystem fs;
+
+ public LegacyMasterFileSystem(Configuration conf, Path rootDir) {
+ this.fs = rootDir.getFileSystem(conf);
+ this.rootDir = rootDir;
+ this.activeNsDir = new Path(rootDir, HConstants.BASE_NAMESPACE_DIR);
+ }
+
+ public Collection<String> getNamespaces() throws IOException {
+ return getNamespaces(activeNsDir);
+ }
+
+ protected Collection<String> getNamespaces(Path baseDir) throws IOException {
+ FileStatus[] activeNsDirs = fs.globStatus(new Path(baseDir, "*"));
+ if (activeNsDirs == null || activeNsDirs.length == 0) {
+ return Collections.emptyList();
+ }
+ ArrayList<String> namespaces = new ArrayList<String>(activeNsDirs.length);
+ for (int i = 0; i < activeNsDirs.length; ++i) {
+ namespaces.add(activeNsDirs[i].getPath().getName());
+ }
+ return namespaces;
+ }
+
+ public Collection<TableName> getTables(String namespace) throws IOException {
+ Path baseDir = new Path(activeNsDir, namespace);
+ FileStatus[] dirs = fs.listStatus(baseDir, new UserTableDirFilter(fs));
+ if (dirs == null || dirs.length == 0) {
+ return Collections.emptyList();
+ }
+
+ ArrayList<TableName> tables = new ArrayList<TableName>(dirs.length);
+ for (int i = 0; i < dirs.length; ++i) {
+ tables.add(TableName.valueOf(namespace, dirs[i].getPath().getName()));
+ }
+ return tables;
+ }
+
+ public Collection<HRegionInfo> getRegions(TableName tableName) throws IOException {
+ return null;
+ }
+
+ protected Collection<HRegionInfo> getRegions(Path baseDir) throws IOException {
+ FileStatus[] dirs = fs.listStatus(baseDir, new RegionDirFilter(fs));
+ if (dirs == null || dirs.length == 0) {
+ return Collections.emptyList();
+ }
+
+ ArrayList<HRegionInfo> hriList = new ArrayList<HRegionInfo>(dirs.length);
+ for (int i = 0; i < dirs.length; ++i) {
+ // TODO: Load HRI
+ }
+ return hriList;
+ }
+
+ /**
+ * A {@link PathFilter} that returns usertable directories. To get all directories use the
+ * {@link BlackListDirFilter} with a <tt>null</tt> blacklist
+ */
+ private static class UserTableDirFilter extends FSUtils.BlackListDirFilter {
+ public UserTableDirFilter(FileSystem fs) {
+ super(fs, HConstants.HBASE_NON_TABLE_DIRS);
+ }
+
+ protected boolean isValidName(final String name) {
+ if (!super.isValidName(name))
+ return false;
+
+ try {
+ TableName.isLegalTableQualifierName(Bytes.toBytes(name));
+ } catch (IllegalArgumentException e) {
+ LOG.info("INVALID NAME " + name);
+ return false;
+ }
+ return true;
+ }
+ }
+
+ /**
+ * Filter for all dirs that don't start with '.'
+ */
+ private static class RegionDirFilter implements PathFilter {
+ // This pattern will accept 0.90+ style hex region dirs and older numeric region dir names.
+ final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$");
+ final FileSystem fs;
+
+ public RegionDirFilter(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ @Override
+ public boolean accept(Path rd) {
+ if (!regionDirPattern.matcher(rd.getName()).matches()) {
+ return false;
+ }
+
+ try {
+ return fs.getFileStatus(rd).isDirectory();
+ } catch (IOException ioe) {
+ // Maybe the file was moved or the fs was disconnected.
+ LOG.warn("Skipping file " + rd +" due to IOException", ioe);
+ return false;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyRegionFileSystem.java
new file mode 100644
index 0000000..9f8cc70
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/LegacyRegionFileSystem.java
@@ -0,0 +1,293 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.fs;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.fs.HFileSystem;
+import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSHDFSUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
+
+import com.google.common.annotations.VisibleForTesting;
+
+@InterfaceAudience.Private
+public class LegacyRegionFileSystem extends HRegionFileSystem {
+ private static final Log LOG = LogFactory.getLog(LegacyRegionFileSystem.class);
+
+ /** Temporary subdirectory of the region directory used for compaction output. */
+ protected static final String REGION_TEMP_DIR = ".tmp";
+
+ private final Configuration conf;
+ private final HRegionInfo hri;
+ private final Path regionDir;
+ private final FileSystem fs;
+
+ public LegacyRegionFileSystem(Configuration conf, HRegionInfo hri) {
+ this.fs = FSUtils.getCurrentFileSystem(conf);
+ this.hri = hri;
+ this.conf = conf;
+ this.regionDir = getRegionDir(getTableDir(getRootDir()));
+ }
+
+ // ==========================================================================
+ // ??? methods - do we still need this stuff in 2.0?
+ // ==========================================================================
+ @Override
+ public void writeRecoveryCheckPoint() throws IOException {
+ //checkRegionInfoOnFilesystem
+ }
+
+ @Override
+ public void cleanup() throws IOException {
+ /*
+ // Remove temporary data left over from old regions
+ status.setStatus("Cleaning up temporary data from old regions");
+ fs.cleanupTempDir();
+
+ status.setStatus("Cleaning up detritus from prior splits");
+ // Get rid of any splits or merges that were lost in-progress. Clean out
+ // these directories here on open. We may be opening a region that was
+ // being split but we crashed in the middle of it all.
+ fs.cleanupAnySplitDetritus();
+ fs.cleanupMergesDir();
+ */
+ }
+
+ // ==========================================================================
+ // PUBLIC methods - create/open/destroy
+ // ==========================================================================
+ @Override
+ public void open(boolean rdonly) throws IOException {
+ }
+
+ @Override
+ public void create() throws IOException {
+ }
+
+ @Override
+ public void destroy() throws IOException {
+ }
+
+ // ==========================================================================
+ // PUBLIC methods - add/remove/list store files
+ // ==========================================================================
+ public Path createTempName(final String suffix) {
+ return new Path(getTempDir(), generateUniqueName(suffix));
+ }
+
+ public Path commitStoreFile(final String familyName, final Path buildPath,
+ final long seqNum, final boolean generateNewName) throws IOException {
+ return null;
+ }
+
+ public void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles)
+ throws IOException {
+ }
+
+ public Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
+ throws IOException {
+ return null;
+ }
+
+ public void removeStoreFile(String familyName, Path filePath)
+ throws IOException {
+ }
+
+ public void removeStoreFiles(String familyName, Collection<StoreFile> storeFiles)
+ throws IOException {
+ }
+
+ public Path getStoreFilePath(final String familyName, final String fileName) {
+ return null;
+ }
+
+ /**
+ * Returns the store files available for the family.
+ * This methods performs the filtering based on the valid store files.
+ * @param familyName Column Family Name
+ * @return a set of {@link StoreFileInfo} for the specified family.
+ */
+ public Collection<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate)
+ throws IOException {
+ /*
+ Path familyDir = getStoreDir(this.regionDir, familyName);
+ FileStatus[] files = FSUtils.listStatus(this.fs, familyDir);
+ if (files == null) {
+ LOG.debug("No StoreFiles for: " + familyDir);
+ return null;
+ }
+
+ ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(files.length);
+ for (FileStatus status: files) {
+ if (validate && !StoreFileInfo.isValid(status)) {
+ LOG.warn("Invalid StoreFile: " + status.getPath());
+ continue;
+ }
+ StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo,
+ regionInfoForFs, familyName, status.getPath());
+ storeFiles.add(info);
+ }
+ return storeFiles;
+ */
+ return null;
+ }
+
+ public StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
+ throws IOException {
+ return null;
+ }
+
+ /**
+ * @return the set of families present on disk
+ * @throws IOException
+ */
+ public Collection<String> getFamilies() throws IOException {
+ FileStatus[] fds = FSUtils.listStatus(fs, this.regionDir, new FamilyDirFilter(fs));
+ if (fds == null) return null;
+
+ ArrayList<String> families = new ArrayList<String>(fds.length);
+ for (FileStatus status: fds) {
+ families.add(status.getPath().getName());
+ }
+
+ return families;
+ }
+
+ // ==========================================================================
+ // PROTECTED Internals
+ // ==========================================================================
+
+ /**
+ * Generate a unique file name, used by createTempName() and commitStoreFile()
+ * @param suffix extra information to append to the generated name
+ * @return Unique file name
+ */
+ protected static String generateUniqueName(final String suffix) {
+ String name = UUID.randomUUID().toString().replaceAll("-", "");
+ if (suffix != null) name += suffix;
+ return name;
+ }
+
+ public HRegionInfo getRegionInfo() {
+ return hri;
+ }
+
+ // ==========================================================================
+ // PROTECTED FS-Layout Internals
+ // ==========================================================================
+ /** @return {@link Path} to the region's temp directory, used for file creations */
+ Path getTempDir() {
+ return new Path(getRegionDir(), REGION_TEMP_DIR);
+ }
+
+ protected Path getRootDir() {
+ return new Path("/");
+ }
+
+ protected Path getTableDir(Path baseDir) {
+ TableName table = getTable();
+ return new Path(baseDir, new Path(table.getNamespaceAsString(), table.getQualifierAsString()));
+ }
+
+ protected Path getRegionDir(Path baseDir) {
+ return new Path(baseDir, getRegionInfo().getEncodedName());
+ }
+
+ protected Path getStoreDir(Path baseDir, String familyName) {
+ return new Path(baseDir, familyName);
+ }
+
+ protected Path getStoreFilePath(Path baseDir, String family, String name) {
+ return getStoreFilePath(getStoreDir(baseDir, family), name);
+ }
+
+ protected Path getStoreFilePath(Path baseDir, String name) {
+ return new Path(baseDir, name);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s(%s)", getClass().getName(), regionDir);
+ }
+
+ /**
+ * Filter for all dirs that are legal column family names. This is generally used for colfam
+ * dirs <hbase.rootdir>/<tabledir>/<regiondir>/<colfamdir>.
+ */
+ private static class FamilyDirFilter implements PathFilter {
+ final FileSystem fs;
+
+ public FamilyDirFilter(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ @Override
+ public boolean accept(Path rd) {
+ try {
+ // throws IAE if invalid
+ HColumnDescriptor.isLegalFamilyName(Bytes.toBytes(rd.getName()));
+ } catch (IllegalArgumentException iae) {
+ // path name is an invalid family name and thus is excluded.
+ return false;
+ }
+
+ try {
+ return fs.getFileStatus(rd).isDirectory();
+ } catch (IOException ioe) {
+ // Maybe the file was moved or the fs was disconnected.
+ LOG.warn("Skipping file " + rd +" due to IOException", ioe);
+ return false;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AFsLayout.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AFsLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AFsLayout.java
deleted file mode 100644
index 8fd651f..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AFsLayout.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.fs.layout;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystemFactory;
-import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
-import org.apache.hadoop.hbase.util.FSUtils;
-
-public abstract class AFsLayout {
- protected final static Log LOG = LogFactory.getLog(AFsLayout.class);
-
- public abstract HRegionFileSystemFactory getHRegionFileSystemFactory();
-
- public abstract Path makeHFileLinkPath(SnapshotManifest snapshotManifest, HRegionInfo regionInfo, String familyName, String hfileName);
-
- public abstract Path getRegionArchiveDir(Path rootDir, TableName tableName, Path regiondir);
-
- public abstract Path getTableDirFromRegionDir(Path regionDir);
-
- public abstract List<Path> getRegionDirPaths(FileSystem fs, Path tableDir) throws IOException;
-
- public abstract List<FileStatus> getRegionDirFileStats(FileSystem fs, Path tableDir, FSUtils.RegionDirFilter filter) throws IOException;
-
- public abstract Path getRegionDir(Path tableDir, String name);
-
- protected AFsLayout() {
- super();
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AHFileLinkManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AHFileLinkManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AHFileLinkManager.java
deleted file mode 100644
index 0eb2489..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/AHFileLinkManager.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.fs.layout;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.io.HFileLink;
-import org.apache.hadoop.hbase.util.Pair;
-
-public abstract class AHFileLinkManager {
- public abstract HFileLink buildFromHFileLinkPattern(Configuration conf, Path hFileLinkPattern) throws IOException;
-
- public abstract HFileLink buildFromHFileLinkPattern(final Path rootDir, final Path archiveDir, final Path hFileLinkPattern);
-
- public abstract Path createPath(final TableName table, final String region, final String family, final String hfile);
-
- public abstract HFileLink build(final Configuration conf, final TableName table, final String region, final String family, final String hfile)
- throws IOException;
-
- public abstract boolean isHFileLink(final Path path);
-
- public abstract boolean isHFileLink(String fileName);
-
- public abstract String getReferencedHFileName(final String fileName);
-
- public abstract Path getHFileLinkPatternRelativePath(Path path);
-
- public abstract String getReferencedRegionName(final String fileName);
-
- public abstract TableName getReferencedTableName(final String fileName);
-
- public abstract String createHFileLinkName(final HRegionInfo hfileRegionInfo, final String hfileName);
-
- public abstract String createHFileLinkName(final TableName tableName, final String regionName, final String hfileName);
-
- public abstract boolean create(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final HRegionInfo hfileRegionInfo, final String hfileName)
- throws IOException;
-
- public abstract boolean createFromHFileLink(final Configuration conf, final FileSystem fs, final Path dstFamilyPath, final String hfileLinkName) throws IOException;
-
- public abstract String createBackReferenceName(final String tableNameStr, final String regionName);
-
- public abstract Path getHFileFromBackReference(final Path rootDir, final Path linkRefPath);
-
- public abstract Pair<TableName, String> parseBackReferenceName(String name);
-
- public abstract Path getHFileFromBackReference(final Configuration conf, final Path linkRefPath) throws IOException;
-
- public AHFileLinkManager() {
- super();
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/FsLayout.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/FsLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/FsLayout.java
deleted file mode 100644
index c97d2a0..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/FsLayout.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.fs.layout;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystemFactory;
-import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Throwables;
-import com.google.common.primitives.Ints;
-
-import edu.umd.cs.findbugs.annotations.NonNull;
-import edu.umd.cs.findbugs.annotations.Nullable;
-
-/**
- * This class exists mostly to allow us to access the layouts statically for convenience, as
- * though the layout classes are Util files.
- */
-public class FsLayout {
- private static final Log LOG = LogFactory.getLog(FsLayout.class);
-
- public static final String FS_LAYOUT_CHOICE = "hbase.fs.layout.choose";
- public static final String FS_LAYOUT_DETECT = "hbase.fs.layout.detect";
- public static final String FS_LAYOUT_DETECT_STRICT = "hbase.fs.layout.detect.strict";
- public static final String FS_LAYOUT_FILE_NAME = ".fslayout";
-
- // TODO: How can we avoid having a volatile variable (slightly slower reads)?
- // TODO: Move FsLayout class/contents into FSUtils?
- private static volatile AFsLayout fsLayout = null;
-
- public static final PathFilter FS_LAYOUT_PATHFILTER = new PathFilter() {
- @Override
- public boolean accept(Path p) {
- return p.getName().equals(FS_LAYOUT_FILE_NAME);
- }
- };
-
- @VisibleForTesting
- static AFsLayout getRaw() {
- return fsLayout;
- }
-
- public static AFsLayout get() {
- AFsLayout curLayout = fsLayout;
- if (curLayout == null) {
- return initialize(null);
- } else {
- return curLayout;
- }
- }
-
- @VisibleForTesting
- public static void reset() {
- LOG.debug("Resetting FS layout to null");
- fsLayout = null;
- }
-
- @VisibleForTesting
- public static void setLayoutForTesting(@NonNull AFsLayout inputLayout) {
- LOG.debug("Setting FS layout to: " + inputLayout.getClass().getSimpleName());
- fsLayout = inputLayout;
- }
-
- /*
- * TODO: Should this be required to be called manually?
- * Maybe call it manually in some processes (master/regionserver) and automatically everywhere else
- */
- @VisibleForTesting
- static synchronized AFsLayout initialize(Configuration conf) {
- try {
- if (fsLayout != null) {
- LOG.debug("Already initialized FS layout, not going to re-initialize");
- return fsLayout;
- }
- if (conf == null) {
- conf = HBaseConfiguration.create();
- }
- String choice = conf.get(FS_LAYOUT_CHOICE, null);
- boolean autodetect = conf.getBoolean(FS_LAYOUT_DETECT, false);
- if (choice != null && autodetect) {
- throw new IllegalStateException("Configuration both chooses a layout and "
- + "tries to automatically detect the layout");
- }
- if (choice != null) {
- Class<?> layoutClass = Class.forName(choice);
- Method getMethod = layoutClass.getMethod("get");
- return (AFsLayout) getMethod.invoke(null);
- }
- if (autodetect) {
- LOG.debug("Trying to detect hbase layout on filesystem");
- FileSystem fs = FSUtils.getCurrentFileSystem(conf);
- Path rootDir = FSUtils.getRootDir(conf);
- AFsLayout fsLayoutFromFile = readLayoutFile(fs, rootDir);
- if (fsLayoutFromFile == null) {
- if (conf.getBoolean(FS_LAYOUT_DETECT_STRICT, false)) {
- throw new IllegalStateException("Tried to detect fs layout, but there was no layout file at the root!");
- } else {
- LOG.debug("Didn't find a layout file, assuming classical hbase fs layout");
- fsLayout = StandardHBaseFsLayout.get();
- }
- } else {
- LOG.info("Detected hbase fs layout: " + fsLayoutFromFile.getClass().getSimpleName());
- fsLayout = fsLayoutFromFile;
- }
- } else {
- fsLayout = StandardHBaseFsLayout.get();
- }
- } catch (Exception e) {
- Throwables.propagate(e);
- }
- return fsLayout;
- }
-
- public static AFsLayout readLayoutFile(FileSystem fs, Path rootDir)
- throws FileNotFoundException, IOException, ClassNotFoundException,
- NoSuchMethodException, SecurityException, IllegalAccessException,
- IllegalArgumentException, InvocationTargetException {
- Path layoutFilePath = new Path(rootDir, FS_LAYOUT_FILE_NAME);
- FileStatus[] statuses = fs.listStatus(rootDir, FS_LAYOUT_PATHFILTER);
- if (statuses.length != 1) {
- return null;
- }
- FileStatus stat = statuses[0];
- int len = Ints.checkedCast(stat.getLen());
- byte[] inputStreamBytes = new byte[len];
- FSDataInputStream inputStream = fs.open(layoutFilePath);
- inputStream.readFully(inputStreamBytes);
- inputStream.close();
- String layoutClassName = Bytes.toString(inputStreamBytes);
- Class<?> layoutClass = Class.forName(layoutClassName);
- Method getMethod = layoutClass.getMethod("get");
- return (AFsLayout) getMethod.invoke(null);
- }
-
- public static void writeLayoutFile(FileSystem fs, Path rootDir, AFsLayout fsLayout, boolean overwrite)
- throws IOException {
- Path layoutFilePath = new Path(rootDir, FS_LAYOUT_FILE_NAME);
- FSDataOutputStream outputStream = fs.create(layoutFilePath, overwrite);
- try {
- outputStream.write(Bytes.toBytes(fsLayout.getClass().getCanonicalName()));
- } finally {
- outputStream.close();
- }
- }
-
- public static boolean deleteLayoutFile(FileSystem fs, Path rootDir) throws IOException {
- Path layoutFilePath = new Path(rootDir, FS_LAYOUT_FILE_NAME);
- return fs.delete(layoutFilePath, false);
- }
-
- public static HRegionFileSystemFactory getHRegionFileSystemFactory() {
- return get().getHRegionFileSystemFactory();
- }
-
- public static Path getRegionDir(Path tableDir, HRegionInfo regionInfo) {
- return getRegionDir(tableDir, regionInfo.getEncodedName());
- }
-
- public static Path getRegionDir(Path tableDir, String name) {
- return get().getRegionDir(tableDir, name);
- }
-
- @Nullable
- public static List<FileStatus> getRegionDirFileStats(FileSystem fs, Path tableDir, RegionDirFilter filter) throws IOException {
- return get().getRegionDirFileStats(fs, tableDir, filter);
- }
-
- public static List<Path> getRegionDirPaths(FileSystem fs, Path tableDir) throws IOException {
- return get().getRegionDirPaths(fs, tableDir);
- }
-
- public static Path getTableDirFromRegionDir(Path regionDir) {
- return get().getTableDirFromRegionDir(regionDir);
- }
-
- public static Path getRegionArchiveDir(Path rootDir, TableName tableName, Path regiondir) {
- return get().getRegionArchiveDir(rootDir, tableName, regiondir);
- }
-
- public static Path makeHFileLinkPath(SnapshotManifest snapshotManifest, HRegionInfo regionInfo, String familyName, String hfileName) {
- return get().makeHFileLinkPath(snapshotManifest, regionInfo, familyName, hfileName);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/HierarchicalFsLayout.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/HierarchicalFsLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/HierarchicalFsLayout.java
deleted file mode 100644
index d17428a..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/HierarchicalFsLayout.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.fs.layout;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
-import org.apache.hadoop.hbase.regionserver.HierarchicalHRegionFileSystemFactory;
-import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter;
-import org.apache.hadoop.hbase.util.HFileArchiveUtil;
-
-
-public class HierarchicalFsLayout extends AFsLayout {
- private static final String OLD_REGION_NAME_PADDING = "abcdef1234abcdef1234abcdef1234ab";
- private static final HierarchicalFsLayout LAYOUT = new HierarchicalFsLayout();
-
- static {
- assert OLD_REGION_NAME_PADDING.length() == 32;
- }
-
- public static HierarchicalFsLayout get() { return LAYOUT; }
-
- private HierarchicalFsLayout() { }
-
- @Override
- public HierarchicalHRegionFileSystemFactory getHRegionFileSystemFactory() {
- return new HierarchicalHRegionFileSystemFactory();
- }
-
- @Override
- public Path getRegionDir(Path tableDir, String name) {
- return getHumongousRegionDir(tableDir, name);
- }
-
- private Path getHumongousRegionDir(final Path tabledir, final String name) {
- if (name.length() != HRegionInfo.MD5_HEX_LENGTH) {
- String table = tabledir.getName();
- String namespace = tabledir.getParent().getName();
-
- // Meta and old root table use the old encoded name format still
- if (!namespace.equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
- throw new IllegalArgumentException("The region with encoded name " + name
- + " is not a humongous region, cannot get humongous region dir from it.");
- }
- if (!table.equals(TableName.META_TABLE_NAME.getQualifierAsString()) &&
- !table.equals(TableName.OLD_ROOT_TABLE_NAME.getQualifierAsString())) {
- throw new IllegalArgumentException("The region with encoded name " + name
- + " is not a humongous region, cannot get humongous region dir from it.");
- }
-
- // Add padding to guarantee we will have enough characters
- return new Path(new Path(tabledir, makeBucketName(name, OLD_REGION_NAME_PADDING)), name);
- }
- return new Path(new Path(tabledir, makeBucketName(name, null)), name);
- }
-
- private String makeBucketName(String regionName, String padding) {
- if (padding != null) {
- regionName = regionName + padding;
- }
- return regionName.substring(HRegionInfo.MD5_HEX_LENGTH
- - HRegionFileSystem.HUMONGOUS_DIR_NAME_SIZE);
- }
-
- @Override
- public List<FileStatus> getRegionDirFileStats(FileSystem fs, Path tableDir, RegionDirFilter filter)
- throws IOException {
- FileStatus[] buckets = FSUtils.listStatus(fs, tableDir);
- if (buckets == null) {
- return null;
- }
- List<FileStatus> stats = new ArrayList<FileStatus>();
- for (FileStatus bucket : buckets) {
- FileStatus[] regionDirs = null;
- if (filter != null) {
- regionDirs = fs.listStatus(bucket.getPath(), filter);
- } else {
- regionDirs = fs.listStatus(bucket.getPath());
- }
- for (FileStatus regionDir : regionDirs) {
- stats.add(regionDir);
- }
- }
- if (stats.size() == 0) {
- return null;
- }
- return stats;
- }
-
- /**
- * Given a particular table dir, return all the regiondirs inside it, excluding files such as
- * .tableinfo
- * @param fs A file system for the Path
- * @param tableDir Path to a specific table directory <hbase.rootdir>/<tabledir>
- * @return List of paths to valid region directories in table dir.
- * @throws IOException
- */
- @Override
- public List<Path> getRegionDirPaths(final FileSystem fs, final Path tableDir) throws IOException {
- // assumes we are in a table dir.
- FileStatus[] rds = fs.listStatus(tableDir, new FSUtils.RegionDirFilter(fs));
- List<Path> regionDirs = new ArrayList<Path>();
- for (FileStatus rdfs : rds) {
- // get all region dirs from bucket dir
- FileStatus[] bucket_rds = fs.listStatus(rdfs.getPath(),
- new FSUtils.RegionDirFilter(fs));
- for (FileStatus bucket_rdfs : bucket_rds) {
- regionDirs.add(bucket_rdfs.getPath());
- }
- }
- return regionDirs;
- }
-
- @Override
- public Path getTableDirFromRegionDir(Path regionDir) {
- return regionDir.getParent().getParent();
- }
-
- /**
- * Get the archive directory for a given region under the specified table
- * @param tableName the table name. Cannot be null.
- * @param regiondir the path to the region directory. Cannot be null.
- * @return {@link Path} to the directory to archive the given region, or <tt>null</tt> if it
- * should not be archived
- */
- @Override
- public Path getRegionArchiveDir(Path rootDir,
- TableName tableName,
- Path regiondir) {
- // get the archive directory for a table
- Path archiveDir = HFileArchiveUtil.getTableArchivePath(rootDir, tableName);
-
- // then add on the region path under the archive
- String encodedRegionName = regiondir.getName();
- String parentName = regiondir.getParent().getName();
-
- return new Path(archiveDir, new Path(parentName, encodedRegionName));
- }
-
- @Override
- public Path makeHFileLinkPath(SnapshotManifest snapshotManifest, HRegionInfo regionInfo, String familyName, String hfileName) {
- return new Path(new Path(getHumongousRegionDir(snapshotManifest.getSnapshotDir(),
- regionInfo.getEncodedName()), familyName), hfileName);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/StandardHBaseFsLayout.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/StandardHBaseFsLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/StandardHBaseFsLayout.java
deleted file mode 100644
index 98edb78..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/layout/StandardHBaseFsLayout.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.fs.layout;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystemFactory;
-import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter;
-import org.apache.hadoop.hbase.util.HFileArchiveUtil;
-
-
-public class StandardHBaseFsLayout extends AFsLayout {
- private static final StandardHBaseFsLayout LAYOUT = new StandardHBaseFsLayout();
-
- public static StandardHBaseFsLayout get() { return LAYOUT; }
-
- private StandardHBaseFsLayout() { }
-
- @Override
- public HRegionFileSystemFactory getHRegionFileSystemFactory() {
- return new HRegionFileSystemFactory();
- }
-
- @Override
- public Path getRegionDir(Path tableDir, String name) {
- return new Path(tableDir, name);
- }
-
- @Override
- public List<FileStatus> getRegionDirFileStats(FileSystem fs, Path tableDir, RegionDirFilter filter)
- throws IOException {
- FileStatus[] rds = FSUtils.listStatus(fs, tableDir, filter);
- if (rds == null) {
- return null;
- }
- List<FileStatus> regionStatus = new ArrayList<FileStatus>(rds.length);
- for (FileStatus rdfs : rds) {
- regionStatus.add(rdfs);
- }
- return regionStatus;
- }
-
- /**
- * Given a particular table dir, return all the regiondirs inside it, excluding files such as
- * .tableinfo
- * @param fs A file system for the Path
- * @param tableDir Path to a specific table directory <hbase.rootdir>/<tabledir>
- * @return List of paths to valid region directories in table dir.
- * @throws IOException
- */
- @Override
- public List<Path> getRegionDirPaths(FileSystem fs, Path tableDir) throws IOException {
- // assumes we are in a table dir.
- FileStatus[] rds = fs.listStatus(tableDir, new FSUtils.RegionDirFilter(fs));
- List<Path> regionDirs = new ArrayList<Path>();
- for (FileStatus rdfs : rds) {
- regionDirs.add(rdfs.getPath());
- }
- return regionDirs;
- }
-
- @Override
- public Path getTableDirFromRegionDir(Path regionDir) {
- return regionDir.getParent();
- }
-
- /**
- * Get the archive directory for a given region under the specified table
- * @param tableName the table name. Cannot be null.
- * @param regiondir the path to the region directory. Cannot be null.
- * @return {@link Path} to the directory to archive the given region, or <tt>null</tt> if it
- * should not be archived
- */
- @Override
- public Path getRegionArchiveDir(Path rootDir,
- TableName tableName,
- Path regiondir) {
- // get the archive directory for a table
- Path archiveDir = HFileArchiveUtil.getTableArchivePath(rootDir, tableName);
- // then add on the region path under the archive
- String encodedRegionName = regiondir.getName();
- return new Path(archiveDir, encodedRegionName);
- }
-
- @Override
- public Path makeHFileLinkPath(SnapshotManifest snapshotManifest, HRegionInfo regionInfo, String familyName, String hfileName) {
- return new Path(new Path(new Path(snapshotManifest.getSnapshotDir(),
- regionInfo.getEncodedName()), familyName), hfileName);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index d18dada..4ad310d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -248,7 +248,7 @@ public class HFile {
protected FileSystem fs;
protected Path path;
protected FSDataOutputStream ostream;
- protected CellComparator comparator =
+ protected CellComparator comparator =
CellComparator.COMPARATOR;
protected InetSocketAddress[] favoredNodes;
private HFileContext fileContext;
@@ -830,33 +830,6 @@ public class HFile {
}
/**
- * Returns all HFiles belonging to the given region directory. Could return an
- * empty list.
- *
- * @param fs The file system reference.
- * @param regionDir The region directory to scan.
- * @return The list of files found.
- * @throws IOException When scanning the files fails.
- */
- static List<Path> getStoreFiles(FileSystem fs, Path regionDir)
- throws IOException {
- List<Path> regionHFiles = new ArrayList<Path>();
- PathFilter dirFilter = new FSUtils.DirFilter(fs);
- FileStatus[] familyDirs = fs.listStatus(regionDir, dirFilter);
- for(FileStatus dir : familyDirs) {
- FileStatus[] files = fs.listStatus(dir.getPath());
- for (FileStatus file : files) {
- if (!file.isDirectory() &&
- (!file.getPath().toString().contains(HConstants.HREGION_OLDLOGDIR_NAME)) &&
- (!file.getPath().toString().contains(HConstants.RECOVERED_EDITS_DIR))) {
- regionHFiles.add(file.getPath());
- }
- }
- }
- return regionHFiles;
- }
-
- /**
* Checks the given {@link HFile} format version, and throws an exception if
* invalid. Note that if the version number comes from an input file and has
* not been verified, the caller needs to re-throw an {@link IOException} to
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
index 757ad1f..0067de6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
@@ -24,6 +24,7 @@ import java.io.DataInput;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -56,10 +57,11 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.fs.HRegionFileSystem;
import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.BloomFilter;
import org.apache.hadoop.hbase.util.BloomFilterUtil;
import org.apache.hadoop.hbase.util.BloomFilterFactory;
@@ -181,31 +183,28 @@ public class HFilePrettyPrinter extends Configured implements Tool {
String regionName = cmd.getOptionValue("r");
byte[] rn = Bytes.toBytes(regionName);
HRegionInfo hri = HRegionInfo.parseRegionInfoFromRegionName(rn);
- Path rootDir = FSUtils.getRootDir(getConf());
- Path tableDir = FSUtils.getTableDir(rootDir, TableName.valueOf(
- hri.getTable().getNameAsString()));
- FileSystem fs = FileSystem.get(getConf());
- Path regionDir = HRegionFileSystem.create(getConf(), fs, tableDir, hri).getRegionDir();
- if (verbose)
- System.out.println("region dir -> " + regionDir);
- List<Path> regionFiles = HFile.getStoreFiles(fs,
- regionDir);
- if (verbose)
- System.out.println("Number of region files found -> "
- + regionFiles.size());
- if (verbose) {
+ HRegionFileSystem rfs = HRegionFileSystem.open(getConf(), hri, true);
+ printIfVerbose("region dir -> " + rfs);
+ for (String family: rfs.getFamilies()) {
+ Collection<StoreFileInfo> storeFiles = rfs.getStoreFiles(family);
+ printIfVerbose("Number of region files found -> " + storeFiles.size());
int i = 1;
- for (Path p : regionFiles) {
- if (verbose)
- System.out.println("Found file[" + i++ + "] -> " + p);
+ for (StoreFileInfo storeFile : storeFiles) {
+ printIfVerbose("Found file[%d] -> %s", i++, storeFile.getPath());
+ files.add(storeFile.getPath());
}
}
- files.addAll(regionFiles);
}
return true;
}
+ private void printIfVerbose(String format, Object... args) {
+ if (verbose) {
+ System.out.println(String.format(format, args));
+ }
+ }
+
/**
* Runs the command-line pretty-printer, and returns the desired command
* exit code (zero for success, non-zero for failure).
@@ -245,8 +244,7 @@ public class HFilePrettyPrinter extends Configured implements Tool {
}
private void processFile(Path file) throws IOException {
- if (verbose)
- System.out.println("Scanning -> " + file);
+ printIfVerbose("Scanning -> " + file);
FileSystem fs = file.getFileSystem(getConf());
if (!fs.exists(file)) {
System.err.println("ERROR, file doesnt exist: " + file);
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index 9d78c1b..86bacb5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.fs.layout.FsLayout;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.fs.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.RegionDoesNotExistException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
@@ -188,8 +189,7 @@ public class CatalogJanitor extends ScheduledChore {
HTableDescriptor htd = getTableDescriptor(mergedRegion.getTable());
HRegionFileSystem regionFs = null;
try {
- regionFs = HRegionFileSystem.openRegionFromFileSystem(
- this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
+ regionFs = HRegionFileSystem.open(this.services.getConfiguration(), mergedRegion, true);
} catch (IOException e) {
LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
}
@@ -368,25 +368,14 @@ public class CatalogJanitor extends ScheduledChore {
Path rootdir = this.services.getMasterFileSystem().getRootDir();
Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());
- HRegionFileSystem hrfs = HRegionFileSystem.create(
- this.services.getConfiguration(), fs, tabledir, daughter);
+ HRegionFileSystem hrfs = HRegionFileSystem.create(this.services.getConfiguration(), daughter);
Path daughterRegionDir = hrfs.getRegionDir();
- HRegionFileSystem regionFs = null;
-
- try {
- if (!FSUtils.isExists(fs, daughterRegionDir)) {
- return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
- }
- } catch (IOException ioe) {
- LOG.warn("Error trying to determine if daughter region exists, " +
- "assuming exists and has references", ioe);
- return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
- }
-
+ HRegionFileSystem regionFs;
try {
- regionFs = HRegionFileSystem.openRegionFromFileSystem(
- this.services.getConfiguration(), fs, tabledir, daughter, true);
+ regionFs = HRegionFileSystem.open(this.services.getConfiguration(), daughter, true);
+ } catch (RegionDoesNotExistException e) {
+ return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
} catch (IOException e) {
LOG.warn("Error trying to determine referenced files from : " + daughter.getEncodedName()
+ ", to: " + parent.getEncodedName() + " assuming has references", e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 24f3071..408199d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -51,10 +51,9 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.HFileArchiver;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.fs.layout.FsLayout;
+import org.apache.hadoop.hbase.fs.HRegionFileSystem;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.wal.DefaultWALProvider;
import org.apache.hadoop.hbase.wal.WALSplitter;
import org.apache.hadoop.hbase.util.Bytes;
@@ -504,7 +503,7 @@ public class MasterFileSystem {
return rd;
}
-
+
/**
* Checks if meta region exists
*
@@ -625,7 +624,7 @@ public class MasterFileSystem {
Path regionDir = FsLayout.getRegionDir(tableDir, region);
Path familyDir = new Path(regionDir, Bytes.toString(familyName));
-
+
if (fs.delete(familyDir, true) == false) {
if (fs.exists(familyDir)) {
throw new IOException("Could not delete family "
@@ -702,38 +701,7 @@ public class MasterFileSystem {
this.services.getTableDescriptors().add(htd);
return htd;
}
-
- // TODO: Can't get rid of this in totality because the caller of this method
- // (TestSplitTransactionOnCluster.testSSHCleanupDaughterRegionsOfABortedSplit)
- // is testing something where the FS does not agree with the meta
- // At best can just make an assertion about # of region dirs in MFS and not expose
- // what they actually are
- public List<Path> getRegionDirs(TableName tableName) throws IOException {
- FileSystem fs = getFileSystem();
- Path rootDir = FSUtils.getRootDir(conf);
- Path tableDir = FSUtils.getTableDir(rootDir, tableName);
- return FsLayout.getRegionDirPaths(fs, tableDir);
- }
-
- // Only returns region filesystems for regions in meta
- // Will ignore anything on filesystem
- public List<HRegionFileSystem> getRegionFileSystems(Configuration conf,
- Connection connection, TableName tableName) throws IOException {
-
- FileSystem fs = getFileSystem();
- Path rootDir = FSUtils.getRootDir(conf);
- Path tableDir = FSUtils.getTableDir(rootDir, tableName);
-
- List<HRegionInfo> regionInfos = MetaTableAccessor.getTableRegions(connection, tableName);
-
- List<HRegionFileSystem> results = new ArrayList<HRegionFileSystem>();
- for (HRegionInfo regionInfo : regionInfos) {
- HRegionFileSystem hrfs = HRegionFileSystem.create(conf, fs, tableDir, regionInfo);
- results.add(hrfs);
- }
- return results;
- }
-
+
/**
* The function is used in SSH to set recovery mode based on configuration after all outstanding
* log split tasks drained.
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index bd672dc..ded09bb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -46,7 +46,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.fs.HRegionFileSystem;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
@@ -646,7 +646,7 @@ public class RegionStates {
if (MetaTableAccessor.getRegion(server.getConnection(), hri.getEncodedNameAsBytes()) ==
null) {
regionOffline(hri);
- HRegionFileSystem.deleteRegionDir(server.getConfiguration(), hri);
+ HRegionFileSystem.destroy(server.getConfiguration(), hri);
}
} catch (IOException e) {
LOG.warn("Got exception while deleting " + hri + " directories from file system.", e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7743d77/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
index 9d54b00..06905b9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
@@ -54,8 +54,8 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.fs.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.regionserver.compactions.NoLimitCompactionThroughputController;
import org.apache.hadoop.hbase.mapreduce.JobUtil;