You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2019/09/03 18:08:09 UTC
[hbase] branch branch-2.1 updated: HBASE-22961 Deprecate hbck1 in
core
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new a904c92 HBASE-22961 Deprecate hbck1 in core
a904c92 is described below
commit a904c9231609c420d72420edcef7cc20f2e61376
Author: stack <st...@apache.org>
AuthorDate: Sat Aug 31 09:24:33 2019 -0700
HBASE-22961 Deprecate hbck1 in core
Adds deprecations on HBaseFsck and on supporting classes such as
the reporting Interface. Provides alternatives in FSUtils for
progress reporting and deprecates methods that use hbck1 facility.
Signed-off-by: Peter Somogyi <ps...@apache.org>
---
.../java/org/apache/hadoop/hbase/util/FSUtils.java | 120 ++++++++++++++++++---
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 6 +-
.../hadoop/hbase/util/HbckErrorReporter.java | 11 +-
3 files changed, 115 insertions(+), 22 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
index 913c892..770575e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
@@ -1166,6 +1166,17 @@ public abstract class FSUtils extends CommonFSUtils {
}
/**
+ * Called every so-often by storefile map builder getTableStoreFilePathMap to
+ * report progress.
+ */
+ interface ProgressReporter {
+ /**
+ * @param status File or directory we are about to process.
+ */
+ void progress(FileStatus status);
+ }
+
+ /**
* Runs through the HBase rootdir/tablename and creates a reverse lookup map for
* table StoreFile names to the full Path.
* <br>
@@ -1184,7 +1195,8 @@ public abstract class FSUtils extends CommonFSUtils {
public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
final FileSystem fs, final Path hbaseRootDir, TableName tableName)
throws IOException, InterruptedException {
- return getTableStoreFilePathMap(map, fs, hbaseRootDir, tableName, null, null, null);
+ return getTableStoreFilePathMap(map, fs, hbaseRootDir, tableName, null, null,
+ (ProgressReporter)null);
}
/**
@@ -1204,15 +1216,55 @@ public abstract class FSUtils extends CommonFSUtils {
* @param tableName name of the table to scan.
* @param sfFilter optional path filter to apply to store files
* @param executor optional executor service to parallelize this operation
- * @param errors ErrorReporter instance or null
+ * @param progressReporter Instance or null; gets called every time we move to new region of
+ * family dir and for each store file.
* @return Map keyed by StoreFile name with a value of the full Path.
* @throws IOException When scanning the directory fails.
- * @throws InterruptedException
+ * @deprecated Since 2.3.0. For removal in hbase4. Use ProgressReporter override instead.
*/
+ @Deprecated
public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> resultMap,
final FileSystem fs, final Path hbaseRootDir, TableName tableName, final PathFilter sfFilter,
- ExecutorService executor, final HbckErrorReporter errors)
+ ExecutorService executor, final HbckErrorReporter progressReporter)
throws IOException, InterruptedException {
+ return getTableStoreFilePathMap(resultMap, fs, hbaseRootDir, tableName, sfFilter, executor,
+ new ProgressReporter() {
+ @Override
+ public void progress(FileStatus status) {
+ // status is not used in this implementation.
+ progressReporter.progress();
+ }
+ });
+ }
+
+ /*
+ * Runs through the HBase rootdir/tablename and creates a reverse lookup map for
+ * table StoreFile names to the full Path. Note that because this method can be called
+ * on a 'live' HBase system that we will skip files that no longer exist by the time
+ * we traverse them and similarly the user of the result needs to consider that some
+ * entries in this map may not exist by the time this call completes.
+ * <br>
+ * Example...<br>
+ * Key = 3944417774205889744 <br>
+ * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
+ *
+ * @param resultMap map to add values. If null, this method will create and populate one
+ * to return
+ * @param fs The file system to use.
+ * @param hbaseRootDir The root directory to scan.
+ * @param tableName name of the table to scan.
+ * @param sfFilter optional path filter to apply to store files
+ * @param executor optional executor service to parallelize this operation
+ * @param progressReporter Instance or null; gets called every time we move to new region of
+ * family dir and for each store file.
+ * @return Map keyed by StoreFile name with a value of the full Path.
+ * @throws IOException When scanning the directory fails.
+ * @throws InterruptedException
+ */
+ public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> resultMap,
+ final FileSystem fs, final Path hbaseRootDir, TableName tableName, final PathFilter sfFilter,
+ ExecutorService executor, final ProgressReporter progressReporter)
+ throws IOException, InterruptedException {
final Map<String, Path> finalResultMap =
resultMap == null ? new ConcurrentHashMap<>(128, 0.75f, 32) : resultMap;
@@ -1233,8 +1285,8 @@ public abstract class FSUtils extends CommonFSUtils {
final List<Future<?>> futures = new ArrayList<>(regionDirs.size());
for (FileStatus regionDir : regionDirs) {
- if (null != errors) {
- errors.progress();
+ if (null != progressReporter) {
+ progressReporter.progress(regionDir);
}
final Path dd = regionDir.getPath();
@@ -1257,8 +1309,8 @@ public abstract class FSUtils extends CommonFSUtils {
return;
}
for (FileStatus familyDir : familyDirs) {
- if (null != errors) {
- errors.progress();
+ if (null != progressReporter) {
+ progressReporter.progress(familyDir);
}
Path family = familyDir.getPath();
if (family.getName().equals(HConstants.RECOVERED_EDITS_DIR)) {
@@ -1268,8 +1320,8 @@ public abstract class FSUtils extends CommonFSUtils {
// put in map
FileStatus[] familyStatus = fs.listStatus(family);
for (FileStatus sfStatus : familyStatus) {
- if (null != errors) {
- errors.progress();
+ if (null != progressReporter) {
+ progressReporter.progress(sfStatus);
}
Path sf = sfStatus.getPath();
if (sfFilter == null || sfFilter.accept(sf)) {
@@ -1348,12 +1400,45 @@ public abstract class FSUtils extends CommonFSUtils {
* @param hbaseRootDir The root directory to scan.
* @return Map keyed by StoreFile name with a value of the full Path.
* @throws IOException When scanning the directory fails.
- * @throws InterruptedException
*/
- public static Map<String, Path> getTableStoreFilePathMap(
- final FileSystem fs, final Path hbaseRootDir)
+ public static Map<String, Path> getTableStoreFilePathMap(final FileSystem fs,
+ final Path hbaseRootDir)
throws IOException, InterruptedException {
- return getTableStoreFilePathMap(fs, hbaseRootDir, null, null, null);
+ return getTableStoreFilePathMap(fs, hbaseRootDir, null, null, (ProgressReporter)null);
+ }
+
+ /**
+ * Runs through the HBase rootdir and creates a reverse lookup map for
+ * table StoreFile names to the full Path.
+ * <br>
+ * Example...<br>
+ * Key = 3944417774205889744 <br>
+ * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
+ *
+ * @param fs The file system to use.
+ * @param hbaseRootDir The root directory to scan.
+ * @param sfFilter optional path filter to apply to store files
+ * @param executor optional executor service to parallelize this operation
+ * @param progressReporter Instance or null; gets called every time we move to new region of
+ * family dir and for each store file.
+ * @return Map keyed by StoreFile name with a value of the full Path.
+ * @throws IOException When scanning the directory fails.
+ * @deprecated Since 2.3.0. Will be removed in hbase4. Used {@link
+ * #getTableStoreFilePathMap(FileSystem, Path, PathFilter, ExecutorService, ProgressReporter)}
+ */
+ @Deprecated
+ public static Map<String, Path> getTableStoreFilePathMap(final FileSystem fs,
+ final Path hbaseRootDir, PathFilter sfFilter, ExecutorService executor,
+ HbckErrorReporter progressReporter)
+ throws IOException, InterruptedException {
+ return getTableStoreFilePathMap(fs, hbaseRootDir, sfFilter, executor,
+ new ProgressReporter() {
+ @Override
+ public void progress(FileStatus status) {
+ // status is not used in this implementation.
+ progressReporter.progress();
+ }
+ });
}
/**
@@ -1368,14 +1453,15 @@ public abstract class FSUtils extends CommonFSUtils {
* @param hbaseRootDir The root directory to scan.
* @param sfFilter optional path filter to apply to store files
* @param executor optional executor service to parallelize this operation
- * @param errors ErrorReporter instance or null
+ * @param progressReporter Instance or null; gets called every time we move to new region of
+ * family dir and for each store file.
* @return Map keyed by StoreFile name with a value of the full Path.
* @throws IOException When scanning the directory fails.
* @throws InterruptedException
*/
public static Map<String, Path> getTableStoreFilePathMap(
final FileSystem fs, final Path hbaseRootDir, PathFilter sfFilter,
- ExecutorService executor, HbckErrorReporter errors)
+ ExecutorService executor, ProgressReporter progressReporter)
throws IOException, InterruptedException {
ConcurrentHashMap<String, Path> map = new ConcurrentHashMap<>(1024, 0.75f, 32);
@@ -1385,7 +1471,7 @@ public abstract class FSUtils extends CommonFSUtils {
// only include the directory paths to tables
for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) {
getTableStoreFilePathMap(map, fs, hbaseRootDir,
- FSUtils.getTableName(tableDir), sfFilter, executor, errors);
+ FSUtils.getTableName(tableDir), sfFilter, executor, progressReporter);
}
return map;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 9be24f9..7c13f54 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -154,7 +154,9 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminServic
* HBaseFsck (hbck) is a tool for checking and repairing region consistency and
* table integrity problems in a corrupted HBase. This tool was written for hbase-1.x. It does not
* work with hbase-2.x; it can read state but is not allowed to change state; i.e. effect 'repair'.
- * See hbck2 (HBASE-19121) for a hbck tool for hbase2.
+ * Even though it can 'read' state, given how so much has changed in how hbase1 and hbase2 operate,
+ * it will often misread. See hbck2 (HBASE-19121) for a hbck tool for hbase2. This class is
+ * deprecated.
*
* <p>
* Region consistency checks verify that hbase:meta, region deployment on region
@@ -197,7 +199,9 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminServic
* If hbck is run from the command line, there are a handful of arguments that
* can be used to limit the kinds of repairs hbck will do. See the code in
* {@link #printUsageAndExit()} for more details.
+ * @deprecated For removal in hbase-4.0.0. Use HBCK2 instead.
*/
+@Deprecated
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
@InterfaceStability.Evolving
public class HBaseFsck extends Configured implements Closeable {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HbckErrorReporter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HbckErrorReporter.java
index 500b690..52012df 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HbckErrorReporter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HbckErrorReporter.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,12 +20,15 @@ package org.apache.hadoop.hbase.util;
import java.util.ArrayList;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.yetus.audience.InterfaceStability;
+/**
+ * Used by {@link HBaseFsck} reporting system.
+ * @deprecated Since 2.3.0. To be removed in hbase4. Use HBCK2 instead. Remove when
+ * {@link HBaseFsck} is removed.
+ */
+@Deprecated
@InterfaceAudience.Private
-@InterfaceStability.Evolving
public interface HbckErrorReporter {
-
enum ERROR_CODE {
UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META,