You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2018/08/28 14:45:43 UTC
hbase git commit: HBASE-20649 Validate HFiles do not have PREFIX_TREE
DataBlockEncoding; ADDEDNDUM ADD MISSING FILE
Repository: hbase
Updated Branches:
refs/heads/branch-2.1 147694bb0 -> 4340930c7
HBASE-20649 Validate HFiles do not have PREFIX_TREE DataBlockEncoding; ADDEDNDUM ADD MISSING FILE
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4340930c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4340930c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4340930c
Branch: refs/heads/branch-2.1
Commit: 4340930c716599256c823c78d614b4062ec3457c
Parents: 147694b
Author: Michael Stack <st...@apache.org>
Authored: Tue Aug 28 07:45:27 2018 -0700
Committer: Michael Stack <st...@apache.org>
Committed: Tue Aug 28 07:45:27 2018 -0700
----------------------------------------------------------------------
.../hbase/tool/HFileContentValidator.java | 126 +++++++++++++++++++
1 file changed, 126 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4340930c/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/HFileContentValidator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/HFileContentValidator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/HFileContentValidator.java
new file mode 100644
index 0000000..d60844b
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/HFileContentValidator.java
@@ -0,0 +1,126 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.tool;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.util.AbstractHBaseTool;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
+
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
+public class HFileContentValidator extends AbstractHBaseTool {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HFileContentValidator.class);
+
+ /**
+ * Check HFile contents are readable by HBase 2.
+ *
+ * @param conf used configuration
+ * @return number of HFiles corrupted HBase
+ * @throws IOException if a remote or network exception occurs
+ */
+ private boolean validateHFileContent(Configuration conf) throws IOException {
+ FileSystem fileSystem = FSUtils.getCurrentFileSystem(conf);
+
+ ExecutorService threadPool = createThreadPool(conf);
+ HFileCorruptionChecker checker;
+
+ try {
+ checker = new HFileCorruptionChecker(conf, threadPool, false);
+
+ Path rootDir = FSUtils.getRootDir(conf);
+ LOG.info("Validating HFile contents under {}", rootDir);
+
+ Collection<Path> tableDirs = FSUtils.getTableDirs(fileSystem, rootDir);
+ checker.checkTables(tableDirs);
+
+ Path archiveRootDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
+ LOG.info("Validating HFile contents under {}", archiveRootDir);
+
+ List<Path> archiveTableDirs = FSUtils.getTableDirs(fileSystem, archiveRootDir);
+ checker.checkTables(archiveTableDirs);
+ } finally {
+ threadPool.shutdown();
+
+ try {
+ threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ int checkedFiles = checker.getHFilesChecked();
+ Collection<Path> corrupted = checker.getCorrupted();
+
+ if (corrupted.isEmpty()) {
+ LOG.info("Checked {} HFiles, none of them are corrupted.", checkedFiles);
+ LOG.info("There are no incompatible HFiles.");
+
+ return true;
+ } else {
+ LOG.info("Checked {} HFiles, {} are corrupted.", checkedFiles, corrupted.size());
+
+ for (Path path : corrupted) {
+ LOG.info("Corrupted file: {}", path);
+ }
+
+ LOG.info("Change data block encodings before upgrading. "
+ + "Check https://s.apache.org/prefixtree for instructions.");
+
+ return false;
+ }
+ }
+
+ private ExecutorService createThreadPool(Configuration conf) {
+ int availableProcessors = Runtime.getRuntime().availableProcessors();
+ int numThreads = conf.getInt("hfilevalidator.numthreads", availableProcessors);
+ return Executors.newFixedThreadPool(numThreads,
+ Threads.getNamedThreadFactory("hfile-validator"));
+ }
+
+ @Override
+ protected void addOptions() {
+ }
+
+ @Override
+ protected void processOptions(CommandLine cmd) {
+ }
+
+ @Override
+ protected int doWork() throws Exception {
+ return (validateHFileContent(getConf())) ? EXIT_SUCCESS : EXIT_FAILURE;
+ }
+}