You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2017/09/26 00:39:08 UTC
[4/4] hbase git commit: Amend HBASE-18786 FileNotFoundException
should not be silently handled for primary region replicas
Amend HBASE-18786 FileNotFoundException should not be silently handled for primary region replicas
Remove now invalid unit test TestCorruptedRegionStoreFile
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e6e7ae4a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e6e7ae4a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e6e7ae4a
Branch: refs/heads/branch-1.4
Commit: e6e7ae4aaeab277961ba0f468ef6a5088c18e305
Parents: 0c93008
Author: Andrew Purtell <ap...@apache.org>
Authored: Mon Sep 25 17:36:04 2017 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Sep 25 17:37:01 2017 -0700
----------------------------------------------------------------------
.../TestCorruptedRegionStoreFile.java | 263 -------------------
1 file changed, 263 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e6e7ae4a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
deleted file mode 100644
index daa03b7..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCorruptedRegionStoreFile.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.regionserver;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.io.HFileLink;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSVisitor;
-import org.apache.hadoop.hbase.util.TestTableName;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-@Category(LargeTests.class)
-public class TestCorruptedRegionStoreFile {
- private static final Log LOG = LogFactory.getLog(TestCorruptedRegionStoreFile.class);
-
- private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
- private static final String FAMILY_NAME_STR = "f";
- private static final byte[] FAMILY_NAME = Bytes.toBytes(FAMILY_NAME_STR);
-
- private static final int NUM_FILES = 10;
- private static final int ROW_PER_FILE = 2000;
- private static final int NUM_ROWS = NUM_FILES * ROW_PER_FILE;
-
- @Rule public TestTableName TEST_TABLE = new TestTableName();
-
- private final ArrayList<Path> storeFiles = new ArrayList<Path>();
- private Path tableDir;
- private int rowCount;
-
- private static void setupConf(Configuration conf) {
- // Disable compaction so the store file count stays constant
- conf.setLong("hbase.hstore.compactionThreshold", NUM_FILES + 1);
- conf.setLong("hbase.hstore.blockingStoreFiles", NUM_FILES * 2);
- }
-
- private void setupTable(final TableName tableName) throws IOException {
- // load the table
- Table table = UTIL.createTable(tableName, FAMILY_NAME);
- try {
- rowCount = 0;
- byte[] value = new byte[1024];
- byte[] q = Bytes.toBytes("q");
- while (rowCount < NUM_ROWS) {
- Put put = new Put(Bytes.toBytes(String.format("%010d", rowCount)));
- put.setDurability(Durability.SKIP_WAL);
- put.add(FAMILY_NAME, q, value);
- table.put(put);
-
- if ((rowCount++ % ROW_PER_FILE) == 0) {
- // flush it
- ((HTable)table).flushCommits();
- UTIL.getHBaseAdmin().flush(tableName);
- }
- }
- } finally {
- UTIL.getHBaseAdmin().flush(tableName);
- table.close();
- }
-
- assertEquals(NUM_ROWS, rowCount);
-
- // get the store file paths
- storeFiles.clear();
- tableDir = FSUtils.getTableDir(getRootDir(), tableName);
- FSVisitor.visitTableStoreFiles(getFileSystem(), tableDir, new FSVisitor.StoreFileVisitor() {
- @Override
- public void storeFile(final String region, final String family, final String hfile)
- throws IOException {
- HFileLink link = HFileLink.build(UTIL.getConfiguration(), tableName, region, family, hfile);
- storeFiles.add(link.getOriginPath());
- }
- });
- assertTrue("Expected at least " + NUM_FILES + " store files", storeFiles.size() >= NUM_FILES);
- LOG.info("Store files: " + storeFiles);
- }
-
- @Before
- public void setup() throws Exception {
- setupConf(UTIL.getConfiguration());
- UTIL.startMiniCluster(2, 3);
-
- setupTable(TEST_TABLE.getTableName());
- }
-
- @After
- public void tearDown() throws Exception {
- try {
- UTIL.shutdownMiniCluster();
- } catch (Exception e) {
- LOG.warn("failure shutting down cluster", e);
- }
- }
-
- private void removeStoreFile(FileSystem fs, Path tmpStoreFilePath) throws Exception {
- try (FSDataInputStream input = fs.open(storeFiles.get(0))) {
- fs.copyToLocalFile(true, storeFiles.get(0), tmpStoreFilePath);
- LOG.info("Move file to local");
- evictHFileCache(storeFiles.get(0));
- // make sure that all the replicas have been deleted on DNs.
- for (;;) {
- try {
- input.read(0, new byte[1], 0, 1);
- } catch (FileNotFoundException e) {
- break;
- }
- Thread.sleep(1000);
- }
- }
- }
-
- @Test(timeout=180000)
- public void testLosingFileDuringScan() throws Exception {
- assertEquals(rowCount, fullScanAndCount(TEST_TABLE.getTableName()));
-
- final FileSystem fs = getFileSystem();
- final Path tmpStoreFilePath = new Path(UTIL.getDataTestDir(), "corruptedHFile");
-
- // try to query with the missing file
- int count = fullScanAndCount(TEST_TABLE.getTableName(), new ScanInjector() {
- private boolean hasFile = true;
-
- @Override
- public void beforeScanNext(Table table) throws Exception {
- // move the path away (now the region is corrupted)
- if (hasFile) {
- removeStoreFile(fs, tmpStoreFilePath);
- hasFile = false;
- }
- }
- });
- assertTrue("expected one file lost: rowCount=" + count + " lostRows=" + (NUM_ROWS - count),
- count >= (NUM_ROWS - ROW_PER_FILE));
- }
-
- @Test(timeout=180000)
- public void testLosingFileAfterScannerInit() throws Exception {
- assertEquals(rowCount, fullScanAndCount(TEST_TABLE.getTableName()));
-
- final FileSystem fs = getFileSystem();
- final Path tmpStoreFilePath = new Path(UTIL.getDataTestDir(), "corruptedHFile");
-
- // try to query with the missing file
- int count = fullScanAndCount(TEST_TABLE.getTableName(), new ScanInjector() {
- private boolean hasFile = true;
-
- @Override
- public void beforeScan(Table table, Scan scan) throws Exception {
- // move the path away (now the region is corrupted)
- if (hasFile) {
- removeStoreFile(fs, tmpStoreFilePath);
- hasFile = false;
- }
- }
- });
- assertTrue("expected one file lost: rowCount=" + count + " lostRows=" + (NUM_ROWS - count),
- count >= (NUM_ROWS - ROW_PER_FILE));
- }
-
- // ==========================================================================
- // Helpers
- // ==========================================================================
- private FileSystem getFileSystem() {
- return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
- }
-
- private Path getRootDir() {
- return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
- }
-
- private void evictHFileCache(final Path hfile) throws Exception {
- for (RegionServerThread rst: UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
- HRegionServer rs = rst.getRegionServer();
- rs.getCacheConfig().getBlockCache().evictBlocksByHfileName(hfile.getName());
- }
- }
-
- private int fullScanAndCount(final TableName tableName) throws Exception {
- return fullScanAndCount(tableName, new ScanInjector());
- }
-
- private int fullScanAndCount(final TableName tableName, final ScanInjector injector)
- throws Exception {
- Table table = UTIL.getConnection().getTable(tableName);
- int count = 0;
- try {
- Scan scan = new Scan();
- scan.setCaching(1);
- scan.setCacheBlocks(false);
- injector.beforeScan(table, scan);
- ResultScanner scanner = table.getScanner(scan);
- try {
- while (true) {
- injector.beforeScanNext(table);
- Result result = scanner.next();
- injector.afterScanNext(table, result);
- if (result == null) break;
- if ((count++ % (ROW_PER_FILE / 2)) == 0) {
- LOG.debug("scan next " + count);
- }
- }
- } finally {
- scanner.close();
- injector.afterScan(table);
- }
- } finally {
- table.close();
- }
- return count;
- }
-
- private class ScanInjector {
- protected void beforeScan(Table table, Scan scan) throws Exception {}
- protected void beforeScanNext(Table table) throws Exception {}
- protected void afterScanNext(Table table, Result result) throws Exception {}
- protected void afterScan(Table table) throws Exception {}
- }
-}