You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ha...@apache.org on 2021/11/05 10:16:41 UTC
[hbase] branch master updated: HBASE-25322 Redundant Reference file
in bottom region of split (#3814)
This is an automated email from the ASF dual-hosted git repository.
haxiaolin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new a6c09a8 HBASE-25322 Redundant Reference file in bottom region of split (#3814)
a6c09a8 is described below
commit a6c09a84a79c36712a8dc8a2f13f7ae4560f58a6
Author: Xiaolin Ha <li...@gmail.com>
AuthorDate: Fri Nov 5 18:14:20 2021 +0800
HBASE-25322 Redundant Reference file in bottom region of split (#3814)
Signed-off-by: Duo Zhang <zh...@apache.org>
---
.../hbase/regionserver/HRegionFileSystem.java | 3 +-
.../master/assignment/AssignmentTestingUtil.java | 10 ++++-
.../hbase/master/assignment/TestRegionSplit.java | 51 ++++++++++++++++++++++
3 files changed, 61 insertions(+), 3 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index 5b26f9a..69d9c8f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -667,9 +667,9 @@ public class HRegionFileSystem {
// If it is outside the range, return directly.
f.initReader();
try {
+ Cell splitKey = PrivateCellUtil.createFirstOnRow(splitRow);
if (top) {
//check if larger than last key.
- Cell splitKey = PrivateCellUtil.createFirstOnRow(splitRow);
Optional<Cell> lastKey = f.getLastKey();
// If lastKey is null means storefile is empty.
if (!lastKey.isPresent()) {
@@ -680,7 +680,6 @@ public class HRegionFileSystem {
}
} else {
//check if smaller than first key
- Cell splitKey = PrivateCellUtil.createLastOnRow(splitRow);
Optional<Cell> firstKey = f.getFirstKey();
// If firstKey is null means storefile is empty.
if (!firstKey.isPresent()) {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/AssignmentTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/AssignmentTestingUtil.java
index cd13905..4d7a483 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/AssignmentTestingUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/AssignmentTestingUtil.java
@@ -159,6 +159,11 @@ public final class AssignmentTestingUtil {
public static void insertData(final HBaseTestingUtil UTIL, final TableName tableName,
int rowCount, int startRowNum, String... cfs) throws IOException {
+ insertData(UTIL, tableName, rowCount, startRowNum, false, cfs);
+ }
+
+ public static void insertData(final HBaseTestingUtil UTIL, final TableName tableName,
+ int rowCount, int startRowNum, boolean flushOnce, String... cfs) throws IOException {
Table t = UTIL.getConnection().getTable(tableName);
Put p;
for (int i = 0; i < rowCount / 2; i++) {
@@ -172,9 +177,12 @@ public final class AssignmentTestingUtil {
p.addColumn(Bytes.toBytes(cf), Bytes.toBytes("q"), Bytes.toBytes(i));
}
t.put(p);
- if (i % 5 == 0) {
+ if (i % 5 == 0 && !flushOnce) {
UTIL.getAdmin().flush(tableName);
}
}
+ if (flushOnce) {
+ UTIL.getAdmin().flush(tableName);
+ }
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java
index 62f966a..105f72f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java
@@ -19,7 +19,9 @@ package org.apache.hadoop.hbase.master.assignment;
import static org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil.insertData;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
@@ -39,6 +41,7 @@ import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
+
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -157,6 +160,54 @@ public class TestRegionSplit {
regionInfoMap.get(tableRegions.get(1).getRegionInfo()));
}
+ @Test
+ public void testSplitStoreFiles() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName,
+ null, columnFamilyName);
+ // flush the memstore
+ insertData(UTIL, tableName, rowCount, startRowNum, true, columnFamilyName);
+
+ // assert the hfile count of the table
+ int storeFilesCountSum = 0;
+ for(HRegion region : UTIL.getHBaseCluster().getRegions(tableName)){
+ storeFilesCountSum += region.getStore(Bytes.toBytes(columnFamilyName)).getStorefiles().size();
+ }
+ assertEquals(1, storeFilesCountSum);
+
+ // split at the start row
+ byte[] splitKey = Bytes.toBytes("" + startRowNum);
+
+ assertNotNull("Not able to find a splittable region", regions);
+ assertEquals("Not able to find a splittable region", 1, regions.length);
+
+ // Split region of the table
+ long procId = procExec.submitProcedure(
+ new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
+ // Wait the completion
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ assertEquals("Not able to split table",
+ 2, UTIL.getHBaseCluster().getRegions(tableName).size());
+
+ // assert sum of the hfiles of all regions
+ int childStoreFilesSum = 0;
+ for(HRegion region : UTIL.getHBaseCluster().getRegions(tableName)){
+ childStoreFilesSum += region.getStore(Bytes.toBytes(columnFamilyName)).getStorefiles().size();
+ }
+ assertEquals(1, childStoreFilesSum);
+
+ List<HRegion> tableRegions = UTIL.getHBaseCluster().getRegions(tableName);
+ assertEquals("Table region not correct.", 2, tableRegions.size());
+ Map<RegionInfo, ServerName> regionInfoMap = UTIL.getHBaseCluster().getMaster()
+ .getAssignmentManager().getRegionStates().getRegionAssignments();
+ assertEquals(regionInfoMap.get(tableRegions.get(0).getRegionInfo()),
+ regionInfoMap.get(tableRegions.get(1).getRegionInfo()));
+ }
+
private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
}