You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2020/02/15 20:29:36 UTC
[hbase] 02/18: HBASE-22749: Distributed MOB compactions. (fixed UTs
and code clean up)
This is an automated email from the ASF dual-hosted git repository.
busbey pushed a commit to branch HBASE-22749
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 584d4d35a951694d732c6b84d088e92f6d06b497
Author: Vladimir Rodionov <vr...@apache.org>
AuthorDate: Thu Dec 12 16:55:57 2019 -0800
HBASE-22749: Distributed MOB compactions. (fixed UTs and code clean up)
---
.../hadoop/hbase/IntegrationTestMobCompaction.java | 7 +-
.../hadoop/hbase/master/MasterRpcServices.java | 38 ++--
.../hadoop/hbase/master/MobFileCleanerChore.java | 55 +++---
.../hbase/master/MobFileCompactionChore.java | 46 ++---
.../hadoop/hbase/mob/DefaultMobStoreCompactor.java | 108 +++++-----
.../java/org/apache/hadoop/hbase/mob/MobUtils.java | 11 +-
.../hadoop/hbase/regionserver/StoreFileInfo.java | 219 ++++++++++-----------
.../hbase/client/TestAsyncRegionAdminApi.java | 4 +-
.../hadoop/hbase/mob/FaultyMobStoreCompactor.java | 4 +-
.../org/apache/hadoop/hbase/mob/MobStressTool.java | 4 +-
.../regionserver/TestBulkLoadReplication.java | 69 +------
.../hbase/regionserver/TestMobStoreCompaction.java | 29 ++-
.../regionserver/TestRegionServerMetrics.java | 21 +-
13 files changed, 290 insertions(+), 325 deletions(-)
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java
index 3373ce9..47400a4 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java
@@ -44,8 +44,7 @@ import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.testclassification.IntegrationTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.hbase.thirdparty.com.google.common.base.MoreObjects;
-import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -53,6 +52,8 @@ import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.base.MoreObjects;
+import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
/**
* An integration test to detect regressions in HBASE-22749. Test creates
@@ -189,7 +190,7 @@ public class IntegrationTestMobCompaction extends IntegrationTestBase {
"Total number of data rows to load. Default: '" + DEFAULT_ROWS_COUNT + "'");
addOptWithArg(FAILURE_PROB_KEY,
"Probability of a failure of a region MOB compaction request. Default: '"
- + DEFAULT_FAILURE_PROB + "'");
+ + DEFAULT_FAILURE_PROB + "'");
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 6d97f5d..c1541fbf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -104,6 +104,27 @@ import org.apache.hadoop.hbase.security.access.PermissionStorage;
import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil;
import org.apache.hadoop.hbase.security.access.UserPermission;
import org.apache.hadoop.hbase.security.visibility.VisibilityController;
+
+import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
+
+
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
+import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
+import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AccessControlProtos;
@@ -328,21 +349,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.Trans
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.UpdateReplicationPeerConfigRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.UpdateReplicationPeerConfigResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
-import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
-import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
-import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+
/**
* Implements the master RPC services.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCleanerChore.java
index 0f1d0e1..ca3df66 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCleanerChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCleanerChore.java
@@ -51,11 +51,12 @@ import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
/**
* The Class ExpiredMobFileCleanerChore for running cleaner regularly to remove the expired
* and obsolete (files which have no active references to) mob files.
@@ -69,10 +70,12 @@ public class MobFileCleanerChore extends ScheduledChore {
private long minAgeToArchive;
public MobFileCleanerChore(HMaster master) {
- super(master.getServerName() + "-ExpiredMobFileCleanerChore", master, master.getConfiguration()
- .getInt(MobConstants.MOB_CLEANER_PERIOD, MobConstants.DEFAULT_MOB_CLEANER_PERIOD), master
- .getConfiguration().getInt(MobConstants.MOB_CLEANER_PERIOD,
- MobConstants.DEFAULT_MOB_CLEANER_PERIOD), TimeUnit.SECONDS);
+ super(master.getServerName() + "-ExpiredMobFileCleanerChore", master,
+ master.getConfiguration().getInt(MobConstants.MOB_CLEANER_PERIOD,
+ MobConstants.DEFAULT_MOB_CLEANER_PERIOD),
+ master.getConfiguration().getInt(MobConstants.MOB_CLEANER_PERIOD,
+ MobConstants.DEFAULT_MOB_CLEANER_PERIOD),
+ TimeUnit.SECONDS);
this.master = master;
cleaner = new ExpiredMobFileCleaner();
cleaner.setConf(master.getConfiguration());
@@ -104,8 +107,8 @@ public class MobFileCleanerChore extends ScheduledChore {
}
@Override
- @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="REC_CATCH_EXCEPTION",
- justification="Intentional")
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "REC_CATCH_EXCEPTION",
+ justification = "Intentional")
protected void chore() {
try {
@@ -127,23 +130,23 @@ public class MobFileCleanerChore extends ScheduledChore {
LOG.error("Fail to clean the expired mob files", e);
}
}
+
/**
* Performs housekeeping file cleaning (called by MOB Cleaner chore)
* @param conf configuration
* @param table table name
- * @throws IOException
+ * @throws IOException exception
*/
- public void cleanupObsoleteMobFiles(Configuration conf, TableName table)
- throws IOException {
+ public void cleanupObsoleteMobFiles(Configuration conf, TableName table) throws IOException {
- this.minAgeToArchive = conf.getLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY,
- MobConstants.DEFAULT_MIN_AGE_TO_ARCHIVE);
+ this.minAgeToArchive =
+ conf.getLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, MobConstants.DEFAULT_MIN_AGE_TO_ARCHIVE);
try (final Connection conn = ConnectionFactory.createConnection(conf);
final Admin admin = conn.getAdmin();) {
TableDescriptor htd = admin.getDescriptor(table);
List<ColumnFamilyDescriptor> list = MobUtils.getMobColumnFamilies(htd);
if (list.size() == 0) {
- LOG.info("Skipping non-MOB table [{}]", table);
+ LOG.info("Skipping non-MOB table [{}]", table);
return;
}
// We check only those MOB files, which creation time is less
@@ -191,13 +194,15 @@ public class MobFileCleanerChore extends ScheduledChore {
sf.closeStoreFile(true);
if (mobRefData == null && bulkloadMarkerData == null) {
LOG.warn("Found old store file with no MOB_FILE_REFS: {} - "
- + "can not proceed until all old files will be MOB-compacted.", pp);
+ + "can not proceed until all old files will be MOB-compacted.",
+ pp);
return;
} else if (mobRefData == null && bulkloadMarkerData != null) {
LOG.info("Skipping file without MOB references (bulkloaded file):{}", pp);
continue;
}
- if (mobRefData.length > 1) {
+ // mobRefData will never be null here, but to make FindBugs happy
+ if (mobRefData != null && mobRefData.length > 1) {
// if length = 1 means NULL, that there are no MOB references
// in this store file, but the file was created by new MOB code
String[] mobs = new String(mobRefData).split(",");
@@ -206,7 +211,7 @@ public class MobFileCleanerChore extends ScheduledChore {
}
}
} catch (FileNotFoundException e) {
- LOG.warn("Starting MOB cleaning cycle from the beginning due to error:",e);
+ LOG.warn("Starting MOB cleaning cycle from the beginning due to error:", e);
continue;
}
succeed = true;
@@ -236,9 +241,9 @@ public class MobFileCleanerChore extends ScheduledChore {
// MOB is not in a list of active references, but it can be too
// fresh, skip it in this case
long creationTime = fs.getFileStatus(p).getModificationTime();
- if ( creationTime < maxCreationTimeToArchive) {
- /* DEBUG */ LOG.info(
- " Archiving MOB file{} creation time=" + (fs.getFileStatus(p).getModificationTime()), p);
+ if (creationTime < maxCreationTimeToArchive) {
+ LOG.info("Archiving MOB file {} creation time={}", p,
+ (fs.getFileStatus(p).getModificationTime()));
toArchive.add(p);
} else {
LOG.info("Skipping fresh file: {}", p);
@@ -247,21 +252,20 @@ public class MobFileCleanerChore extends ScheduledChore {
LOG.info("Keepeing active MOB file: {}", p);
}
}
- LOG.info(" MOB Cleaner found {} files for family={}", toArchive.size() , family);
+ LOG.info(" MOB Cleaner found {} files for family={}", toArchive.size(), family);
removeMobFiles(conf, table, family.getBytes(), toArchive);
LOG.info(" MOB Cleaner archived {} files", toArchive.size());
}
}
}
-
/**
* Archives the mob files.
* @param conf The current configuration.
* @param tableName The table name.
* @param family The name of the column family.
* @param storeFiles The files to be archived.
- * @throws IOException
+ * @throws IOException exception
*/
public void removeMobFiles(Configuration conf, TableName tableName, byte[] family,
List<Path> storeFiles) throws IOException {
@@ -273,8 +277,8 @@ public class MobFileCleanerChore extends ScheduledChore {
}
Path mobTableDir = FSUtils.getTableDir(MobUtils.getMobHome(conf), tableName);
FileSystem fs = storeFiles.get(0).getFileSystem(conf);
- Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, MobUtils.getMobRegionInfo(tableName),
- mobTableDir, family);
+ Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf,
+ MobUtils.getMobRegionInfo(tableName), mobTableDir, family);
for (Path p : storeFiles) {
Path archiveFilePath = new Path(storeArchiveDir, p.getName());
@@ -283,7 +287,8 @@ public class MobFileCleanerChore extends ScheduledChore {
continue;
}
LOG.info("MOB Cleaner is archiving: {}", p);
- HFileArchiver.archiveStoreFile(conf, fs, MobUtils.getMobRegionInfo(tableName), mobTableDir, family, p);
+ HFileArchiver.archiveStoreFile(conf, fs, MobUtils.getMobRegionInfo(tableName), mobTableDir,
+ family, p);
}
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCompactionChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCompactionChore.java
index 402718e..251abc8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCompactionChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCompactionChore.java
@@ -38,11 +38,13 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
+
/**
* Periodic MOB compaction chore.
* It runs MOB compaction on region servers in parallel, thus
@@ -61,11 +63,12 @@ public class MobFileCompactionChore extends ScheduledChore {
private int regionBatchSize = 0;// not set - compact all
public MobFileCompactionChore(HMaster master) {
- super(master.getServerName() + "-MobFileCompactionChore", master, master.getConfiguration()
- .getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,
- MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD), master
- .getConfiguration().getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,
- MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD), TimeUnit.SECONDS);
+ super(master.getServerName() + "-MobFileCompactionChore", master,
+ master.getConfiguration().getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,
+ MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD),
+ master.getConfiguration().getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,
+ MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD),
+ TimeUnit.SECONDS);
this.master = master;
this.conf = master.getConfiguration();
this.regionBatchSize =
@@ -86,7 +89,7 @@ public class MobFileCompactionChore extends ScheduledChore {
boolean reported = false;
try (Connection conn = ConnectionFactory.createConnection(conf);
- Admin admin = conn.getAdmin(); ) {
+ Admin admin = conn.getAdmin();) {
TableDescriptors htds = master.getTableDescriptors();
Map<String, TableDescriptor> map = htds.getAll();
@@ -120,13 +123,13 @@ public class MobFileCompactionChore extends ScheduledChore {
hcd.getNameAsString());
}
} catch (IOException e) {
- LOG.error("Failed to compact table="+ htd.getTableName() +" cf="+ hcd.getNameAsString(),
- e);
+ LOG.error(
+ "Failed to compact table=" + htd.getTableName() + " cf=" + hcd.getNameAsString(), e);
} catch (InterruptedException ee) {
Thread.currentThread().interrupt();
master.reportMobCompactionEnd(htd.getTableName());
- LOG.warn("Failed to compact table="+ htd.getTableName() +" cf="+ hcd.getNameAsString(),
- ee);
+ LOG.warn(
+ "Failed to compact table=" + htd.getTableName() + " cf=" + hcd.getNameAsString(), ee);
// Quit the chore
return;
}
@@ -156,25 +159,25 @@ public class MobFileCompactionChore extends ScheduledChore {
Collections.shuffle(regions);
// Create first batch
List<RegionInfo> toCompact = new ArrayList<RegionInfo>();
- for (int i=0; i < this.regionBatchSize; i++) {
+ for (int i = 0; i < this.regionBatchSize; i++) {
toCompact.add(regions.remove(0));
}
// Start compaction now
- for(RegionInfo ri: toCompact) {
+ for (RegionInfo ri : toCompact) {
startCompaction(admin, htd.getTableName(), ri, hcd.getName());
}
List<RegionInfo> compacted = new ArrayList<RegionInfo>();
int totalCompacted = 0;
- while(!toCompact.isEmpty()) {
+ while (!toCompact.isEmpty()) {
// Check status of active compactions
- for (RegionInfo ri: toCompact) {
+ for (RegionInfo ri : toCompact) {
try {
if (admin.getCompactionStateForRegion(ri.getRegionName()) == CompactionState.NONE) {
totalCompacted++;
LOG.info("Finished major compaction: table={} region={}, compacted regions={}",
- htd.getTableName(),ri.getRegionNameAsString(), totalCompacted);
+ htd.getTableName(), ri.getRegionNameAsString(), totalCompacted);
compacted.add(ri);
}
} catch (IOException e) {
@@ -182,11 +185,11 @@ public class MobFileCompactionChore extends ScheduledChore {
}
}
// Update batch: remove compacted regions and add new ones
- for (RegionInfo ri: compacted) {
+ for (RegionInfo ri : compacted) {
toCompact.remove(ri);
if (regions.size() > 0) {
RegionInfo region = regions.remove(0);
- startCompaction(admin, htd.getTableName(),region, hcd.getName());
+ startCompaction(admin, htd.getTableName(), region, hcd.getName());
toCompact.add(region);
}
}
@@ -199,17 +202,16 @@ public class MobFileCompactionChore extends ScheduledChore {
}
- private void startCompaction(Admin admin, TableName table, RegionInfo region, byte[] cf)
+ private void startCompaction(Admin admin, TableName table, RegionInfo region, byte[] cf)
throws IOException, InterruptedException {
- LOG.info("Started major compaction: table={} region={}", table,
- region.getRegionNameAsString());
+ LOG.info("Started major compaction: table={} region={}", table, region.getRegionNameAsString());
admin.majorCompactRegion(region.getRegionName());
// Wait until it really starts
// but with finite timeout
long waitTime = 300000; // 5 min
long startTime = EnvironmentEdgeManager.currentTime();
- while(admin.getCompactionStateForRegion(region.getRegionName()) == CompactionState.NONE) {
+ while (admin.getCompactionStateForRegion(region.getRegionName()) == CompactionState.NONE) {
// Is 1 second too aggressive?
Thread.sleep(1000);
if (EnvironmentEdgeManager.currentTime() - startTime > waitTime) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
index d692723..aa7caa9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
@@ -57,11 +57,12 @@ import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+
/**
* Compact passed set of files in the mob-enabled column family.
*/
@@ -73,11 +74,10 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
protected HMobStore mobStore;
/*
- * MOB file reference set thread local variable. It contains set of
- * a MOB file names, which newly compacted store file has references to.
- * This variable is populated during compaction and the content of it is
- * written into meta section of a newly created store file at the final step
- * of compaction process.
+ * MOB file reference set thread local variable. It contains set of a MOB file names, which newly
+ * compacted store file has references to. This variable is populated during compaction and the
+ * content of it is written into meta section of a newly created store file at the final step of
+ * compaction process.
*/
static ThreadLocal<Set<String>> mobRefSet = new ThreadLocal<Set<String>>() {
@@ -99,8 +99,7 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
};
/*
- * Map : MOB file name - file length
- * Can be expensive for large amount of MOB files?
+ * Map : MOB file name - file length Can be expensive for large amount of MOB files?
*/
static ThreadLocal<HashMap<String, Long>> mobLengthMap =
new ThreadLocal<HashMap<String, Long>>() {
@@ -150,7 +149,6 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
mobSizeThreshold = store.getColumnFamilyDescriptor().getMobThreshold();
}
-
@Override
public List<Path> compact(CompactionRequestImpl request,
ThroughputController throughputController, User user) throws IOException {
@@ -166,8 +164,8 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
if (conf.get(MobConstants.MOB_COMPACTION_TYPE_KEY, MobConstants.DEFAULT_MOB_COMPACTION_TYPE)
.equals(MobConstants.IO_OPTIMIZED_MOB_COMPACTION_TYPE)) {
if (request.isMajor() && request.getPriority() == HStore.PRIORITY_USER) {
- Path mobDir = MobUtils.getMobFamilyPath(conf, store.getTableName(),
- store.getColumnFamilyName());
+ Path mobDir =
+ MobUtils.getMobFamilyPath(conf, store.getTableName(), store.getColumnFamilyName());
List<Path> mobFiles = MobUtils.getReferencedMobFiles(request.getFiles(), mobDir);
if (mobFiles.size() > 0) {
calculateMobLengthMap(mobFiles);
@@ -182,7 +180,7 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
FileSystem fs = mobFiles.get(0).getFileSystem(this.conf);
HashMap<String, Long> map = mobLengthMap.get();
map.clear();
- for (Path p: mobFiles) {
+ for (Path p : mobFiles) {
FileStatus st = fs.getFileStatus(p);
long size = st.getLen();
LOG.info("Ref MOB file={} size={}", p, size);
@@ -190,39 +188,30 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
}
}
-
/**
- * Performs compaction on a column family with the mob flag enabled.
- * This works only when MOB compaction is explicitly requested (by User), or by Master
- * There are two modes of a MOB compaction:<br>
+ * Performs compaction on a column family with the mob flag enabled. This works only when MOB
+ * compaction is explicitly requested (by User), or by Master There are two modes of a MOB
+ * compaction:<br>
* <p>
* <ul>
* <li>1. Full mode - when all MOB data for a region is compacted into a single MOB file.
* <li>2. I/O optimized mode - for use cases with no or infrequent updates/deletes of a <br>
- * MOB data. The main idea behind i/o optimized compaction is to limit maximum size of a MOB
- * file produced during compaction and to limit I/O write/read amplification.
+ * MOB data. The main idea behind i/o optimized compaction is to limit maximum size of a MOB file
+ * produced during compaction and to limit I/O write/read amplification.
* </ul>
* The basic algorithm of compaction is the following: <br>
* 1. If the Put cell has a mob reference tag, the cell's value is the path of the mob file.
* <ol>
- * <li>
- * If the value size of a cell is larger than the threshold, this cell is regarded as a mob,
- * directly copy the (with mob tag) cell into the new store file.
- * </li>
- * <li>
- * Otherwise, retrieve the mob cell from the mob file, and writes a copy of the cell into
- * the new store file.
- * </li>
+ * <li>If the value size of a cell is larger than the threshold, this cell is regarded as a mob,
+ * directly copy the (with mob tag) cell into the new store file.</li>
+ * <li>Otherwise, retrieve the mob cell from the mob file, and writes a copy of the cell into the
+ * new store file.</li>
* </ol>
* 2. If the Put cell doesn't have a reference tag.
* <ol>
- * <li>
- * If the value size of a cell is larger than the threshold, this cell is regarded as a mob,
- * write this cell to a mob file, and write the path of this mob file to the store file.
- * </li>
- * <li>
- * Otherwise, directly write this cell into the store file.
- * </li>
+ * <li>If the value size of a cell is larger than the threshold, this cell is regarded as a mob,
+ * write this cell to a mob file, and write the path of this mob file to the store file.</li>
+ * <li>Otherwise, directly write this cell into the store file.</li>
* </ol>
* @param fd File details
* @param scanner Where to read from.
@@ -245,13 +234,12 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
mobRefSet.get().clear();
boolean isUserRequest = userRequest.get();
boolean compactMOBs = major && isUserRequest;
- boolean ioOptimizedMode = conf.get(MobConstants.MOB_COMPACTION_TYPE_KEY,
- MobConstants.DEFAULT_MOB_COMPACTION_TYPE)
- .equals(MobConstants.IO_OPTIMIZED_MOB_COMPACTION_TYPE);
+ boolean ioOptimizedMode =
+ conf.get(MobConstants.MOB_COMPACTION_TYPE_KEY, MobConstants.DEFAULT_MOB_COMPACTION_TYPE)
+ .equals(MobConstants.IO_OPTIMIZED_MOB_COMPACTION_TYPE);
- boolean discardMobMiss =
- conf.getBoolean(MobConstants.MOB_UNSAFE_DISCARD_MISS_KEY,
- MobConstants.DEFAULT_MOB_DISCARD_MISS);
+ boolean discardMobMiss = conf.getBoolean(MobConstants.MOB_UNSAFE_DISCARD_MISS_KEY,
+ MobConstants.DEFAULT_MOB_DISCARD_MISS);
long maxMobFileSize = conf.getLong(MobConstants.MOB_COMPACTION_MAX_FILE_SIZE_KEY,
MobConstants.DEFAULT_MOB_COMPACTION_MAX_FILE_SIZE);
@@ -321,7 +309,7 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
LOG.error("Missing MOB cell value: file=" + pp + " cell=" + mobCell);
continue;
} else if (mobCell.getValueLength() == 0) {
- //TODO: what to do here? This is data corruption?
+ // TODO: what to do here? This is data corruption?
LOG.warn("Found 0 length MOB cell in a file={} cell={}", pp, mobCell);
}
@@ -331,8 +319,8 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
if (!ioOptimizedMode) {
mobFileWriter.append(mobCell);
mobCells++;
- writer.append(MobUtils.createMobRefCell(mobCell, fileName,
- this.mobStore.getRefCellTags()));
+ writer.append(
+ MobUtils.createMobRefCell(mobCell, fileName, this.mobStore.getRefCellTags()));
} else {
// I/O optimized mode
// Check if MOB cell origin file size is
@@ -340,9 +328,8 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
Long size = mobLengthMap.get().get(fName);
if (size == null) {
// FATAL error, abort compaction
- String msg =
- String.format("Found unreferenced MOB file during compaction %s, aborting.",
- fName);
+ String msg = String.format(
+ "Found unreferenced MOB file during compaction %s, aborting.", fName);
LOG.error(msg);
throw new IOException(msg);
}
@@ -353,15 +340,15 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
mobFileWriter.append(mobCell);
// Update number of mobCells in a current mob writer
mobCells++;
- writer.append(MobUtils.createMobRefCell(mobCell, fileName,
- this.mobStore.getRefCellTags()));
+ writer.append(
+ MobUtils.createMobRefCell(mobCell, fileName, this.mobStore.getRefCellTags()));
// Update total size of the output (we do not take into account
// file compression yet)
long len = getLength(mobFileWriter);
if (len > maxMobFileSize) {
- LOG.debug("Closing output MOB File, length={} file={}",
- len, Bytes.toString(fileName));
+ LOG.debug("Closing output MOB File, length={} file={}", len,
+ Bytes.toString(fileName));
commitOrAbortMobWriter(mobFileWriter, fd.maxSeqId, mobCells, major);
mobFileWriter = newMobWriter(fd);
fileName = Bytes.toBytes(mobFileWriter.getPath().getName());
@@ -532,8 +519,7 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
return mobFileWriter.getPos();
}
-
- private StoreFileWriter newMobWriter(FileDetails fd/*, boolean compactMOBs*/)
+ private StoreFileWriter newMobWriter(FileDetails fd/* , boolean compactMOBs */)
throws IOException {
try {
StoreFileWriter mobFileWriter = mobStore.createWriterInTmp(new Date(fd.latestPutTs),
@@ -549,18 +535,19 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
}
}
- private void commitOrAbortMobWriter(StoreFileWriter mobFileWriter, long maxSeqId,
- long mobCells, boolean major) throws IOException
- {
+ private void commitOrAbortMobWriter(StoreFileWriter mobFileWriter, long maxSeqId, long mobCells,
+ boolean major) throws IOException {
// Commit or abort major mob writer
// If IOException happens during below operation, some
// MOB files can be committed partially, but corresponding
// store file won't be committed, therefore these MOB files
// become orphans and will be deleted during next MOB cleaning chore cycle
- LOG.debug("Commit or abort size={} mobCells={} major={} file={}",
- mobFileWriter.getPos(), mobCells, major, mobFileWriter.getPath().getName());
- Path path = MobUtils.getMobFamilyPath(conf, store.getTableName(), store.getColumnFamilyName());
+
if (mobFileWriter != null) {
+ LOG.info("Commit or abort size={} mobCells={} major={} file={}", mobFileWriter.getPos(),
+ mobCells, major, mobFileWriter.getPath().getName());
+ Path path =
+ MobUtils.getMobFamilyPath(conf, store.getTableName(), store.getColumnFamilyName());
if (mobCells > 0) {
// If the mob file is not empty, commit it.
mobFileWriter.appendMetadata(maxSeqId, major, mobCells);
@@ -568,18 +555,19 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
mobStore.commitFile(mobFileWriter.getPath(), path);
} else {
// If the mob file is empty, delete it instead of committing.
- LOG.debug("Aborting writer for {} because there are no MOB cells",
- mobFileWriter.getPath());
+ LOG.debug("Aborting writer for {} because there are no MOB cells", mobFileWriter.getPath());
// Remove MOB file from reference set
mobRefSet.get().remove(mobFileWriter.getPath().getName());
abortWriter(mobFileWriter);
}
+ } else {
+ LOG.info("Mob file writer is null, skipping commit/abort.");
}
}
protected static String createKey(TableName tableName, String encodedName,
String columnFamilyName) {
- return tableName.getNameAsString()+ "_" + encodedName + "_"+ columnFamilyName;
+ return tableName.getNameAsString() + "_" + encodedName + "_" + columnFamilyName;
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index f20435d..0aed8b2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -428,8 +428,6 @@ public final class MobUtils {
storeFiles);
}
-
-
/**
* Creates a mob reference KeyValue. The value of the mob reference KeyValue is mobCellValueSize +
* mobFileName.
@@ -620,8 +618,8 @@ public final class MobUtils {
}
/**
- * Indicates whether return null value when the mob file is missing or corrupt.
- * The information is set in the attribute "empty.value.on.mobcell.miss" of scan.
+ * Indicates whether return null value when the mob file is missing or corrupt. The information is
+ * set in the attribute "empty.value.on.mobcell.miss" of scan.
* @param scan The current scan.
* @return True if the readEmptyValueOnMobCellMiss is enabled.
*/
@@ -676,12 +674,11 @@ public final class MobUtils {
if (index < 0) {
return null;
}
- return mobFileName.substring(index+1);
+ return mobFileName.substring(index + 1);
}
/**
- * Get list of referenced MOB files from a given collection
- * of store files
+ * Get list of referenced MOB files from a given collection of store files
* @param storeFiles store files
* @param mobDir MOB file directory
* @return list of MOB file paths
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
index b51300a..c715d6b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
@@ -53,36 +53,34 @@ public class StoreFileInfo {
private static final Logger LOG = LoggerFactory.getLogger(StoreFileInfo.class);
/**
- * A non-capture group, for hfiles, so that this can be embedded.
- * HFiles are uuid ([0-9a-z]+). Bulk loaded hfiles has (_SeqId_[0-9]+_) has suffix.
- * The mob del file has (_del) as suffix.
+ * A non-capture group, for hfiles, so that this can be embedded. HFiles are uuid ([0-9a-z]+).
+ * Bulk loaded hfiles has (_SeqId_[0-9]+_) has suffix. The mob del file has (_del) as suffix.
*/
public static final String HFILE_NAME_REGEX = "[0-9a-f]+(?:(?:_SeqId_[0-9]+_)|(?:_del))?";
/** Regex that will work for hfiles */
- private static final Pattern HFILE_NAME_PATTERN =
- Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
+ private static final Pattern HFILE_NAME_PATTERN = Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
/**
- * A non-capture group, for del files, so that this can be embedded.
- * A del file has (_del) as suffix.
+ * A non-capture group, for del files, so that this can be embedded. A del file has (_del) as
+ * suffix.
*/
public static final String DELFILE_NAME_REGEX = "[0-9a-f]+(?:_del)";
/** Regex that will work for del files */
private static final Pattern DELFILE_NAME_PATTERN =
- Pattern.compile("^(" + DELFILE_NAME_REGEX + ")");
+ Pattern.compile("^(" + DELFILE_NAME_REGEX + ")");
/**
- * Regex that will work for straight reference names ({@code <hfile>.<parentEncRegion>})
- * and hfilelink reference names ({@code <table>=<region>-<hfile>.<parentEncRegion>})
- * If reference, then the regex has more than just one group.
- * Group 1, hfile/hfilelink pattern, is this file's id.
- * Group 2 '(.+)' is the reference's parent region name.
+ * Regex that will work for straight reference names ({@code <hfile>.<parentEncRegion>}) and
+ * hfilelink reference names ({@code
+ * <table>
+ * =<region>-<hfile>.<parentEncRegion>}) If reference, then the regex has more than just one
+ * group. Group 1, hfile/hfilelink pattern, is this file's id. Group 2 '(.+)' is the reference's
+ * parent region name.
*/
- private static final Pattern REF_NAME_PATTERN =
- Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
- HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
+ private static final Pattern REF_NAME_PATTERN = Pattern
+ .compile(String.format("^(%s|%s)\\.(.+)$", HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
public static final String STORE_FILE_READER_NO_READAHEAD = "hbase.store.reader.no-readahead";
public static final boolean DEFAULT_STORE_FILE_READER_NO_READAHEAD = false;
@@ -134,50 +132,51 @@ public class StoreFileInfo {
this(conf, fs, null, initialPath, primaryReplica);
}
- private StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
- final Path initialPath, final boolean primaryReplica) throws IOException {
- assert fs != null;
- assert initialPath != null;
- assert conf != null;
-
- this.fs = fs;
- this.conf = conf;
- this.initialPath = initialPath;
- this.primaryReplica = primaryReplica;
- this.noReadahead = this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD, DEFAULT_STORE_FILE_READER_NO_READAHEAD);
- Path p = initialPath;
- if (HFileLink.isHFileLink(p)) {
- // HFileLink
- this.reference = null;
- this.link = HFileLink.buildFromHFileLinkPattern(conf, p);
- LOG.trace("{} is a link", p);
- } else if (isReference(p)) {
- this.reference = Reference.read(fs, p);
- Path referencePath = getReferredToFile(p);
- if (HFileLink.isHFileLink(referencePath)) {
- // HFileLink Reference
- this.link = HFileLink.buildFromHFileLinkPattern(conf, referencePath);
- } else {
- // Reference
- this.link = null;
- }
- LOG.trace("{} is a {} reference to {}", p, reference.getFileRegion(), referencePath);
- } else if (isHFile(p) || isMobFile(p) || isMobRefFile(p)) {
- // HFile
- if (fileStatus != null) {
- this.createdTimestamp = fileStatus.getModificationTime();
- this.size = fileStatus.getLen();
- } else {
- FileStatus fStatus = fs.getFileStatus(initialPath);
- this.createdTimestamp = fStatus.getModificationTime();
- this.size = fStatus.getLen();
- }
- this.reference = null;
- this.link = null;
- } else {
- throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
- }
- }
+ private StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
+ final Path initialPath, final boolean primaryReplica) throws IOException {
+ assert fs != null;
+ assert initialPath != null;
+ assert conf != null;
+
+ this.fs = fs;
+ this.conf = conf;
+ this.initialPath = initialPath;
+ this.primaryReplica = primaryReplica;
+ this.noReadahead = this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD,
+ DEFAULT_STORE_FILE_READER_NO_READAHEAD);
+ Path p = initialPath;
+ if (HFileLink.isHFileLink(p)) {
+ // HFileLink
+ this.reference = null;
+ this.link = HFileLink.buildFromHFileLinkPattern(conf, p);
+ LOG.trace("{} is a link", p);
+ } else if (isReference(p)) {
+ this.reference = Reference.read(fs, p);
+ Path referencePath = getReferredToFile(p);
+ if (HFileLink.isHFileLink(referencePath)) {
+ // HFileLink Reference
+ this.link = HFileLink.buildFromHFileLinkPattern(conf, referencePath);
+ } else {
+ // Reference
+ this.link = null;
+ }
+ LOG.trace("{} is a {} reference to {}", p, reference.getFileRegion(), referencePath);
+ } else if (isHFile(p) || isMobFile(p) || isMobRefFile(p)) {
+ // HFile
+ if (fileStatus != null) {
+ this.createdTimestamp = fileStatus.getModificationTime();
+ this.size = fileStatus.getLen();
+ } else {
+ FileStatus fStatus = fs.getFileStatus(initialPath);
+ this.createdTimestamp = fStatus.getModificationTime();
+ this.size = fStatus.getLen();
+ }
+ this.reference = null;
+ this.link = null;
+ } else {
+ throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
+ }
+ }
/**
* Create a Store File Info
@@ -227,11 +226,11 @@ public class StoreFileInfo {
this.conf = conf;
this.primaryReplica = false;
this.initialPath = (fileStatus == null) ? null : fileStatus.getPath();
- this.createdTimestamp = (fileStatus == null) ? 0 :fileStatus.getModificationTime();
+ this.createdTimestamp = (fileStatus == null) ? 0 : fileStatus.getModificationTime();
this.reference = reference;
this.link = link;
this.noReadahead = this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD,
- DEFAULT_STORE_FILE_READER_NO_READAHEAD);
+ DEFAULT_STORE_FILE_READER_NO_READAHEAD);
}
/**
@@ -251,8 +250,9 @@ public class StoreFileInfo {
}
/*
- * @return the Reference object associated to this StoreFileInfo.
- * null if the StoreFile is not a reference.
+ * @return the Reference object associated to this StoreFileInfo.
+ * null if the StoreFile is not a
+ * reference.
*/
public Reference getReference() {
return this.reference;
@@ -278,8 +278,7 @@ public class StoreFileInfo {
return this.hdfsBlocksDistribution;
}
- StoreFileReader createReader(ReaderContext context, CacheConfig cacheConf)
- throws IOException {
+ StoreFileReader createReader(ReaderContext context, CacheConfig cacheConf) throws IOException {
StoreFileReader reader = null;
if (this.reference != null) {
reader = new HalfStoreFileReader(context, hfileInfo, cacheConf, reference, refCount, conf);
@@ -316,12 +315,9 @@ public class StoreFileInfo {
status = fs.getFileStatus(initialPath);
}
long length = status.getLen();
- ReaderContextBuilder contextBuilder = new ReaderContextBuilder()
- .withInputStreamWrapper(in)
- .withFileSize(length)
- .withPrimaryReplicaReader(this.primaryReplica)
- .withReaderType(type)
- .withFileSystem(fs);
+ ReaderContextBuilder contextBuilder =
+ new ReaderContextBuilder().withInputStreamWrapper(in).withFileSize(length)
+ .withPrimaryReplicaReader(this.primaryReplica).withReaderType(type).withFileSystem(fs);
if (this.reference != null) {
contextBuilder.withFilePath(this.getPath());
} else {
@@ -425,8 +421,8 @@ public class StoreFileInfo {
@Override
public String toString() {
- return this.getPath() +
- (isReference() ? "->" + getReferredToFile(this.getPath()) + "-" + reference : "");
+ return this.getPath()
+ + (isReference() ? "->" + getReferredToFile(this.getPath()) + "-" + reference : "");
}
/**
@@ -463,9 +459,10 @@ public class StoreFileInfo {
parts[0] = fileName.substring(0, lastIndex);
parts[1] = fileName.substring(lastIndex + 1);
String name = parts[0] + "." + parts[1];
- Matcher m = REF_NAME_PATTERN.matcher(name);
+ Matcher m = REF_NAME_PATTERN.matcher(name);
return m.matches() && m.groupCount() > 1;
}
+
/**
* @param path Path to check.
* @return True if the path has format of a del file.
@@ -508,8 +505,8 @@ public class StoreFileInfo {
}
/*
- * Return path to the file referred to by a Reference. Presumes a directory
- * hierarchy of <code>${hbase.rootdir}/data/${namespace}/tablename/regionname/familyname</code>.
+ * Return path to the file referred to by a Reference. Presumes a directory hierarchy of
+ * <code>${hbase.rootdir}/data/${namespace}/tablename/regionname/familyname</code>.
* @param p Path to a Reference file.
* @return Calculated path to parent region file.
* @throws IllegalArgumentException when path regex fails to match.
@@ -518,8 +515,7 @@ public class StoreFileInfo {
Matcher m = REF_NAME_PATTERN.matcher(p.getName());
if (m == null || !m.matches()) {
LOG.warn("Failed match of store file name {}", p.toString());
- throw new IllegalArgumentException("Failed match of store file name " +
- p.toString());
+ throw new IllegalArgumentException("Failed match of store file name " + p.toString());
}
// Other region name is suffix on the passed Reference file name
@@ -530,9 +526,9 @@ public class StoreFileInfo {
LOG.trace("reference {} to region={} hfile={}", p, otherRegion, nameStrippedOfSuffix);
// Build up new path with the referenced region in place of our current
- // region in the reference path. Also strip regionname suffix from name.
- return new Path(new Path(new Path(tableDir, otherRegion),
- p.getParent().getName()), nameStrippedOfSuffix);
+ // region in the reference path. Also strip regionname suffix from name.
+ return new Path(new Path(new Path(tableDir, otherRegion), p.getParent().getName()),
+ nameStrippedOfSuffix);
}
/**
@@ -541,8 +537,7 @@ public class StoreFileInfo {
* @return <tt>true</tt> if the file could be a valid store file, <tt>false</tt> otherwise
*/
public static boolean validateStoreFileName(final String fileName) {
- if (HFileLink.isHFileLink(fileName) || isReference(fileName))
- return(true);
+ if (HFileLink.isHFileLink(fileName) || isReference(fileName)) return (true);
return !fileName.contains("-");
}
@@ -551,12 +546,10 @@ public class StoreFileInfo {
* @param fileStatus The {@link FileStatus} of the file
* @return <tt>true</tt> if the file is valid
*/
- public static boolean isValid(final FileStatus fileStatus)
- throws IOException {
+ public static boolean isValid(final FileStatus fileStatus) throws IOException {
final Path p = fileStatus.getPath();
- if (fileStatus.isDirectory())
- return false;
+ if (fileStatus.isDirectory()) return false;
// Check for empty hfile. Should never be the case but can happen
// after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646
@@ -570,21 +563,19 @@ public class StoreFileInfo {
}
/**
- * helper function to compute HDFS blocks distribution of a given reference
- * file.For reference file, we don't compute the exact value. We use some
- * estimate instead given it might be good enough. we assume bottom part
- * takes the first half of reference file, top part takes the second half
- * of the reference file. This is just estimate, given
- * midkey ofregion != midkey of HFile, also the number and size of keys vary.
- * If this estimate isn't good enough, we can improve it later.
- * @param fs The FileSystem
- * @param reference The reference
- * @param status The reference FileStatus
+ * helper function to compute HDFS blocks distribution of a given reference file.For reference
+ * file, we don't compute the exact value. We use some estimate instead given it might be good
+ * enough. we assume bottom part takes the first half of reference file, top part takes the second
+ * half of the reference file. This is just estimate, given midkey ofregion != midkey of HFile,
+ * also the number and size of keys vary. If this estimate isn't good enough, we can improve it
+ * later.
+ * @param fs The FileSystem
+ * @param reference The reference
+ * @param status The reference FileStatus
* @return HDFS blocks distribution
*/
- private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
- final FileSystem fs, final Reference reference, final FileStatus status)
- throws IOException {
+ private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(final FileSystem fs,
+ final Reference reference, final FileStatus status) throws IOException {
if (status == null) {
return null;
}
@@ -593,11 +584,11 @@ public class StoreFileInfo {
long length = 0;
if (Reference.isTopFileRegion(reference.getFileRegion())) {
- start = status.getLen()/2;
- length = status.getLen() - status.getLen()/2;
+ start = status.getLen() / 2;
+ length = status.getLen() - status.getLen() / 2;
} else {
start = 0;
- length = status.getLen()/2;
+ length = status.getLen() / 2;
}
return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
}
@@ -609,16 +600,16 @@ public class StoreFileInfo {
if (!(that instanceof StoreFileInfo)) return false;
- StoreFileInfo o = (StoreFileInfo)that;
+ StoreFileInfo o = (StoreFileInfo) that;
if (initialPath != null && o.initialPath == null) return false;
if (initialPath == null && o.initialPath != null) return false;
- if (initialPath != o.initialPath && initialPath != null
- && !initialPath.equals(o.initialPath)) return false;
+ if (initialPath != o.initialPath && initialPath != null && !initialPath.equals(o.initialPath))
+ return false;
if (reference != null && o.reference == null) return false;
if (reference == null && o.reference != null) return false;
- if (reference != o.reference && reference != null
- && !reference.equals(o.reference)) return false;
+ if (reference != o.reference && reference != null && !reference.equals(o.reference))
+ return false;
if (link != null && o.link == null) return false;
if (link == null && o.link != null) return false;
@@ -631,9 +622,9 @@ public class StoreFileInfo {
public int hashCode() {
int hash = 17;
hash = hash * 31 + ((reference == null) ? 0 : reference.hashCode());
- hash = hash * 31 + ((initialPath == null) ? 0 : initialPath.hashCode());
+ hash = hash * 31 + ((initialPath == null) ? 0 : initialPath.hashCode());
hash = hash * 31 + ((link == null) ? 0 : link.hashCode());
- return hash;
+ return hash;
}
/**
@@ -676,8 +667,7 @@ public class StoreFileInfo {
StoreFileReader reader = null;
if (this.coprocessorHost != null) {
reader = this.coprocessorHost.preStoreFileReaderOpen(fs, this.getPath(),
- context.getInputStreamWrapper(), context.getFileSize(),
- cacheConf, reference);
+ context.getInputStreamWrapper(), context.getFileSize(), cacheConf, reference);
}
return reader;
}
@@ -687,8 +677,7 @@ public class StoreFileInfo {
StoreFileReader res = reader;
if (this.coprocessorHost != null) {
res = this.coprocessorHost.postStoreFileReaderOpen(fs, this.getPath(),
- context.getInputStreamWrapper(), context.getFileSize(),
- cacheConf, reference, reader);
+ context.getInputStreamWrapper(), context.getFileSize(), cacheConf, reference, reader);
}
return res;
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java
index aeff96e..fe79a65 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java
@@ -263,9 +263,9 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
byte[][] families = { Bytes.toBytes("mob") };
loadData(tableName, families, 3000, 8);
- admin.majorCompact(tableName, CompactType.MOB).get();
+ admin.majorCompact(tableName).get();
- CompactionState state = admin.getCompactionState(tableName, CompactType.MOB).get();
+ CompactionState state = admin.getCompactionState(tableName).get();
assertNotEquals(CompactionState.NONE, state);
waitUntilMobCompactionFinished(tableName);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/FaultyMobStoreCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/FaultyMobStoreCompactor.java
index 057d81f..2003d63 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/FaultyMobStoreCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/FaultyMobStoreCompactor.java
@@ -100,8 +100,8 @@ public class FaultyMobStoreCompactor extends DefaultMobStoreCompactor {
mobRefSet.get().clear();
boolean isUserRequest = userRequest.get();
boolean compactMOBs = major && isUserRequest;
- boolean discardMobMiss =
- conf.getBoolean(MobConstants.MOB_UNSAFE_DISCARD_MISS_KEY, MobConstants.DEFAULT_MOB_DISCARD_MISS);
+ boolean discardMobMiss = conf.getBoolean(MobConstants.MOB_UNSAFE_DISCARD_MISS_KEY,
+ MobConstants.DEFAULT_MOB_DISCARD_MISS);
boolean mustFail = false;
if (compactMOBs) {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressTool.java
index 146fb21..81f5cf9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressTool.java
@@ -23,12 +23,14 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
+
+
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class MobStressTool extends AbstractHBaseTool {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java
index 30e456e..43e3e40 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java
@@ -21,21 +21,19 @@ import static org.apache.hadoop.hbase.HConstants.REPLICATION_CLUSTER_ID;
import static org.apache.hadoop.hbase.HConstants.REPLICATION_CONF_DIR;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Random;
-import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
@@ -62,21 +60,14 @@ import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionObserver;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
-import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobFileName;
-import org.apache.hadoop.hbase.mob.MobUtils;
-//import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.TestReplicationBase;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.tool.BulkLoadHFilesTool;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.After;
@@ -258,31 +249,6 @@ public class TestBulkLoadReplication extends TestReplicationBase {
assertEquals(9, BULK_LOADS_COUNT.get());
}
- @Ignore
- @Test
- public void testPartionedMOBCompactionBulkLoadDoesntReplicate() throws Exception {
-// Path path = createMobFiles(UTIL3);
- ColumnFamilyDescriptor descriptor =
- new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(famName);
- ExecutorService pool = null;
- try {
- pool = Executors.newFixedThreadPool(1);
-// PartitionedMobCompactor compactor =
-// new PartitionedMobCompactor(UTIL3.getConfiguration(), UTIL3.getTestFileSystem(), tableName,
-// descriptor, pool);
- BULK_LOAD_LATCH = new CountDownLatch(1);
- BULK_LOADS_COUNT.set(0);
-// compactor.compact(Arrays.asList(UTIL3.getTestFileSystem().listStatus(path)), true);
- assertTrue(BULK_LOAD_LATCH.await(1, TimeUnit.SECONDS));
- Thread.sleep(400);
- assertEquals(1, BULK_LOADS_COUNT.get());
- } finally {
- if(pool != null && !pool.isTerminated()) {
- pool.shutdownNow();
- }
- }
- }
-
protected void assertBulkLoadConditions(TableName tableName, byte[] row, byte[] value,
HBaseTestingUtility utility, Table...tables) throws Exception {
@@ -350,37 +316,6 @@ public class TestBulkLoadReplication extends TestReplicationBase {
return hFileLocation.getAbsoluteFile().getAbsolutePath();
}
-
-// private Path createMobFiles(HBaseTestingUtility util) throws IOException {
-// Path testDir = FSUtils.getRootDir(util.getConfiguration());
-// Path mobTestDir = new Path(testDir, MobConstants.MOB_DIR_NAME);
-// Path basePath = new Path(new Path(mobTestDir, tableName.getNameAsString()), "f");
-// HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
-// MobFileName mobFileName = null;
-// byte[] mobFileStartRow = new byte[32];
-// for (byte rowKey : Bytes.toBytes("01234")) {
-// mobFileName = MobFileName.create(mobFileStartRow, MobUtils.formatDate(new Date()),
-// UUID.randomUUID().toString().replaceAll("-", ""));
-// StoreFileWriter mobFileWriter =
-// new StoreFileWriter.Builder(util.getConfiguration(),
-// new CacheConfig(util.getConfiguration()), util.getTestFileSystem()).withFileContext(meta)
-// .withFilePath(new Path(basePath, mobFileName.getFileName())).build();
-// long now = System.currentTimeMillis();
-// try {
-// for (int i = 0; i < 10; i++) {
-// byte[] key = Bytes.add(Bytes.toBytes(rowKey), Bytes.toBytes(i));
-// byte[] dummyData = new byte[5000];
-// new Random().nextBytes(dummyData);
-// mobFileWriter.append(
-// new KeyValue(key, famName, Bytes.toBytes("1"), now, KeyValue.Type.Put, dummyData));
-// }
-// } finally {
-// mobFileWriter.close();
-// }
-// }
-// return basePath;
-// }
-
public static class BulkReplicationTestObserver implements RegionCoprocessor {
String clusterName;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
index fa44c18..6b252c6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
@@ -29,6 +29,7 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Random;
import java.util.Set;
@@ -37,8 +38,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparatorImpl;
-import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -63,7 +62,11 @@ import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobFileCache;
import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
+import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
@@ -158,6 +161,9 @@ public class TestMobStoreCompaction {
/**
* During compaction, the mob threshold size is changed.
+ * The test is no longer valid. Major MOB compaction must be triggered by User
+ * HRegion does not provide public API to trigger major-compaction by User
+ * This test will move to mob sub-package.
*/
@Test
public void testLargerValue() throws Exception {
@@ -178,8 +184,21 @@ public class TestMobStoreCompaction {
// Change the threshold larger than the data size
setMobThreshold(region, COLUMN_FAMILY, 500);
region.initialize();
- region.compactStores();
-
+
+ List<HStore> stores = region.getStores();
+ for (HStore store: stores) {
+ // Force major compaction
+ store.triggerMajorCompaction();
+ Optional<CompactionContext> context =
+ store.requestCompaction(HStore.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY,
+ User.getCurrent());
+ if (!context.isPresent()) {
+ continue;
+ }
+ region.compact(context.get(), store,
+ NoLimitThroughputController.INSTANCE, User.getCurrent());
+ }
+
assertEquals("After compaction: store files", 1, countStoreFiles());
assertEquals("After compaction: mob file count", compactionThreshold, countMobFiles());
assertEquals("After compaction: referenced mob file count", 0, countReferencedMobFiles());
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index bc44daa..cfed256 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -25,6 +25,8 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CompatibilityFactory;
import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -53,6 +55,10 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.master.LoadBalancer;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
+import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
+import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
@@ -494,7 +500,20 @@ public class TestRegionServerMetrics {
setMobThreshold(region, cf, 100);
// metrics are reset by the region initialization
region.initialize();
- region.compact(true);
+ // This is how we MOB compact region
+ List<HStore> stores = region.getStores();
+ for (HStore store: stores) {
+ // Force major compaction
+ store.triggerMajorCompaction();
+ Optional<CompactionContext> context =
+ store.requestCompaction(HStore.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY,
+ User.getCurrent());
+ if (!context.isPresent()) {
+ continue;
+ }
+ region.compact(context.get(), store,
+ NoLimitThroughputController.INSTANCE, User.getCurrent());
+ }
metricsRegionServer.getRegionServerWrapper().forceRecompute();
assertCounter("cellsCountCompactedFromMob", numHfiles);
assertCounter("cellsCountCompactedToMob", 0);