You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2020/02/15 20:29:38 UTC
[hbase] 04/18: HBASE-22749: code review
This is an automated email from the ASF dual-hosted git repository.
busbey pushed a commit to branch HBASE-22749
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 65261a690ec848dd3ea21e97b140d114e585db79
Author: Vladimir Rodionov <vr...@apache.org>
AuthorDate: Thu Jan 23 17:59:14 2020 -0800
HBASE-22749: code review
---
.../hadoop/hbase/IntegrationTestMobCompaction.java | 5 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 1 -
.../hadoop/hbase/mob/DefaultMobStoreCompactor.java | 134 ++++++++++++---------
.../hadoop/hbase/mob/DefaultMobStoreFlusher.java | 8 +-
.../org/apache/hadoop/hbase/mob/MobConstants.java | 77 +++++++-----
.../hadoop/hbase/mob/MobFileCleanerChore.java | 73 ++++++-----
.../hadoop/hbase/mob/MobFileCompactionChore.java | 81 +++++++++----
.../apache/hadoop/hbase/mob/MobStoreEngine.java | 4 +-
.../java/org/apache/hadoop/hbase/mob/MobUtils.java | 25 +---
.../hadoop/hbase/regionserver/HMobStore.java | 3 +-
.../apache/hadoop/hbase/regionserver/HRegion.java | 2 +-
.../hadoop/hbase/regionserver/StoreFileInfo.java | 99 ++++++++-------
.../hadoop/hbase/regionserver/StoreFileWriter.java | 12 +-
.../hbase/regionserver/compactions/Compactor.java | 2 +-
.../hadoop/hbase/mob/FaultyMobStoreCompactor.java | 8 +-
.../org/apache/hadoop/hbase/mob/MobStressTool.java | 6 +-
...MobCompaction.java => MobStressToolRunner.java} | 75 ++----------
.../hadoop/hbase/mob/TestMobCompactionBase.java | 77 +++++-------
.../hadoop/hbase/mob/TestMobCompactionOptMode.java | 2 +-
.../mob/TestMobCompactionOptRegionBatchMode.java | 2 +-
...anerChore.java => TestMobFileCleanerChore.java} | 65 +++++-----
.../TestMobStoreCompaction.java | 21 ++--
.../{regionserver => mob}/TestMobStoreScanner.java | 4 +-
.../regionserver/TestBulkLoadReplication.java | 4 -
.../regionserver/TestRegionServerMetrics.java | 6 +-
25 files changed, 404 insertions(+), 392 deletions(-)
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java
index 0ee1e09..9aa1978 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java
@@ -65,7 +65,6 @@ import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
* @see <a href="https://issues.apache.org/jira/browse/HBASE-22749">HBASE-22749</a>
* <p>
* Sample usage:
- *
* <pre>
* hbase org.apache.hadoop.hbase.IntegrationTestMobCompaction -Dservers=10 -Drows=1000000
* -Dfailprob=0.2
@@ -233,8 +232,8 @@ public class IntegrationTestMobCompaction extends IntegrationTestBase {
conf.setInt("hbase.hstore.blockingStoreFiles", 150);
conf.setInt("hbase.hstore.compaction.throughput.lower.bound", 52428800);
conf.setInt("hbase.hstore.compaction.throughput.higher.bound", 2 * 52428800);
- conf.setDouble("injected.fault.probability", failureProb);
- conf.set(MobStoreEngine.DEFAULT_MOB_COMPACTOR_CLASS_KEY,
+ conf.setDouble("hbase.mob.compaction.fault.probability", failureProb);
+ conf.set(MobStoreEngine.MOB_COMPACTOR_CLASS_KEY,
FaultyMobStoreCompactor.class.getName());
conf.setBoolean("hbase.table.sanity.checks", false);
conf.setLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, 20000);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 32d2fdc..69b0876 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -147,7 +147,6 @@ import org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.master.zksyncer.MasterAddressSyncer;
import org.apache.hadoop.hbase.master.zksyncer.MetaLocationSyncer;
-import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobFileCleanerChore;
import org.apache.hadoop.hbase.mob.MobFileCompactionChore;
import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
index aa7caa9..871a9d8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
@@ -72,6 +72,7 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
private static final Logger LOG = LoggerFactory.getLogger(DefaultMobStoreCompactor.class);
protected long mobSizeThreshold;
protected HMobStore mobStore;
+ protected boolean ioOptimizedMode = false;
/*
* MOB file reference set thread local variable. It contains set of a MOB file names, which newly
@@ -99,15 +100,15 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
};
/*
- * Map : MOB file name - file length Can be expensive for large amount of MOB files?
+ * Map : MOB file name - file length Can be expensive for large amount of MOB files.
*/
static ThreadLocal<HashMap<String, Long>> mobLengthMap =
- new ThreadLocal<HashMap<String, Long>>() {
- @Override
- protected HashMap<String, Long> initialValue() {
- return new HashMap<String, Long>();
- }
- };
+ new ThreadLocal<HashMap<String, Long>>() {
+ @Override
+ protected HashMap<String, Long> initialValue() {
+ return new HashMap<String, Long>();
+ }
+ };
private final InternalScannerFactory scannerFactory = new InternalScannerFactory() {
@@ -145,24 +146,33 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
if (!(store instanceof HMobStore)) {
throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
}
- mobStore = (HMobStore) store;
- mobSizeThreshold = store.getColumnFamilyDescriptor().getMobThreshold();
+ this.mobStore = (HMobStore) store;
+ this.mobSizeThreshold = store.getColumnFamilyDescriptor().getMobThreshold();
+ this.ioOptimizedMode = conf.get(MobConstants.MOB_COMPACTION_TYPE_KEY,
+ MobConstants.DEFAULT_MOB_COMPACTION_TYPE).
+ equals(MobConstants.OPTIMIZED_MOB_COMPACTION_TYPE);
+
}
@Override
public List<Path> compact(CompactionRequestImpl request,
ThroughputController throughputController, User user) throws IOException {
- LOG.info("Mob compaction: major=" + request.isMajor() + " isAll=" + request.isAllFiles()
- + " priority=" + request.getPriority());
+ String tableName = store.getTableName().toString();
+ String regionName = store.getRegionInfo().getRegionNameAsString();
+ String familyName = store.getColumnFamilyName();
+ LOG.info("MOB compaction: major={} isAll={} priority={} throughput controller={}" +
+ " table={} cf={} region={}",
+ request.isMajor(), request.isAllFiles(), request.getPriority(),
+ throughputController, tableName, familyName, regionName);
if (request.getPriority() == HStore.PRIORITY_USER) {
userRequest.set(Boolean.TRUE);
} else {
userRequest.set(Boolean.FALSE);
}
- LOG.info("Mob compaction files: " + request.getFiles());
+ LOG.debug("MOB compaction table={} cf={} region={} files: ", tableName, familyName,
+ regionName, request.getFiles());
// Check if I/O optimized MOB compaction
- if (conf.get(MobConstants.MOB_COMPACTION_TYPE_KEY, MobConstants.DEFAULT_MOB_COMPACTION_TYPE)
- .equals(MobConstants.IO_OPTIMIZED_MOB_COMPACTION_TYPE)) {
+ if (ioOptimizedMode) {
if (request.isMajor() && request.getPriority() == HStore.PRIORITY_USER) {
Path mobDir =
MobUtils.getMobFamilyPath(conf, store.getTableName(), store.getColumnFamilyName());
@@ -170,9 +180,11 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
if (mobFiles.size() > 0) {
calculateMobLengthMap(mobFiles);
}
- LOG.info("I/O optimized MOB compaction. Total referenced MOB files: {}", mobFiles.size());
+ LOG.info("Table={} cf={} region={}. I/O optimized MOB compaction. "+
+ "Total referenced MOB files: {}", tableName, familyName, regionName, mobFiles.size());
}
}
+
return compact(request, scannerFactory, writerFactory, throughputController, user);
}
@@ -183,7 +195,7 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
for (Path p : mobFiles) {
FileStatus st = fs.getFileStatus(p);
long size = st.getLen();
- LOG.info("Ref MOB file={} size={}", p, size);
+ LOG.debug("Referenced MOB file={} size={}", p, size);
map.put(p.getName(), fs.getFileStatus(p).getLen());
}
}
@@ -234,20 +246,18 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
mobRefSet.get().clear();
boolean isUserRequest = userRequest.get();
boolean compactMOBs = major && isUserRequest;
- boolean ioOptimizedMode =
- conf.get(MobConstants.MOB_COMPACTION_TYPE_KEY, MobConstants.DEFAULT_MOB_COMPACTION_TYPE)
- .equals(MobConstants.IO_OPTIMIZED_MOB_COMPACTION_TYPE);
-
boolean discardMobMiss = conf.getBoolean(MobConstants.MOB_UNSAFE_DISCARD_MISS_KEY,
MobConstants.DEFAULT_MOB_DISCARD_MISS);
-
+ if (discardMobMiss) {
+ LOG.warn("{}=true. This is unsafe setting recommended only"+
+ " during upgrade process from MOB 1.0 to MOB 2.0 versions.",
+ MobConstants.MOB_UNSAFE_DISCARD_MISS_KEY);
+ }
long maxMobFileSize = conf.getLong(MobConstants.MOB_COMPACTION_MAX_FILE_SIZE_KEY,
MobConstants.DEFAULT_MOB_COMPACTION_MAX_FILE_SIZE);
- LOG.info("Compact MOB={} optimized={} maximum MOB file size={} major={}", compactMOBs,
- ioOptimizedMode, maxMobFileSize, major);
-
+ LOG.info("Compact MOB={} optimized={} maximum MOB file size={} major={} store={}", compactMOBs,
+ ioOptimizedMode, maxMobFileSize, major, getStoreInfo());
FileSystem fs = FileSystem.get(conf);
-
// Since scanner.next() can return 'false' but still be delivering data,
// we have to use a do/while loop.
List<Cell> cells = new ArrayList<>();
@@ -298,7 +308,7 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
mobCell = mobStore.resolve(c, true, false).getCell();
} catch (FileNotFoundException fnfe) {
if (discardMobMiss) {
- LOG.debug("Missing MOB cell: file={} not found cell={}", pp, c);
+ LOG.error("Missing MOB cell: file={} not found cell={}", fName, c);
continue;
} else {
throw fnfe;
@@ -306,11 +316,12 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
}
if (discardMobMiss && mobCell.getValueLength() == 0) {
- LOG.error("Missing MOB cell value: file=" + pp + " cell=" + mobCell);
+ LOG.error("Missing MOB cell value: file={} cell={}", pp, mobCell);
continue;
} else if (mobCell.getValueLength() == 0) {
- // TODO: what to do here? This is data corruption?
- LOG.warn("Found 0 length MOB cell in a file={} cell={}", pp, mobCell);
+ String errMsg = String.format("Found 0 length MOB cell in a file=%s cell=%s",
+ fName, mobCell);
+ throw new IOException(errMsg);
}
if (mobCell.getValueLength() > mobSizeThreshold) {
@@ -329,8 +340,8 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
if (size == null) {
// FATAL error, abort compaction
String msg = String.format(
- "Found unreferenced MOB file during compaction %s, aborting.", fName);
- LOG.error(msg);
+ "Found unreferenced MOB file during compaction %s, aborting compaction %s",
+ fName, getStoreInfo());
throw new IOException(msg);
}
// Can not be null
@@ -344,11 +355,10 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
MobUtils.createMobRefCell(mobCell, fileName, this.mobStore.getRefCellTags()));
// Update total size of the output (we do not take into account
// file compression yet)
- long len = getLength(mobFileWriter);
-
+ long len = mobFileWriter.getPos();
if (len > maxMobFileSize) {
- LOG.debug("Closing output MOB File, length={} file={}", len,
- Bytes.toString(fileName));
+ LOG.debug("Closing output MOB File, length={} file={}, store=", len,
+ Bytes.toString(fileName), getStoreInfo());
commitOrAbortMobWriter(mobFileWriter, fd.maxSeqId, mobCells, major);
mobFileWriter = newMobWriter(fd);
fileName = Bytes.toBytes(mobFileWriter.getPath().getName());
@@ -384,7 +394,7 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
if (ioOptimizedMode) {
// Update total size of the output (we do not take into account
// file compression yet)
- long len = getLength(mobFileWriter);
+ long len = mobFileWriter.getPos();
if (len > maxMobFileSize) {
commitOrAbortMobWriter(mobFileWriter, fd.maxSeqId, mobCells, major);
mobFileWriter = newMobWriter(fd);
@@ -411,9 +421,8 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
// Add MOB reference to a MOB reference set
mobRefSet.get().add(MobUtils.getMobFileName(c));
} else {
- // TODO ????
- LOG.error("Corrupted MOB reference: " + c);
- writer.append(c);
+ String errMsg = String.format("Corrupted MOB reference: %s", c.toString());
+ throw new IOException(errMsg);
}
} else if (c.getValueLength() <= mobSizeThreshold) {
// If the value size of a cell is not larger than the threshold, directly write it to
@@ -431,7 +440,7 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
cellsCountCompactedToMob++;
cellsSizeCompactedToMob += c.getValueLength();
if (ioOptimizedMode) {
- long len = getLength(mobFileWriter);
+ long len = mobFileWriter.getPos();
if (len > maxMobFileSize) {
commitOrAbortMobWriter(mobFileWriter, fd.maxSeqId, mobCells, major);
mobFileWriter = newMobWriter(fd);
@@ -486,8 +495,9 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
throw new InterruptedIOException(
"Interrupted while control throughput of compacting " + compactionName);
} catch (IOException t) {
- LOG.error("Mob compaction failed for region:{} ", store.getRegionInfo().getEncodedName());
- throw t;
+ String msg = "Mob compaction failed for region: " +
+ store.getRegionInfo().getEncodedName();
+ throw new IOException(msg, t);
} finally {
// Clone last cell in the final because writer will append last cell when committing. If
// don't clone here and once the scanner get closed, then the memory of last cell will be
@@ -498,15 +508,15 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
// Remove all MOB references because compaction failed
mobRefSet.get().clear();
// Abort writer
- LOG.debug("Aborting writer for {} because of a compaction failure",
- mobFileWriter.getPath());
+ LOG.debug("Aborting writer for {} because of a compaction failure, Store {}",
+ mobFileWriter.getPath(), getStoreInfo());
abortWriter(mobFileWriter);
}
}
// Commit last MOB writer
commitOrAbortMobWriter(mobFileWriter, fd.maxSeqId, mobCells, major);
-
+ clearThreadLocals();
mobStore.updateCellsCountCompactedFromMob(cellsCountCompactedFromMob);
mobStore.updateCellsCountCompactedToMob(cellsCountCompactedToMob);
mobStore.updateCellsSizeCompactedFromMob(cellsSizeCompactedFromMob);
@@ -515,11 +525,20 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
return true;
}
- private long getLength(StoreFileWriter mobFileWriter) throws IOException {
- return mobFileWriter.getPos();
+ private String getStoreInfo() {
+ return String.format("[table=%s family=%s region=%s]", store.getTableName().getNameAsString(),
+ store.getColumnFamilyName(), store.getRegionInfo().getEncodedName()) ;
}
- private StoreFileWriter newMobWriter(FileDetails fd/* , boolean compactMOBs */)
+ private void clearThreadLocals() {
+ Set<String> set = mobRefSet.get();
+ if (set != null) set.clear();
+ HashMap<String, Long> map = mobLengthMap.get();
+ if (map != null) map.clear();
+ }
+
+
+ private StoreFileWriter newMobWriter(FileDetails fd)
throws IOException {
try {
StoreFileWriter mobFileWriter = mobStore.createWriterInTmp(new Date(fd.latestPutTs),
@@ -530,8 +549,8 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
return mobFileWriter;
} catch (IOException e) {
// Bailing out
- LOG.error("Failed to create mob writer, ", e);
- throw e;
+ throw new IOException(String.format("Failed to create mob writer, store=%s",
+ getStoreInfo()), e);
}
}
@@ -544,8 +563,9 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
// become orphans and will be deleted during next MOB cleaning chore cycle
if (mobFileWriter != null) {
- LOG.info("Commit or abort size={} mobCells={} major={} file={}", mobFileWriter.getPos(),
- mobCells, major, mobFileWriter.getPath().getName());
+ LOG.debug("Commit or abort size={} mobCells={} major={} file={}, store={}",
+ mobFileWriter.getPos(), mobCells, major, mobFileWriter.getPath().getName(),
+ getStoreInfo());
Path path =
MobUtils.getMobFamilyPath(conf, store.getTableName(), store.getColumnFamilyName());
if (mobCells > 0) {
@@ -555,20 +575,18 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
mobStore.commitFile(mobFileWriter.getPath(), path);
} else {
// If the mob file is empty, delete it instead of committing.
- LOG.debug("Aborting writer for {} because there are no MOB cells", mobFileWriter.getPath());
+ LOG.debug("Aborting writer for {} because there are no MOB cells, store={}",
+ mobFileWriter.getPath(), getStoreInfo());
// Remove MOB file from reference set
mobRefSet.get().remove(mobFileWriter.getPath().getName());
abortWriter(mobFileWriter);
}
} else {
- LOG.info("Mob file writer is null, skipping commit/abort.");
+ LOG.debug("Mob file writer is null, skipping commit/abort, store=",
+ getStoreInfo());
}
}
- protected static String createKey(TableName tableName, String encodedName,
- String columnFamilyName) {
- return tableName.getNameAsString() + "_" + encodedName + "_" + columnFamilyName;
- }
@Override
protected List<Path> commitWriter(StoreFileWriter writer, FileDetails fd,
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
index 70591fb..5c4c602 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
@@ -254,7 +254,7 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
status.setStatus("Flushing mob file " + store + ": closing flushed file");
mobFileWriter.close();
mobStore.commitFile(mobFileWriter.getPath(), targetPath);
- LOG.debug("Flush store file: {}", writer.getPath());
+ LOG.debug("Flush store file: {}, store: {}", writer.getPath(), getStoreInfo());
mobStore.updateMobFlushCount();
mobStore.updateMobFlushedCellsCount(mobCount);
mobStore.updateMobFlushedCellsSize(mobSize);
@@ -272,6 +272,7 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
}
}
+ @Override
protected void finalizeWriter(StoreFileWriter writer, long cacheFlushSeqNum,
MonitoredTask status) throws IOException {
// Write out the log sequence number that corresponds to this output
@@ -283,4 +284,9 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
status.setStatus("Flushing " + store + ": closing flushed file");
writer.close();
}
+
+ private String getStoreInfo() {
+ return String.format("[table=%s family=%s region=%s]", store.getTableName().getNameAsString(),
+ store.getColumnFamilyName(), store.getRegionInfo().getEncodedName()) ;
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java
index b2d54fb..77f3dc0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java
@@ -35,26 +35,18 @@ public final class MobConstants {
public static final String MOB_CACHE_BLOCKS = "hbase.mob.cache.blocks";
public static final String MOB_SCAN_REF_ONLY = "hbase.mob.scan.ref.only";
public static final String EMPTY_VALUE_ON_MOBCELL_MISS = "empty.value.on.mobcell.miss";
-
public static final String MOB_FILE_CACHE_SIZE_KEY = "hbase.mob.file.cache.size";
public static final int DEFAULT_MOB_FILE_CACHE_SIZE = 1000;
-
public static final String MOB_DIR_NAME = "mobdir";
public static final String MOB_REGION_NAME = ".mob";
public static final byte[] MOB_REGION_NAME_BYTES = Bytes.toBytes(MOB_REGION_NAME);
-
public static final String MOB_CLEANER_PERIOD = "hbase.master.mob.cleaner.period";
-
- @Deprecated
- public static final String DEPRECATED_MOB_CLEANER_PERIOD =
- "hbase.master.mob.ttl.cleaner.period";
-
+ public static final String DEPRECATED_MOB_CLEANER_PERIOD = "hbase.master.mob.ttl.cleaner.period";
public static final int DEFAULT_MOB_CLEANER_PERIOD = 24 * 60 * 60; // one day
-
public static final String MOB_CACHE_EVICT_PERIOD = "hbase.mob.cache.evict.period";
public static final String MOB_CACHE_EVICT_REMAIN_RATIO = "hbase.mob.cache.evict.remain.ratio";
- public static final Tag MOB_REF_TAG = new ArrayBackedTag(TagType.MOB_REFERENCE_TAG_TYPE,
- HConstants.EMPTY_BYTE_ARRAY);
+ public static final Tag MOB_REF_TAG =
+ new ArrayBackedTag(TagType.MOB_REFERENCE_TAG_TYPE, HConstants.EMPTY_BYTE_ARRAY);
public static final float DEFAULT_EVICT_REMAIN_RATIO = 0.5f;
public static final long DEFAULT_MOB_CACHE_EVICT_PERIOD = 3600L;
@@ -62,13 +54,13 @@ public final class MobConstants {
public final static String TEMP_DIR_NAME = ".tmp";
/**
- * The max number of a MOB table regions that is allowed in a batch of the mob compaction.
- * By setting this number to a custom value, users can control the overall effect
- * of a major compaction of a large MOB-enabled table.
+ * The max number of a MOB table regions that is allowed in a batch of the mob compaction. By
+ * setting this number to a custom value, users can control the overall effect of a major
+ * compaction of a large MOB-enabled table.
*/
public static final String MOB_MAJOR_COMPACTION_REGION_BATCH_SIZE =
- "hbase.mob.major.compaction.region.batch.size";
+ "hbase.mob.major.compaction.region.batch.size";
/**
* Default is 0 - means no limit - all regions of a MOB table will be compacted at once
@@ -77,28 +69,24 @@ public final class MobConstants {
public static final int DEFAULT_MOB_MAJOR_COMPACTION_REGION_BATCH_SIZE = 0;
/**
- * The period that MobCompactionChore runs. The unit is second.
- * The default value is one week.
+ * The period that MobCompactionChore runs. The unit is second. The default value is one week.
*/
- public static final String MOB_COMPACTION_CHORE_PERIOD =
- "hbase.mob.compaction.chore.period";
- public static final int DEFAULT_MOB_COMPACTION_CHORE_PERIOD =
- 24 * 60 * 60 * 7; // a week
+ public static final String MOB_COMPACTION_CHORE_PERIOD = "hbase.mob.compaction.chore.period";
+ public static final int DEFAULT_MOB_COMPACTION_CHORE_PERIOD = 24 * 60 * 60 * 7; // a week
public static final String MOB_COMPACTOR_CLASS_KEY = "hbase.mob.compactor.class";
/**
- * Mob compaction type: "full", "io_optimized"
- * "full" - run full major compaction (during migration)
- * "io_optimized" - optimized version for use case with infrequent updates/deletes
+ * Mob compaction type: "full", "optimized" "full" - run full major compaction (during migration)
+ * "optimized" - optimized version for use case with infrequent updates/deletes
*/
- public final static String MOB_COMPACTION_TYPE_KEY = "hbase.mob.compaction.type";
-
- public final static String DEFAULT_MOB_COMPACTION_TYPE = "full";
- public final static String IO_OPTIMIZED_MOB_COMPACTION_TYPE = "optimized";
+ public final static String OPTIMIZED_MOB_COMPACTION_TYPE = "optimized";
public final static String FULL_MOB_COMPACTION_TYPE = "full";
+ public final static String MOB_COMPACTION_TYPE_KEY = "hbase.mob.compaction.type";
+
+ public final static String DEFAULT_MOB_COMPACTION_TYPE = FULL_MOB_COMPACTION_TYPE;
/**
* Maximum size of a MOB compaction selection
@@ -111,8 +99,8 @@ public final class MobConstants {
public static final long DEFAULT_MOB_COMPACTION_MAX_FILE_SIZE = 1024 * 1024 * 1024;
/**
- * Use this configuration option with caution, only during upgrade procedure
- * to handle missing MOB cells during compaction.
+ * Use this configuration option with caution, only during upgrade procedure to handle missing MOB
+ * cells during compaction.
*/
public static final String MOB_UNSAFE_DISCARD_MISS_KEY = "hbase.unsafe.mob.discard.miss";
@@ -125,6 +113,35 @@ public final class MobConstants {
public static final long DEFAULT_MIN_AGE_TO_ARCHIVE = 3600000; // 1h
+ /**
+ * Old configuration parameters (obsolete)
+ */
+
+ public final static String BULKLOAD_DIR_NAME = ".bulkload";
+ public final static byte[] MOB_TABLE_LOCK_SUFFIX = Bytes.toBytes(".mobLock");
+ public final static String EMPTY_STRING = "";
+ /**
+ * If the size of a mob file is less than this value, it's regarded as a small file and needs to
+ * be merged in mob compaction. The default value is 1280MB.
+ */
+ public static final String MOB_COMPACTION_MERGEABLE_THRESHOLD =
+ "hbase.mob.compaction.mergeable.threshold";
+ public static final long DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD = 10 * 128 * 1024 * 1024;
+ public static final String MOB_DELFILE_MAX_COUNT = "hbase.mob.delfile.max.count";
+ public static final int DEFAULT_MOB_DELFILE_MAX_COUNT = 3;
+ /**
+ * The max number of the mob files that is allowed in a batch of the mob compaction. The mob
+ * compaction merges the small mob files to bigger ones. If the number of the small files is very
+ * large, it could lead to a "too many opened file handlers" in the merge. And the merge has to be
+ * split into batches. This value limits the number of mob files that are selected in a batch of
+ * the mob compaction. The default value is 100. Default is 0 - means no limit - all regions of a
+ * MOB table will be compacted at once
+ */
+ public static final String MOB_COMPACTION_BATCH_SIZE = "hbase.mob.compaction.batch.size";
+ public static final int DEFAULT_MOB_COMPACTION_BATCH_SIZE = 100;
+ public static final String MOB_COMPACTION_THREADS_MAX = "hbase.mob.compaction.threads.max";
+ public static final int DEFAULT_MOB_COMPACTION_THREADS_MAX = 1;
+
private MobConstants() {
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java
index 8594474..c7d713d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.yetus.audience.InterfaceAudience;
@@ -58,16 +59,15 @@ import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesti
* The class MobFileCleanerChore for running cleaner regularly to remove the expired
* and obsolete (files which have no active references to) mob files.
*/
-@SuppressWarnings("deprecation")
@InterfaceAudience.Private
public class MobFileCleanerChore extends ScheduledChore {
private static final Logger LOG = LoggerFactory.getLogger(MobFileCleanerChore.class);
private final HMaster master;
private ExpiredMobFileCleaner cleaner;
-
+
static {
- Configuration.addDeprecation(MobConstants.DEPRECATED_MOB_CLEANER_PERIOD,
+ Configuration.addDeprecation(MobConstants.DEPRECATED_MOB_CLEANER_PERIOD,
MobConstants.MOB_CLEANER_PERIOD);
}
@@ -126,8 +126,9 @@ public class MobFileCleanerChore extends ScheduledChore {
try {
cleaner.cleanExpiredMobFiles(htd.getTableName().getNameAsString(), hcd);
} catch (IOException e) {
- LOG.error("Failed to clean the expired mob files table=" + htd.getTableName()
- + " family=" + hcd.getNameAsString(), e);
+ String errMsg = String.format("Failed to clean the expired mob files table=%s" +
+ " family=%s", htd.getTableName().getNameAsString(), hcd.getNameAsString());
+ LOG.error(errMsg, e);
}
}
}
@@ -137,7 +138,9 @@ public class MobFileCleanerChore extends ScheduledChore {
cleanupObsoleteMobFiles(master.getConfiguration(), htd.getTableName());
LOG.info("Cleaning obsolete MOB files finished for table={}", htd.getTableName());
} catch (IOException e) {
- LOG.error("Failed to clean the obsolete mob files for table=" + htd.getTableName(), e);
+ String errMsg =
+ String.format("Failed to clean the obsolete mob files for table=",htd.getTableName());
+ LOG.error(errMsg, e);
}
}
}
@@ -159,8 +162,8 @@ public class MobFileCleanerChore extends ScheduledChore {
// So, if MOB file creation time is greater than this maxTimeToArchive,
// this will be skipped and won't be archived.
long maxCreationTimeToArchive = EnvironmentEdgeManager.currentTime() - minAgeToArchive;
- LOG.info("Only MOB files whose creation time older than {} will be archived",
- maxCreationTimeToArchive);
+ LOG.info("Only MOB files whose creation time older than {} will be archived, table={}",
+ maxCreationTimeToArchive, table);
try (final Connection conn = ConnectionFactory.createConnection(conf);
final Admin admin = conn.getAdmin();) {
TableDescriptor htd = admin.getDescriptor(table);
@@ -178,6 +181,7 @@ public class MobFileCleanerChore extends ScheduledChore {
Set<String> allActiveMobFileName = new HashSet<String>();
FileSystem fs = FileSystem.get(conf);
for (Path regionPath : regionDirs) {
+ region:
for (ColumnFamilyDescriptor hcd : list) {
String family = hcd.getNameAsString();
Path storePath = new Path(regionPath, family);
@@ -185,9 +189,9 @@ public class MobFileCleanerChore extends ScheduledChore {
Set<String> regionMobs = new HashSet<String>();
while (!succeed) {
if (!fs.exists(storePath)) {
- LOG.warn("Directory {} was deleted during cleaner procedure execution,"
- +" skipping.", storePath);
- break;
+ LOG.warn("Directory {} was deleted during cleaner procedure execution,"
+ +" skipping region {}", storePath, regionPath.getName());
+ continue region;
}
RemoteIterator<LocatedFileStatus> rit = fs.listLocatedStatus(storePath);
List<Path> storeFiles = new ArrayList<Path>();
@@ -199,9 +203,11 @@ public class MobFileCleanerChore extends ScheduledChore {
}
}
LOG.info("Found {} store files in: {}", storeFiles.size(), storePath);
+ Path currentPath = null;
try {
for (Path pp : storeFiles) {
- LOG.debug("Store file: {}", pp);
+ currentPath = pp;
+ LOG.trace("Store file: {}", pp);
HStoreFile sf =
new HStoreFile(fs, pp, conf, CacheConfig.DISABLED, BloomType.NONE, true);
sf.initReader();
@@ -224,7 +230,7 @@ public class MobFileCleanerChore extends ScheduledChore {
// in this store file, but the file was created by new MOB code
String[] mobs = new String(mobRefData).split(",");
if (LOG.isTraceEnabled()) {
- LOG.debug("Found: {} mob references: {}", mobs.length, Arrays.toString(mobs));
+ LOG.trace("Found: {} mob references: {}", mobs.length, Arrays.toString(mobs));
} else {
LOG.debug("Found: {} mob references", mobs.length);
}
@@ -232,21 +238,30 @@ public class MobFileCleanerChore extends ScheduledChore {
}
}
} catch (FileNotFoundException e) {
- LOG.warn("Starting MOB cleaning cycle from the beginning due to error", e);
+ String warnMsg =
+ String.format("Missing file:%s" +
+ " Starting MOB cleaning cycle from the beginning due to error",
+ currentPath);
+ LOG.warn(warnMsg, e);
continue;
}
succeed = true;
}
- // Add MOB refs for current region/family
+ if (!succeed) {
+ continue; // continue next cf
+ }
+ // Add MOB references for current region/family
allActiveMobFileName.addAll(regionMobs);
} // END column families
} // END regions
// Check if number of MOB files too big (over 1M)
if (allActiveMobFileName.size() > 1000000) {
- LOG.warn("Found too many active MOB files: {}, this may result in high memory pressure.",
- allActiveMobFileName.size());
+ LOG.warn("Found too many active MOB files: {}, table={}, "+
+ "this may result in high memory pressure.",
+ allActiveMobFileName.size(), table);
}
- LOG.debug("Found: {} active mob refs", allActiveMobFileName.size());
+ LOG.debug("Found: {} active mob refs for table={}",
+ allActiveMobFileName.size(), table);
allActiveMobFileName.stream().forEach(LOG::trace);
// Now scan MOB directories and find MOB files with no references to them
@@ -263,20 +278,21 @@ public class MobFileCleanerChore extends ScheduledChore {
// fresh, skip it in this case
long creationTime = fs.getFileStatus(p).getModificationTime();
if (creationTime < maxCreationTimeToArchive) {
- LOG.debug("Archiving MOB file {} creation time={}", p,
+ LOG.trace("Archiving MOB file {} creation time={}", p,
(fs.getFileStatus(p).getModificationTime()));
toArchive.add(p);
} else {
- LOG.debug("Skipping fresh file: {}", p);
+ LOG.trace("Skipping fresh file: {}. Creation time={}", p,
+ fs.getFileStatus(p).getModificationTime());
}
} else {
- LOG.debug("Keeping active MOB file: {}", p);
+ LOG.trace("Keeping active MOB file: {}", p);
}
}
- LOG.info(" MOB Cleaner found {} files to archive for table={} family={}",
+ LOG.info(" MOB Cleaner found {} files to archive for table={} family={}",
toArchive.size(), table, family);
- removeMobFiles(conf, table, family.getBytes(), toArchive);
- LOG.info(" MOB Cleaner archived {} files, table={} family={}",
+ archiveMobFiles(conf, table, family.getBytes(), toArchive);
+ LOG.info(" MOB Cleaner archived {} files, table={} family={}",
toArchive.size(), table, family);
}
}
@@ -290,20 +306,21 @@ public class MobFileCleanerChore extends ScheduledChore {
* @param storeFiles The files to be archived.
* @throws IOException exception
*/
- public void removeMobFiles(Configuration conf, TableName tableName, byte[] family,
+ public void archiveMobFiles(Configuration conf, TableName tableName, byte[] family,
List<Path> storeFiles) throws IOException {
if (storeFiles.size() == 0) {
// nothing to remove
- LOG.debug("Skipping archiving old MOB files - no files found.");
+ LOG.debug("Skipping archiving old MOB files - no files found for table={} cf={}",
+ tableName, Bytes.toString(family));
return;
}
Path mobTableDir = FSUtils.getTableDir(MobUtils.getMobHome(conf), tableName);
FileSystem fs = storeFiles.get(0).getFileSystem(conf);
for (Path p : storeFiles) {
- LOG.info("MOB Cleaner is archiving: {}", p);
- HFileArchiver.archiveStoreFile(conf, fs, MobUtils.getMobRegionInfo(tableName),
+ LOG.debug("MOB Cleaner is archiving: {}", p);
+ HFileArchiver.archiveStoreFile(conf, fs, MobUtils.getMobRegionInfo(tableName),
mobTableDir, family, p);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCompactionChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCompactionChore.java
index 2628013..82d4a82 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCompactionChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCompactionChore.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -96,11 +97,12 @@ public class MobFileCompactionChore extends ScheduledChore {
for (TableDescriptor htd : map.values()) {
if (!master.getTableStateManager().isTableState(htd.getTableName(),
TableState.State.ENABLED)) {
- LOG.debug("Skipping MOB compaction on table {} because it is not ENABLED",
+ LOG.info("Skipping MOB compaction on table {} because it is not ENABLED",
htd.getTableName());
continue;
} else {
- LOG.debug("Starting MOB compaction on table {}", htd.getTableName());
+ LOG.info("Starting MOB compaction on table {}, checking {} column families",
+ htd.getTableName(), htd.getColumnFamilyCount());
}
for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
try {
@@ -109,27 +111,33 @@ public class MobFileCompactionChore extends ScheduledChore {
master.reportMobCompactionStart(htd.getTableName());
reported = true;
}
- LOG.info(" Major compacting {} cf={}", htd.getTableName(), hcd.getNameAsString());
+ LOG.info("Major MOB compacting table={} cf={}", htd.getTableName(),
+ hcd.getNameAsString());
if (regionBatchSize == MobConstants.DEFAULT_MOB_MAJOR_COMPACTION_REGION_BATCH_SIZE) {
- LOG.debug("Batch compaction is disabled, {}=0", "hbase.mob.compaction.batch.size");
+ LOG.debug("Table={} cf ={}: batch MOB compaction is disabled, {}=0 -"+
+ " all regions will be "+ "compacted in parallel", htd.getTableName(),
+ hcd.getNameAsString(), "hbase.mob.compaction.batch.size");
admin.majorCompact(htd.getTableName(), hcd.getName());
} else {
- LOG.debug("Performing compaction in batches, {}={}",
- "hbase.mob.compaction.batch.size", regionBatchSize);
+ LOG.info("Table={} cf={}: performing MOB major compaction in batches "+
+ "'hbase.mob.compaction.batch.size'={}", htd.getTableName(),
+ hcd.getNameAsString(), regionBatchSize);
performMajorCompactionInBatches(admin, htd, hcd);
}
} else {
- LOG.debug("Skipping column family {} because it is not MOB-enabled",
- hcd.getNameAsString());
+ LOG.info("Skipping table={} column family={} because it is not MOB-enabled",
+ htd.getTableName(), hcd.getNameAsString());
}
} catch (IOException e) {
- LOG.error(
- "Failed to compact table=" + htd.getTableName() + " cf=" + hcd.getNameAsString(), e);
+ String errMsg = String.format("Failed to compact table=%s cf=%s",
+ htd.getTableName(), hcd.getNameAsString());
+ LOG.error(errMsg, e);
} catch (InterruptedException ee) {
Thread.currentThread().interrupt();
master.reportMobCompactionEnd(htd.getTableName());
- LOG.warn(
- "Failed to compact table=" + htd.getTableName() + " cf=" + hcd.getNameAsString(), ee);
+ String warnMsg = String.format("Failed to compact table=%s cf=%s",
+ htd.getTableName(), hcd.getNameAsString());
+ LOG.warn(warnMsg, ee);
// Quit the chore
return;
}
@@ -150,15 +158,17 @@ public class MobFileCompactionChore extends ScheduledChore {
List<RegionInfo> regions = admin.getRegions(htd.getTableName());
if (regions.size() <= this.regionBatchSize) {
- LOG.debug("Performing compaction in non-batched mode, regions={}, batch size={}",
- regions.size(), regionBatchSize);
+ LOG.debug(
+ "Table={} cf={} - performing major MOB compaction in non-batched mode,"
+ + "regions={}, batch size={}",
+ htd.getTableName(), hcd.getNameAsString(), regions.size(), regionBatchSize);
admin.majorCompact(htd.getTableName(), hcd.getName());
return;
}
// Shuffle list of regions in case if they come ordered by region server
Collections.shuffle(regions);
// Create first batch
- List<RegionInfo> toCompact = new ArrayList<RegionInfo>();
+ List<RegionInfo> toCompact = new ArrayList<RegionInfo>(regions.size());
for (int i = 0; i < this.regionBatchSize; i++) {
toCompact.add(regions.remove(0));
}
@@ -168,7 +178,8 @@ public class MobFileCompactionChore extends ScheduledChore {
startCompaction(admin, htd.getTableName(), ri, hcd.getName());
}
- List<RegionInfo> compacted = new ArrayList<RegionInfo>();
+ List<RegionInfo> compacted = new ArrayList<RegionInfo>(toCompact.size());
+ List<RegionInfo> failed = new ArrayList<RegionInfo>();
int totalCompacted = 0;
while (!toCompact.isEmpty()) {
// Check status of active compactions
@@ -176,14 +187,26 @@ public class MobFileCompactionChore extends ScheduledChore {
try {
if (admin.getCompactionStateForRegion(ri.getRegionName()) == CompactionState.NONE) {
totalCompacted++;
- LOG.info("Finished major compaction: table={} region={}, compacted regions={}",
- htd.getTableName(), ri.getRegionNameAsString(), totalCompacted);
+ LOG.info(
+ "Finished major MOB compaction: table={} cf={} region={}," + " compacted regions={}",
+ htd.getTableName(), hcd.getNameAsString(), ri.getRegionNameAsString(),
+ totalCompacted);
compacted.add(ri);
}
} catch (IOException e) {
- LOG.warn("Could not get compaction state for region {}", ri.getEncodedName());
+ LOG.error("Could not get compaction state for table={} cf={} region={}, compaction will"+
+ " aborted for the region.",
+ htd.getTableName(), hcd.getNameAsString(), ri.getEncodedName());
+ LOG.error("Because of:", e);
+ failed.add(ri);
}
}
+ // Remove failed regions to avoid
+ // endless compaction loop
+ for(RegionInfo ri: failed) {
+ toCompact.remove(ri);
+ }
+ failed.clear();
// Update batch: remove compacted regions and add new ones
for (RegionInfo ri : compacted) {
toCompact.remove(ri);
@@ -194,19 +217,24 @@ public class MobFileCompactionChore extends ScheduledChore {
}
}
compacted.clear();
- LOG.debug("Wait for 10 sec, toCompact size={} regions left={} compacted so far={}",
- toCompact.size(), regions.size(), totalCompacted);
+
+ LOG.debug(
+ "Table={} cf={}. Wait for 10 sec, toCompact size={} regions left={}"
+ + " compacted so far={}", htd.getTableName(), hcd.getNameAsString(), toCompact.size(),
+ regions.size(), totalCompacted);
Thread.sleep(10000);
}
- LOG.info("Finished major compacting {}. cf={}", htd.getTableName(), hcd.getNameAsString());
+ LOG.info("Finished major MOB compacting table={}. cf={}", htd.getTableName(),
+ hcd.getNameAsString());
}
private void startCompaction(Admin admin, TableName table, RegionInfo region, byte[] cf)
throws IOException, InterruptedException {
- LOG.info("Started major compaction: table={} region={}", table, region.getRegionNameAsString());
- admin.majorCompactRegion(region.getRegionName());
+ LOG.info("Started major compaction: table={} cf={} region={}", table,
+ Bytes.toString(cf), region.getRegionNameAsString());
+ admin.majorCompactRegion(region.getRegionName(), cf);
// Wait until it really starts
// but with finite timeout
long waitTime = 300000; // 5 min
@@ -215,8 +243,9 @@ public class MobFileCompactionChore extends ScheduledChore {
// Is 1 second too aggressive?
Thread.sleep(1000);
if (EnvironmentEdgeManager.currentTime() - startTime > waitTime) {
- LOG.warn("Waited for {} ms to start major compaction on table: {} region: {}. Aborted.",
- waitTime, table.getNameAsString(), region.getRegionNameAsString());
+ LOG.warn("Waited for {} ms to start major MOB compaction on table={} cf={} region={}."+
+ " Stopped waiting for request confirmation. This is not an ERROR, continue next region."
+ , waitTime, table.getNameAsString(), Bytes.toString(cf),region.getRegionNameAsString());
break;
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java
index 6adb4b5..86fffbf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java
@@ -30,7 +30,7 @@ import org.apache.yetus.audience.InterfaceAudience;
*/
@InterfaceAudience.Private
public class MobStoreEngine extends DefaultStoreEngine {
- public final static String DEFAULT_MOB_COMPACTOR_CLASS_KEY = "hbase.mob.default.compactor";
+ public final static String MOB_COMPACTOR_CLASS_KEY = "hbase.hstore.mobengine.compactor.class";
@Override
protected void createStoreFlusher(Configuration conf, HStore store) throws IOException {
// When using MOB, we use DefaultMobStoreFlusher always
@@ -45,7 +45,7 @@ public class MobStoreEngine extends DefaultStoreEngine {
@Override
protected void createCompactor(Configuration conf, HStore store) throws IOException {
String className =
- conf.get(DEFAULT_MOB_COMPACTOR_CLASS_KEY, DefaultMobStoreCompactor.class.getName());
+ conf.get(MOB_COMPACTOR_CLASS_KEY, DefaultMobStoreCompactor.class.getName());
try {
compactor = ReflectionUtils.instantiateWithCustomCtor(className,
new Class[] { Configuration.class, HStore.class }, new Object[] { conf, store });
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index 0aed8b2..3a6a55d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -79,12 +79,12 @@ public final class MobUtils {
public static final String SEP = "_";
private static final ThreadLocal<SimpleDateFormat> LOCAL_FORMAT =
- new ThreadLocal<SimpleDateFormat>() {
- @Override
- protected SimpleDateFormat initialValue() {
- return new SimpleDateFormat("yyyyMMdd");
- }
- };
+ new ThreadLocal<SimpleDateFormat>() {
+ @Override
+ protected SimpleDateFormat initialValue() {
+ return new SimpleDateFormat("yyyyMMdd");
+ }
+ };
/**
* Private constructor to keep this class from being instantiated.
@@ -665,19 +665,6 @@ public final class MobUtils {
}
/**
- * Gets encoded region name from a MOB file name
- * @param mobFileName MOB file name
- * @return encoded region name or null
- */
- public static String getEncodedRegionName(String mobFileName) {
- int index = mobFileName.lastIndexOf(MobFileName.REGION_SEP);
- if (index < 0) {
- return null;
- }
- return mobFileName.substring(index + 1);
- }
-
- /**
* Get list of referenced MOB files from a given collection of store files
* @param storeFiles store files
* @param mobDir MOB file directory
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
index 8f7b690..d7f2ba3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
@@ -243,8 +243,7 @@ public class HMobStore extends HStore {
}
Path dstPath = new Path(targetPath, sourceFile.getName());
validateMobFile(sourceFile);
- String msg = " FLUSH Renaming flushed file from " + sourceFile + " to " + dstPath;
- LOG.info(msg);
+ LOG.info(" FLUSH Renaming flushed file from {} to {}", sourceFile, dstPath);
Path parent = dstPath.getParent();
if (!region.getFilesystem().exists(parent)) {
region.getFilesystem().mkdirs(parent);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 6a92be4..e1c55ec 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1901,7 +1901,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
}
@VisibleForTesting
- void setTableDescriptor(TableDescriptor desc) {
+ public void setTableDescriptor(TableDescriptor desc) {
htableDescriptor = desc;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
index c715d6b..04adeb5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
@@ -61,19 +61,10 @@ public class StoreFileInfo {
/** Regex that will work for hfiles */
private static final Pattern HFILE_NAME_PATTERN = Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
- /**
- * A non-capture group, for del files, so that this can be embedded. A del file has (_del) as
- * suffix.
- */
- public static final String DELFILE_NAME_REGEX = "[0-9a-f]+(?:_del)";
-
- /** Regex that will work for del files */
- private static final Pattern DELFILE_NAME_PATTERN =
- Pattern.compile("^(" + DELFILE_NAME_REGEX + ")");
/**
* Regex that will work for straight reference names ({@code <hfile>.<parentEncRegion>}) and
- * hfilelink reference names ({@code
+ * hfilelink reference names ({@code
* <table>
* =<region>-<hfile>.<parentEncRegion>}) If reference, then the regex has more than just one
* group. Group 1, hfile/hfilelink pattern, is this file's id. Group 2 '(.+)' is the reference's
@@ -249,8 +240,8 @@ public class StoreFileInfo {
this.coprocessorHost = coprocessorHost;
}
- /*
- * @return the Reference object associated to this StoreFileInfo.
+ /**
+ * @return the Reference object associated to this StoreFileInfo.
* null if the StoreFile is not a
* reference.
*/
@@ -438,6 +429,11 @@ public class StoreFileInfo {
return m.matches() && m.groupCount() > 0;
}
+ /**
+ * Checks if the file is a MOB file
+ * @param path path to a file
+ * @return true, if - yes, false otherwise
+ */
public static boolean isMobFile(final Path path) {
String fileName = path.getName();
String[] parts = fileName.split(MobUtils.SEP);
@@ -449,6 +445,12 @@ public class StoreFileInfo {
return m.matches() && mm.matches();
}
+ /**
+ * Checks if the file is a MOB reference file,
+ * created by snapshot
+ * @param path path to a file
+ * @return true, if - yes, false otherwise
+ */
public static boolean isMobRefFile(final Path path) {
String fileName = path.getName();
int lastIndex = fileName.lastIndexOf(MobUtils.SEP);
@@ -463,22 +465,6 @@ public class StoreFileInfo {
return m.matches() && m.groupCount() > 1;
}
- /**
- * @param path Path to check.
- * @return True if the path has format of a del file.
- */
- public static boolean isDelFile(final Path path) {
- return isDelFile(path.getName());
- }
-
- /**
- * @param fileName Sting version of path to validate.
- * @return True if the file name has format of a del file.
- */
- public static boolean isDelFile(final String fileName) {
- Matcher m = DELFILE_NAME_PATTERN.matcher(fileName);
- return m.matches() && m.groupCount() > 0;
- }
/**
* @param path Path to check.
@@ -537,7 +523,9 @@ public class StoreFileInfo {
* @return <tt>true</tt> if the file could be a valid store file, <tt>false</tt> otherwise
*/
public static boolean validateStoreFileName(final String fileName) {
- if (HFileLink.isHFileLink(fileName) || isReference(fileName)) return (true);
+ if (HFileLink.isHFileLink(fileName) || isReference(fileName)) {
+ return true;
+ }
return !fileName.contains("-");
}
@@ -549,7 +537,9 @@ public class StoreFileInfo {
public static boolean isValid(final FileStatus fileStatus) throws IOException {
final Path p = fileStatus.getPath();
- if (fileStatus.isDirectory()) return false;
+ if (fileStatus.isDirectory()) {
+ return false;
+ }
// Check for empty hfile. Should never be the case but can happen
// after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646
@@ -595,25 +585,48 @@ public class StoreFileInfo {
@Override
public boolean equals(Object that) {
- if (this == that) return true;
- if (that == null) return false;
+ if (this == that) {
+ return true;
+ }
+ if (that == null) {
+ return false;
+ }
- if (!(that instanceof StoreFileInfo)) return false;
+ if (!(that instanceof StoreFileInfo)) {
+ return false;
+ }
StoreFileInfo o = (StoreFileInfo) that;
- if (initialPath != null && o.initialPath == null) return false;
- if (initialPath == null && o.initialPath != null) return false;
- if (initialPath != o.initialPath && initialPath != null && !initialPath.equals(o.initialPath))
+ if (initialPath != null && o.initialPath == null) {
return false;
-
- if (reference != null && o.reference == null) return false;
- if (reference == null && o.reference != null) return false;
- if (reference != o.reference && reference != null && !reference.equals(o.reference))
+ }
+ if (initialPath == null && o.initialPath != null) {
return false;
+ }
+ if (initialPath != o.initialPath && initialPath != null
+ && !initialPath.equals(o.initialPath)) {
+ return false;
+ }
+ if (reference != null && o.reference == null) {
+ return false;
+ }
+ if (reference == null && o.reference != null) {
+ return false;
+ }
+ if (reference != o.reference && reference != null
+ && !reference.equals(o.reference)) {
+ return false;
+ }
- if (link != null && o.link == null) return false;
- if (link == null && o.link != null) return false;
- if (link != o.link && link != null && !link.equals(o.link)) return false;
+ if (link != null && o.link == null) {
+ return false;
+ }
+ if (link == null && o.link != null) {
+ return false;
+ }
+ if (link != o.link && link != null && !link.equals(o.link)) {
+ return false;
+ }
return true;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
index 2f6a8fb..3de97e8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
@@ -38,6 +38,8 @@ import java.util.UUID;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -251,15 +253,7 @@ public class StoreFileWriter implements CellSink, ShipperListener {
*/
public void appendMobMetadata(Set<String> mobRefSet) throws IOException {
if (mobRefSet != null && mobRefSet.size() > 0) {
- StringBuilder sb = new StringBuilder(2 * mobRefSet.size() - 1);
- String[] arr = new String[mobRefSet.size()];
- arr = mobRefSet.toArray(arr);
- for (int i = 0; i < arr.length; i++) {
- sb.append(arr[i]);
- if (i < arr.length - 1) {
- sb.append(",");
- }
- }
+ String sb = StringUtils.join(mobRefSet, ",");
byte[] bytes = Bytes.toBytes(sb.toString());
writer.appendFileInfo(MOB_FILE_REFS, bytes);
} else {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index 46e7a2e..10fac55 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -115,7 +115,7 @@ public abstract class Compactor<T extends CellSink> {
}
/** The sole reason this class exists is that java has no ref/out/pointer parameters. */
- public static class FileDetails {
+ protected static class FileDetails {
/** Maximum key count after compaction (for blooms) */
public long maxKeyCount = 0;
/** Earliest put timestamp if major compaction */
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/FaultyMobStoreCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/FaultyMobStoreCompactor.java
index 2003d63..1dd447c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/FaultyMobStoreCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/FaultyMobStoreCompactor.java
@@ -58,12 +58,12 @@ import org.slf4j.LoggerFactory;
* public void initConfiguration(Configuration conf){
* conf.set(MobStoreEngine.DEFAULT_MOB_COMPACTOR_CLASS_KEY,
FaultyMobStoreCompactor.class.getName());
- conf.setDouble("injected.fault.probability", 0.1);
+ conf.setDouble("hbase.mob.compaction.fault.probability", 0.1);
* }
* }
* }</pre>
- * @see org.apache.hadoop.hbase.mob.TestMobCompaction on how to use and configure
- * this class.
+ * @see org.apache.hadoop.hbase.mob.MobStressToolRunner on how to use and configure
+ * this class.
*
*/
@InterfaceAudience.Private
@@ -81,7 +81,7 @@ public class FaultyMobStoreCompactor extends DefaultMobStoreCompactor {
public FaultyMobStoreCompactor(Configuration conf, HStore store) {
super(conf, store);
- failureProb = conf.getDouble("injected.fault.probability", 0.1);
+ failureProb = conf.getDouble("hbase.mob.compaction.fault.probability", 0.1);
}
@Override
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressTool.java
index 81f5cf9..e9c6969 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressTool.java
@@ -66,9 +66,9 @@ public class MobStressTool extends AbstractHBaseTool {
}
}
- TestMobCompaction test = new TestMobCompaction();
- test.init(getConf(), numRowsToInsert);
- test.testMobCompaction();
+ MobStressToolRunner runner = new MobStressToolRunner();
+ runner.init(getConf(), numRowsToInsert);
+ runner.runStressTest();
return 0;
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressToolRunner.java
similarity index 83%
rename from hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompaction.java
rename to hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressToolRunner.java
index e8e7357..5295ec2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobStressToolRunner.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
@@ -42,16 +41,7 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
@@ -65,16 +55,13 @@ import org.slf4j.LoggerFactory;
c) Trigger archive cleaner (every 3 minutes)
4. Validate MOB data after complete data load.
+ This class is used by MobStressTool only. This is not a unit test
+
*/
@SuppressWarnings("deprecation")
-@Category(LargeTests.class)
-public class TestMobCompaction {
- private static final Logger LOG = LoggerFactory.getLogger(TestMobCompaction.class);
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestMobCompaction.class);
- @Rule
- public TestName testName = new TestName();
+public class MobStressToolRunner {
+ private static final Logger LOG = LoggerFactory.getLogger(MobStressToolRunner.class);
+
private HBaseTestingUtility HTU;
@@ -96,13 +83,14 @@ public class TestMobCompaction {
private static volatile boolean run = true;
- public TestMobCompaction() {
+ public MobStressToolRunner() {
}
public void init(Configuration conf, long numRows) throws IOException {
this.conf = conf;
this.count = numRows;
+ initConf();
printConf();
hdt = createTableDescriptor("testMobCompactTable");
Connection conn = ConnectionFactory.createConnection(this.conf);
@@ -121,7 +109,7 @@ public class TestMobCompaction {
}
private void printConf() {
- LOG.info("To run stress test, please change HBase configuration as following:");
+ LOG.info("Please ensure the following HBase configuration is set:");
LOG.info("hfile.format.version=3");
LOG.info("hbase.master.hfilecleaner.ttl=0");
LOG.info("hbase.hregion.max.filesize=200000000");
@@ -132,7 +120,7 @@ public class TestMobCompaction {
LOG.info("hbase.hstore.compaction.throughput.higher.bound=100000000");
LOG.info("hbase.master.mob.cleaner.period=0");
LOG.info("hbase.mob.default.compactor=org.apache.hadoop.hbase.mob.FaultyMobStoreCompactor");
- LOG.warn("injected.fault.probability=x, where x is between 0. and 1.");
+ LOG.warn("hbase.mob.compaction.fault.probability=x, where x is between 0. and 1.");
}
@@ -147,26 +135,6 @@ public class TestMobCompaction {
HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
}
- @Before
- public void setUp() throws Exception {
- HTU = new HBaseTestingUtility();
- hdt = HTU.createTableDescriptor("testMobCompactTable");
- conf = HTU.getConfiguration();
-
- initConf();
-
- // HTU.getConfiguration().setInt("hbase.mob.compaction.chore.period", 0);
- HTU.startMiniCluster();
- admin = HTU.getAdmin();
-
- hcd = new HColumnDescriptor(fam);
- hcd.setMobEnabled(true);
- hcd.setMobThreshold(mobLen);
- hcd.setMaxVersions(1);
- hdt.addFamily(hcd);
- table = HTU.createTable(hdt, null);
- }
-
private void initConf() {
conf.setInt("hfile.format.version", 3);
@@ -177,23 +145,18 @@ public class TestMobCompaction {
conf.setInt("hbase.hstore.blockingStoreFiles", 150);
conf.setInt("hbase.hstore.compaction.throughput.lower.bound", 52428800);
conf.setInt("hbase.hstore.compaction.throughput.higher.bound", 2 * 52428800);
- conf.setDouble("injected.fault.probability", failureProb);
+ conf.setDouble("hbase.mob.compaction.fault.probability", failureProb);
// conf.set(MobStoreEngine.DEFAULT_MOB_COMPACTOR_CLASS_KEY,
// FaultyMobStoreCompactor.class.getName());
conf.setLong(MobConstants.MOB_COMPACTION_CHORE_PERIOD, 0);
conf.setLong(MobConstants.MOB_CLEANER_PERIOD, 0);
conf.setLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, 120000);
conf.set(MobConstants.MOB_COMPACTION_TYPE_KEY,
- MobConstants.IO_OPTIMIZED_MOB_COMPACTION_TYPE);
+ MobConstants.OPTIMIZED_MOB_COMPACTION_TYPE);
conf.setLong(MobConstants.MOB_COMPACTION_MAX_FILE_SIZE_KEY, 1000000);
}
- @After
- public void tearDown() throws Exception {
- HTU.shutdownMiniCluster();
- }
-
class MajorCompaction implements Runnable {
@Override
@@ -222,7 +185,7 @@ public class TestMobCompaction {
Thread.sleep(130000);
} catch (Exception e) {
- e.printStackTrace();
+ LOG.error("CleanMobAndArchive", e);
}
}
}
@@ -266,9 +229,7 @@ public class TestMobCompaction {
}
}
- @Ignore
- @Test
- public void testMobCompaction() throws InterruptedException, IOException {
+ public void runStressTest() throws InterruptedException, IOException {
try {
@@ -350,16 +311,6 @@ public class TestMobCompaction {
counter++;
}
-// for (int i=0; i < count; i++) {
-// byte[] key = Bytes.toBytes(i);
-// Get get = new Get(key);
-// Result res = table.get(get);
-// assertTrue(Arrays.equals(res.getValue(fam, qualifier),
-// Bytes.add(key,mobVal)));
-// if (i % 1000 == 0) {
-// LOG.info("GET=" + i);
-// }
-// }
assertEquals(count, counter);
} catch (Exception e) {
e.printStackTrace();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionBase.java
index 50637da..2d9268c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionBase.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.mob;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.Arrays;
@@ -106,17 +107,12 @@ public abstract class TestMobCompactionBase {
hcd.setMobThreshold(mobLen);
hcd.setMaxVersions(1);
hdt.addFamily(hcd);
- byte[][] splitKeys = generateSplitKeys();
+ RegionSplitter.UniformSplit splitAlgo = new RegionSplitter.UniformSplit();
+ byte[][] splitKeys = splitAlgo.split(numRegions);
table = HTU.createTable(hdt, splitKeys);
}
- private byte[][] generateSplitKeys() {
- RegionSplitter.UniformSplit splitAlgo = new RegionSplitter.UniformSplit();
- return splitAlgo.split(numRegions);
- }
-
-
protected void initConf() {
conf.setInt("hfile.format.version", 3);
@@ -146,55 +142,49 @@ public abstract class TestMobCompactionBase {
LOG.info("Finished loading {} rows", num);
} catch (Exception e) {
LOG.error("MOB file compaction chore test FAILED", e);
- assertTrue(false);
+ fail("MOB file compaction chore test FAILED");
}
}
@After
public void tearDown() throws Exception {
+ admin.disableTable(hdt.getTableName());
+ admin.deleteTable(hdt.getTableName());
HTU.shutdownMiniCluster();
}
public void baseTestMobFileCompaction() throws InterruptedException, IOException {
- try {
-
- // Load and flush data 3 times
- loadData(rows);
- loadData(rows);
- loadData(rows);
- long num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
- assertEquals(numRegions * 3, num);
- // Major MOB compact
- mobCompact(admin, hdt, hcd);
- // wait until compaction is complete
- while (admin.getCompactionState(hdt.getTableName()) != CompactionState.NONE) {
- Thread.sleep(100);
- }
-
- num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
- assertEquals(numRegions * 4, num);
- // We have guarantee, that compcated file discharger will run during this pause
- // because it has interval less than this wait time
- LOG.info("Waiting for {}ms", minAgeToArchive + 1000);
-
- Thread.sleep(minAgeToArchive + 1000);
- LOG.info("Cleaning up MOB files");
- // Cleanup again
- cleanerChore.cleanupObsoleteMobFiles(conf, table.getName());
+ // Load and flush data 3 times
+ loadData(rows);
+ loadData(rows);
+ loadData(rows);
+ long num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
+ assertEquals(numRegions * 3, num);
+ // Major MOB compact
+ mobCompact(admin, hdt, hcd);
+ // wait until compaction is complete
+ while (admin.getCompactionState(hdt.getTableName()) != CompactionState.NONE) {
+ Thread.sleep(100);
+ }
- num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
- assertEquals(numRegions, num);
+ num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
+ assertEquals(numRegions * 4, num);
+ // We have guarantee, that compacted file discharger will run during this pause
+ // because it has interval less than this wait time
+ LOG.info("Waiting for {}ms", minAgeToArchive + 1000);
- long scanned = scanTable();
- assertEquals(3 * rows, scanned);
+ Thread.sleep(minAgeToArchive + 1000);
+ LOG.info("Cleaning up MOB files");
+ // Cleanup again
+ cleanerChore.cleanupObsoleteMobFiles(conf, table.getName());
- } finally {
+ num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
+ assertEquals(numRegions, num);
- admin.disableTable(hdt.getTableName());
- admin.deleteTable(hdt.getTableName());
- }
+ long scanned = scanTable();
+ assertEquals(3 * rows, scanned);
}
@@ -228,10 +218,9 @@ public abstract class TestMobCompactionBase {
}
return counter;
} catch (Exception e) {
- e.printStackTrace();
- LOG.error("MOB file compaction test FAILED");
+ LOG.error("MOB file compaction test FAILED", e);
if (HTU != null) {
- assertTrue(false);
+ fail(e.getMessage());
} else {
System.exit(-1);
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionOptMode.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionOptMode.java
index 4b7bea2..4508a38 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionOptMode.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionOptMode.java
@@ -66,7 +66,7 @@ public class TestMobCompactionOptMode extends TestMobCompactionBase{
protected void initConf() {
super.initConf();
conf.set(MobConstants.MOB_COMPACTION_TYPE_KEY,
- MobConstants.IO_OPTIMIZED_MOB_COMPACTION_TYPE);
+ MobConstants.OPTIMIZED_MOB_COMPACTION_TYPE);
conf.setLong(MobConstants.MOB_COMPACTION_MAX_FILE_SIZE_KEY, 1000000);
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionOptRegionBatchMode.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionOptRegionBatchMode.java
index 7496f8c..d354ab0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionOptRegionBatchMode.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobCompactionOptRegionBatchMode.java
@@ -76,7 +76,7 @@ public class TestMobCompactionOptRegionBatchMode extends TestMobCompactionBase{
super.initConf();
conf.setInt(MobConstants.MOB_MAJOR_COMPACTION_REGION_BATCH_SIZE, batchSize);
conf.set(MobConstants.MOB_COMPACTION_TYPE_KEY,
- MobConstants.IO_OPTIMIZED_MOB_COMPACTION_TYPE);
+ MobConstants.OPTIMIZED_MOB_COMPACTION_TYPE);
conf.setLong(MobConstants.MOB_COMPACTION_MAX_FILE_SIZE_KEY, 1000000);
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TesMobFileCleanerChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java
similarity index 82%
rename from hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TesMobFileCleanerChore.java
rename to hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java
index f13b824..21d0a64 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TesMobFileCleanerChore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCleanerChore.java
@@ -63,11 +63,11 @@ import org.slf4j.LoggerFactory;
*/
@SuppressWarnings("deprecation")
@Category(MediumTests.class)
-public class TesMobFileCleanerChore {
- private static final Logger LOG = LoggerFactory.getLogger(TesMobFileCleanerChore.class);
+public class TestMobFileCleanerChore {
+ private static final Logger LOG = LoggerFactory.getLogger(TestMobFileCleanerChore.class);
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TesMobFileCleanerChore.class);
+ HBaseClassTestRule.forClass(TestMobFileCleanerChore.class);
@Rule
public TestName testName = new TestName();
@@ -88,7 +88,7 @@ public class TesMobFileCleanerChore {
private MobFileCleanerChore chore;
private long minAgeToArchive = 10000;
- public TesMobFileCleanerChore() {
+ public TestMobFileCleanerChore() {
}
@@ -150,48 +150,43 @@ public class TesMobFileCleanerChore {
@After
public void tearDown() throws Exception {
+ admin.disableTable(hdt.getTableName());
+ admin.deleteTable(hdt.getTableName());
HTU.shutdownMiniCluster();
}
@Test
public void testMobFileCleanerChore() throws InterruptedException, IOException {
- try {
-
- loadData(0, 10);
- loadData(10, 10);
- loadData(20, 10);
- long num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
- assertEquals(3, num);
- // Major compact
- admin.majorCompact(hdt.getTableName(), fam);
- // wait until compaction is complete
- while (admin.getCompactionState(hdt.getTableName()) != CompactionState.NONE) {
- Thread.sleep(100);
- }
-
- num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
- assertEquals(4, num);
- // We have guarantee, that compcated file discharger will run during this pause
- // because it has interval less than this wait time
- LOG.info("Waiting for {}ms", minAgeToArchive + 1000);
+ loadData(0, 10);
+ loadData(10, 10);
+ loadData(20, 10);
+ long num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
+ assertEquals(3, num);
+ // Major compact
+ admin.majorCompact(hdt.getTableName(), fam);
+ // wait until compaction is complete
+ while (admin.getCompactionState(hdt.getTableName()) != CompactionState.NONE) {
+ Thread.sleep(100);
+ }
- Thread.sleep(minAgeToArchive + 1000);
- LOG.info("Cleaning up MOB files");
- // Cleanup again
- chore.cleanupObsoleteMobFiles(conf, table.getName());
+ num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
+ assertEquals(4, num);
+ // We have guarantee, that compcated file discharger will run during this pause
+ // because it has interval less than this wait time
+ LOG.info("Waiting for {}ms", minAgeToArchive + 1000);
- num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
- assertEquals(1, num);
+ Thread.sleep(minAgeToArchive + 1000);
+ LOG.info("Cleaning up MOB files");
+ // Cleanup again
+ chore.cleanupObsoleteMobFiles(conf, table.getName());
- long scanned = scanTable();
- assertEquals(30, scanned);
+ num = getNumberOfMobFiles(conf, table.getName(), new String(fam));
+ assertEquals(1, num);
- } finally {
+ long scanned = scanTable();
+ assertEquals(30, scanned);
- admin.disableTable(hdt.getTableName());
- admin.deleteTable(hdt.getTableName());
- }
LOG.info("MOB Stress Test finished OK");
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobStoreCompaction.java
similarity index 97%
rename from hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
rename to hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobStoreCompaction.java
index 6b252c6..610ccb6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobStoreCompaction.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hbase.regionserver;
+package org.apache.hadoop.hbase.mob;
import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
@@ -62,6 +62,12 @@ import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobFileCache;
import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HStore;
+import org.apache.hadoop.hbase.regionserver.HStoreFile;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.RegionAsTable;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
@@ -161,9 +167,6 @@ public class TestMobStoreCompaction {
/**
* During compaction, the mob threshold size is changed.
- * The test is no longer valid. Major MOB compaction must be triggered by User
- * HRegion does not provide public API to trigger major-compaction by User
- * This test will move to mob sub-package.
*/
@Test
public void testLargerValue() throws Exception {
@@ -184,21 +187,21 @@ public class TestMobStoreCompaction {
// Change the threshold larger than the data size
setMobThreshold(region, COLUMN_FAMILY, 500);
region.initialize();
-
+
List<HStore> stores = region.getStores();
for (HStore store: stores) {
// Force major compaction
store.triggerMajorCompaction();
- Optional<CompactionContext> context =
- store.requestCompaction(HStore.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY,
+ Optional<CompactionContext> context =
+ store.requestCompaction(HStore.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY,
User.getCurrent());
if (!context.isPresent()) {
continue;
}
- region.compact(context.get(), store,
+ region.compact(context.get(), store,
NoLimitThroughputController.INSTANCE, User.getCurrent());
}
-
+
assertEquals("After compaction: store files", 1, countStoreFiles());
assertEquals("After compaction: mob file count", compactionThreshold, countMobFiles());
assertEquals("After compaction: referenced mob file count", 0, countReferencedMobFiles());
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobStoreScanner.java
similarity index 99%
rename from hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
rename to hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobStoreScanner.java
index faae36a..fcafb98 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobStoreScanner.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hbase.regionserver;
+package org.apache.hadoop.hbase.mob;
import java.io.IOException;
import java.util.List;
@@ -44,8 +44,8 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.hfile.CorruptHFileException;
import org.apache.hadoop.hbase.io.hfile.TestHFile;
import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobTestUtil;
import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java
index 43e3e40..de47d9f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java
@@ -29,8 +29,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -47,7 +45,6 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
@@ -74,7 +71,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index cfed256..833e78b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -505,13 +505,13 @@ public class TestRegionServerMetrics {
for (HStore store: stores) {
// Force major compaction
store.triggerMajorCompaction();
- Optional<CompactionContext> context =
- store.requestCompaction(HStore.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY,
+ Optional<CompactionContext> context =
+ store.requestCompaction(HStore.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY,
User.getCurrent());
if (!context.isPresent()) {
continue;
}
- region.compact(context.get(), store,
+ region.compact(context.get(), store,
NoLimitThroughputController.INSTANCE, User.getCurrent());
}
metricsRegionServer.getRegionServerWrapper().forceRecompute();