You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ta...@apache.org on 2020/12/21 19:08:36 UTC
[hbase] 01/01: HBASE-25249 Adding HStoreContext
This is an automated email from the ASF dual-hosted git repository.
taklwu pushed a commit to branch HBASE-25249
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit e5e21fa09a6869fc3de3179ec1dee077e0019a9a
Author: Abhishek Khanna <ak...@amazon.com>
AuthorDate: Mon Aug 3 23:47:07 2020 -0700
HBASE-25249 Adding HStoreContext
Adding HStoreContext which contains the metadata about the HStore. This
meta data can be used across the HFileWriter/Readers and other HStore
consumers without the need of passing around the complete store and
exposing its internals.
---
.../hadoop/hbase/mapreduce/HFileOutputFormat2.java | 5 +-
.../java/org/apache/hadoop/hbase/mob/MobUtils.java | 5 +-
.../hadoop/hbase/regionserver/HMobStore.java | 31 +--
.../apache/hadoop/hbase/regionserver/HStore.java | 241 +++++++++++----------
.../hadoop/hbase/regionserver/HStoreContext.java | 174 +++++++++++++++
.../hadoop/hbase/regionserver/StoreUtils.java | 26 +++
.../hadoop/hbase/tool/BulkLoadHFilesTool.java | 5 +-
.../hadoop/hbase/tool/LoadIncrementalHFiles.java | 0
.../wal/BoundedRecoveredHFilesOutputSink.java | 6 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 2 +-
.../regionserver/TestSecureBulkLoadManager.java | 4 +-
11 files changed, 358 insertions(+), 141 deletions(-)
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
index 75b5246..ee6d533 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.hbase.io.hfile.HFileWriterImpl;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
+import org.apache.hadoop.hbase.regionserver.StoreUtils;
import org.apache.hadoop.hbase.util.BloomFilterUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
@@ -369,8 +370,8 @@ public class HFileOutputFormat2
encoding = encoding == null ? datablockEncodingMap.get(tableAndFamily) : encoding;
encoding = encoding == null ? DataBlockEncoding.NONE : encoding;
HFileContextBuilder contextBuilder = new HFileContextBuilder().withCompression(compression)
- .withDataBlockEncoding(encoding).withChecksumType(HStore.getChecksumType(conf))
- .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blockSize)
+ .withDataBlockEncoding(encoding).withChecksumType(StoreUtils.getChecksumType(conf))
+ .withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf)).withBlockSize(blockSize)
.withColumnFamily(family).withTableName(tableName);
if (HFile.getFormatVersion(conf) >= HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index acc8f74..4ef1b90 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
+import org.apache.hadoop.hbase.regionserver.StoreUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.CommonFSUtils;
@@ -540,8 +541,8 @@ public final class MobUtils {
Compression.Algorithm compression, CacheConfig cacheConfig, Encryption.Context cryptoContext,
boolean isCompaction) throws IOException {
return createWriter(conf, fs, family, new Path(basePath, mobFileName.getFileName()),
- maxKeyCount, compression, cacheConfig, cryptoContext, HStore.getChecksumType(conf),
- HStore.getBytesPerChecksum(conf), family.getBlocksize(), BloomType.NONE, isCompaction);
+ maxKeyCount, compression, cacheConfig, cryptoContext, StoreUtils.getChecksumType(conf),
+ StoreUtils.getBytesPerChecksum(conf), family.getBlocksize(), BloomType.NONE, isCompaction);
}
/**
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
index 5960b80..0f2b148 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
@@ -93,7 +93,6 @@ public class HMobStore extends HStore {
private AtomicLong mobFlushedCellsSize = new AtomicLong();
private AtomicLong mobScanCellsCount = new AtomicLong();
private AtomicLong mobScanCellsSize = new AtomicLong();
- private ColumnFamilyDescriptor family;
private Map<TableName, List<Path>> map = new ConcurrentHashMap<>();
private final IdLock keyLock = new IdLock();
// When we add a MOB reference cell to the HFile, we will add 2 tags along with it
@@ -107,16 +106,15 @@ public class HMobStore extends HStore {
public HMobStore(final HRegion region, final ColumnFamilyDescriptor family,
final Configuration confParam, boolean warmup) throws IOException {
super(region, family, confParam, warmup);
- this.family = family;
this.mobFileCache = region.getMobFileCache();
this.homePath = MobUtils.getMobHome(conf);
this.mobFamilyPath = MobUtils.getMobFamilyPath(conf, this.getTableName(),
- family.getNameAsString());
+ getColumnFamilyName());
List<Path> locations = new ArrayList<>(2);
locations.add(mobFamilyPath);
TableName tn = region.getTableDescriptor().getTableName();
locations.add(HFileArchiveUtil.getStoreArchivePath(conf, tn, MobUtils.getMobRegionInfo(tn)
- .getEncodedName(), family.getNameAsString()));
+ .getEncodedName(), getColumnFamilyName()));
map.put(tn, locations);
List<Tag> tags = new ArrayList<>(2);
tags.add(MobConstants.MOB_REF_TAG);
@@ -209,7 +207,7 @@ public class HMobStore extends HStore {
Compression.Algorithm compression, byte[] startKey,
boolean isCompaction) throws IOException {
MobFileName mobFileName = MobFileName.create(startKey, date, UUID.randomUUID()
- .toString().replaceAll("-", ""), region.getRegionInfo().getEncodedName());
+ .toString().replaceAll("-", ""), getHRegion().getRegionInfo().getEncodedName());
return createWriterInTmp(mobFileName, basePath, maxKeyCount, compression, isCompaction);
}
@@ -226,9 +224,10 @@ public class HMobStore extends HStore {
public StoreFileWriter createWriterInTmp(MobFileName mobFileName, Path basePath,
long maxKeyCount, Compression.Algorithm compression,
boolean isCompaction) throws IOException {
- return MobUtils.createWriter(conf, region.getFilesystem(), family,
- new Path(basePath, mobFileName.getFileName()), maxKeyCount, compression, cacheConf,
- cryptoContext, checksumType, bytesPerChecksum, blocksize, BloomType.NONE, isCompaction);
+ return MobUtils.createWriter(conf, getFileSystem(), getColumnFamilyDescriptor(),
+ new Path(basePath, mobFileName.getFileName()), maxKeyCount, compression, getCacheConfig(),
+ getCryptoContext(), StoreUtils.getChecksumType(conf), StoreUtils.getBytesPerChecksum(conf),
+ blocksize, BloomType.NONE, isCompaction);
}
/**
@@ -245,10 +244,10 @@ public class HMobStore extends HStore {
validateMobFile(sourceFile);
LOG.info(" FLUSH Renaming flushed file from {} to {}", sourceFile, dstPath);
Path parent = dstPath.getParent();
- if (!region.getFilesystem().exists(parent)) {
- region.getFilesystem().mkdirs(parent);
+ if (!getFileSystem().exists(parent)) {
+ getFileSystem().mkdirs(parent);
}
- if (!region.getFilesystem().rename(sourceFile, dstPath)) {
+ if (!getFileSystem().rename(sourceFile, dstPath)) {
throw new IOException("Failed rename of " + sourceFile + " to " + dstPath);
}
}
@@ -261,7 +260,7 @@ public class HMobStore extends HStore {
private void validateMobFile(Path path) throws IOException {
HStoreFile storeFile = null;
try {
- storeFile = new HStoreFile(region.getFilesystem(), path, conf, this.cacheConf,
+ storeFile = new HStoreFile(getFileSystem(), path, conf, getCacheConfig(),
BloomType.NONE, isPrimaryReplicaStore());
storeFile.initReader();
} catch (IOException e) {
@@ -352,9 +351,11 @@ public class HMobStore extends HStore {
locations = map.get(tableName);
if (locations == null) {
locations = new ArrayList<>(2);
- locations.add(MobUtils.getMobFamilyPath(conf, tableName, family.getNameAsString()));
+ locations.add(MobUtils.getMobFamilyPath(conf, tableName, getColumnFamilyDescriptor()
+ .getNameAsString()));
locations.add(HFileArchiveUtil.getStoreArchivePath(conf, tableName,
- MobUtils.getMobRegionInfo(tableName).getEncodedName(), family.getNameAsString()));
+ MobUtils.getMobRegionInfo(tableName).getEncodedName(), getColumnFamilyDescriptor()
+ .getNameAsString()));
map.put(tableName, locations);
}
} finally {
@@ -388,7 +389,7 @@ public class HMobStore extends HStore {
MobFile file = null;
Path path = new Path(location, fileName);
try {
- file = mobFileCache.openFile(fs, path, cacheConf);
+ file = mobFileCache.openFile(fs, path, getCacheConfig());
return readPt != -1 ? file.readCell(search, cacheMobBlocks, readPt)
: file.readCell(search, cacheMobBlocks);
} catch (IOException e) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 3a71c23..1cd5032 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -97,7 +97,6 @@ import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
import org.apache.hadoop.hbase.security.EncryptionUtil;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -157,11 +156,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
protected final MemStore memstore;
// This stores directory in the filesystem.
- protected final HRegion region;
- private final ColumnFamilyDescriptor family;
- private final HRegionFileSystem fs;
+ private final HRegion region;
protected Configuration conf;
- protected CacheConfig cacheConf;
private long lastCompactSize = 0;
volatile boolean forceMajor = false;
private AtomicLong storeSize = new AtomicLong();
@@ -218,13 +214,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
protected final int blocksize;
private HFileDataBlockEncoder dataBlockEncoder;
- /** Checksum configuration */
- protected ChecksumType checksumType;
- protected int bytesPerChecksum;
-
- // Comparing KeyValues
- protected final CellComparator comparator;
-
final StoreEngine<?, ?, ?, ?> storeEngine;
private static final AtomicBoolean offPeakCompactionTracker = new AtomicBoolean();
@@ -236,7 +225,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
private long blockingFileCount;
private int compactionCheckMultiplier;
- protected Encryption.Context cryptoContext = Encryption.Context.NONE;
private AtomicLong flushedCellsCount = new AtomicLong();
private AtomicLong compactedCellsCount = new AtomicLong();
@@ -246,6 +234,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
private AtomicLong compactedCellsSize = new AtomicLong();
private AtomicLong majorCompactedCellsSize = new AtomicLong();
+ private HStoreContext storeContext;
+
/**
* Constructor
* @param family HColumnDescriptor for this column
@@ -254,48 +244,47 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
protected HStore(final HRegion region, final ColumnFamilyDescriptor family,
final Configuration confParam, boolean warmup) throws IOException {
- this.fs = region.getRegionFileSystem();
-
- // Assemble the store's home directory and Ensure it exists.
- fs.createStoreDir(family.getNameAsString());
- this.region = region;
- this.family = family;
// 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
// CompoundConfiguration will look for keys in reverse order of addition, so we'd
// add global config first, then table and cf overrides, then cf metadata.
this.conf = new CompoundConfiguration()
- .add(confParam)
- .addBytesMap(region.getTableDescriptor().getValues())
- .addStringMap(family.getConfiguration())
- .addBytesMap(family.getValues());
- this.blocksize = family.getBlocksize();
+ .add(confParam)
+ .addBytesMap(region.getTableDescriptor().getValues())
+ .addStringMap(family.getConfiguration())
+ .addBytesMap(family.getValues());
+
+ this.region = region;
+ this.storeContext = initializeStoreContext(family);
+
+ // Assemble the store's home directory and Ensure it exists.
+ getRegionFileSystem().createStoreDir(getColumnFamilyName());
+
+ this.blocksize = getColumnFamilyDescriptor().getBlocksize();
// set block storage policy for store directory
- String policyName = family.getStoragePolicy();
+ String policyName = getColumnFamilyDescriptor().getStoragePolicy();
if (null == policyName) {
policyName = this.conf.get(BLOCK_STORAGE_POLICY_KEY, DEFAULT_BLOCK_STORAGE_POLICY);
}
- this.fs.setStoragePolicy(family.getNameAsString(), policyName.trim());
+ getRegionFileSystem().setStoragePolicy(getColumnFamilyName(), policyName.trim());
- this.dataBlockEncoder = new HFileDataBlockEncoderImpl(family.getDataBlockEncoding());
+ this.dataBlockEncoder = new HFileDataBlockEncoderImpl(getColumnFamilyDescriptor()
+ .getDataBlockEncoding());
- this.comparator = region.getCellComparator();
// used by ScanQueryMatcher
long timeToPurgeDeletes =
Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
LOG.trace("Time to purge deletes set to {}ms in {}", timeToPurgeDeletes, this);
// Get TTL
- long ttl = determineTTLFromFamily(family);
+ long ttl = determineTTLFromFamily(getColumnFamilyDescriptor());
// Why not just pass a HColumnDescriptor in here altogether? Even if have
// to clone it?
- scanInfo = new ScanInfo(conf, family, ttl, timeToPurgeDeletes, this.comparator);
+ scanInfo = new ScanInfo(conf, getColumnFamilyDescriptor(), ttl, timeToPurgeDeletes,
+ getComparator());
this.memstore = getMemstore();
this.offPeakHours = OffPeakHours.getInstance(conf);
- // Setting up cache configuration for this family
- createCacheConf(family);
-
this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify", false);
this.blockingFileCount =
@@ -308,7 +297,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
this.compactionCheckMultiplier = DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER;
}
- this.storeEngine = createStoreEngine(this, this.conf, this.comparator);
+ this.storeEngine = createStoreEngine(this, this.conf, getComparator());
List<HStoreFile> hStoreFiles = loadStoreFiles(warmup);
// Move the storeSize calculation out of loadStoreFiles() method, because the secondary read
// replica's refreshStoreFiles() will also use loadStoreFiles() to refresh its store files and
@@ -318,10 +307,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
this.totalUncompressedBytes.addAndGet(getTotalUncompressedBytes(hStoreFiles));
this.storeEngine.getStoreFileManager().loadFiles(hStoreFiles);
- // Initialize checksum type from name. The names are CRC32, CRC32C, etc.
- this.checksumType = getChecksumType(conf);
- // Initialize bytes per checksum
- this.bytesPerChecksum = getBytesPerChecksum(conf);
flushRetriesNumber = conf.getInt(
"hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER);
pauseTime = conf.getInt(HConstants.HBASE_SERVER_PAUSE, HConstants.DEFAULT_HBASE_SERVER_PAUSE);
@@ -330,7 +315,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
"hbase.hstore.flush.retries.number must be > 0, not "
+ flushRetriesNumber);
}
- cryptoContext = EncryptionUtil.createEncryptionContext(conf, family);
int confPrintThreshold =
this.conf.getInt("hbase.region.store.parallel.put.print.threshold", 50);
@@ -347,6 +331,48 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
cacheOnWriteLogged = false;
}
+ private HStoreContext initializeStoreContext(ColumnFamilyDescriptor family) throws IOException {
+ return new HStoreContext.Builder()
+ .withBloomType(family.getBloomFilterType())
+ .withCacheConfig(createCacheConf(family))
+ .withCellComparator(region.getCellComparator())
+ .withColumnFamilyDescriptor(family)
+ .withCompactedFilesSupplier(this::getCompactedFiles)
+ .withRegionFileSystem(region.getRegionFileSystem())
+ .withDefaultHFileContext(getDefaultHFileContext(family))
+ .withFavoredNodesSupplier(this::getFavoredNodes)
+ .withFamilyStoreDirectoryPath(region.getRegionFileSystem()
+ .getStoreDir(family.getNameAsString()))
+ .withRegionCoprocessorHost(region.getCoprocessorHost())
+ .build();
+ }
+
+ private InetSocketAddress[] getFavoredNodes() {
+ InetSocketAddress[] favoredNodes = null;
+ if (region.getRegionServerServices() != null) {
+ favoredNodes = region.getRegionServerServices().getFavoredNodesForRegion(
+ region.getRegionInfo().getEncodedName());
+ }
+ return favoredNodes;
+ }
+
+ private HFileContext getDefaultHFileContext(ColumnFamilyDescriptor family) throws IOException {
+ HFileContext hFileContext = new HFileContextBuilder()
+ .withCompression(HFile.DEFAULT_COMPRESSION_ALGORITHM)
+ .withCompressTags(family.isCompressTags())
+ .withChecksumType(StoreUtils.getChecksumType(conf))
+ .withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf))
+ .withBlockSize(family.getBlocksize())
+ .withHBaseCheckSum(true)
+ .withDataBlockEncoding(family.getDataBlockEncoding())
+ .withEncryptionContext(EncryptionUtil.createEncryptionContext(conf, family))
+ .withColumnFamily(family.getName())
+ .withTableName(region.getTableDescriptor().getTableName().getName())
+ .withCellComparator(region.getCellComparator())
+ .build();
+ return hFileContext;
+ }
+
/**
* @return MemStore Instance to use in this store.
*/
@@ -358,7 +384,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
inMemoryCompaction = MemoryCompactionPolicy.valueOf(
conf.get("hbase.systemtables.compacting.memstore.type", "NONE"));
} else {
- inMemoryCompaction = family.getInMemoryCompaction();
+ inMemoryCompaction = getColumnFamilyDescriptor().getInMemoryCompaction();
}
if (inMemoryCompaction == null) {
inMemoryCompaction =
@@ -368,13 +394,13 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
switch (inMemoryCompaction) {
case NONE:
ms = ReflectionUtils.newInstance(DefaultMemStore.class,
- new Object[] { conf, this.comparator,
+ new Object[] { conf, getComparator(),
this.getHRegion().getRegionServicesForStores()});
break;
default:
Class<? extends CompactingMemStore> clz = conf.getClass(MEMSTORE_CLASS_NAME,
CompactingMemStore.class, CompactingMemStore.class);
- ms = ReflectionUtils.newInstance(clz, new Object[]{conf, this.comparator, this,
+ ms = ReflectionUtils.newInstance(clz, new Object[]{conf, getComparator(), this,
this.getHRegion().getRegionServicesForStores(), inMemoryCompaction});
}
return ms;
@@ -384,10 +410,11 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
* Creates the cache config.
* @param family The current column family.
*/
- protected void createCacheConf(final ColumnFamilyDescriptor family) {
- this.cacheConf = new CacheConfig(conf, family, region.getBlockCache(),
+ protected CacheConfig createCacheConf(final ColumnFamilyDescriptor family) {
+ CacheConfig cacheConf = new CacheConfig(conf, family, region.getBlockCache(),
region.getRegionServicesForStores().getByteBuffAllocator());
- LOG.info("Created cacheConfig: " + this.getCacheConfig() + " for " + this);
+ LOG.info("Created cacheConfig: " + cacheConf + " for " + this);
+ return cacheConf;
}
/**
@@ -400,7 +427,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
*/
protected StoreEngine<?, ?, ?, ?> createStoreEngine(HStore store, Configuration conf,
CellComparator kvComparator) throws IOException {
- return StoreEngine.create(store, conf, comparator);
+ return StoreEngine.create(store, conf, getComparator());
}
/**
@@ -423,7 +450,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
@Override
public String getColumnFamilyName() {
- return this.family.getNameAsString();
+ return this.storeContext.getFamily().getNameAsString();
}
@Override
@@ -433,11 +460,11 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
@Override
public FileSystem getFileSystem() {
- return this.fs.getFileSystem();
+ return storeContext.getRegionFileSystem().getFileSystem();
}
public HRegionFileSystem getRegionFileSystem() {
- return this.fs;
+ return storeContext.getRegionFileSystem();
}
/* Implementation of StoreConfigInformation */
@@ -474,33 +501,14 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
}
/* End implementation of StoreConfigInformation */
- /**
- * Returns the configured bytesPerChecksum value.
- * @param conf The configuration
- * @return The bytesPerChecksum that is set in the configuration
- */
- public static int getBytesPerChecksum(Configuration conf) {
- return conf.getInt(HConstants.BYTES_PER_CHECKSUM,
- HFile.DEFAULT_BYTES_PER_CHECKSUM);
- }
-
- /**
- * Returns the configured checksum algorithm.
- * @param conf The configuration
- * @return The checksum algorithm that is set in the configuration
- */
- public static ChecksumType getChecksumType(Configuration conf) {
- String checksumName = conf.get(HConstants.CHECKSUM_TYPE_NAME);
- if (checksumName == null) {
- return ChecksumType.getDefaultChecksumType();
- } else {
- return ChecksumType.nameToType(checksumName);
- }
- }
@Override
public ColumnFamilyDescriptor getColumnFamilyDescriptor() {
- return this.family;
+ return this.storeContext.getFamily();
+ }
+
+ public Encryption.Context getCryptoContext() {
+ return storeContext.getDefaultFileContext().getEncryptionContext();
}
@Override
@@ -559,7 +567,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
* from the given directory.
*/
private List<HStoreFile> loadStoreFiles(boolean warmup) throws IOException {
- Collection<StoreFileInfo> files = fs.getStoreFiles(getColumnFamilyName());
+ Collection<StoreFileInfo> files = getRegionFileSystem().getStoreFiles(getColumnFamilyName());
return openStoreFiles(files, warmup);
}
@@ -610,7 +618,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
if (ioe != null) {
// close StoreFile readers
boolean evictOnClose =
- cacheConf != null? cacheConf.shouldEvictOnClose(): true;
+ getCacheConfig() != null? getCacheConfig().shouldEvictOnClose(): true;
for (HStoreFile file : results) {
try {
if (file != null) {
@@ -638,7 +646,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
results.removeAll(filesToRemove);
if (!filesToRemove.isEmpty() && this.isPrimaryReplicaStore()) {
LOG.debug("Moving the files {} to archive", filesToRemove);
- this.fs.removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(), filesToRemove);
+ getRegionFileSystem().removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(),
+ filesToRemove);
}
}
@@ -647,7 +656,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
@Override
public void refreshStoreFiles() throws IOException {
- Collection<StoreFileInfo> newFiles = fs.getStoreFiles(getColumnFamilyName());
+ Collection<StoreFileInfo> newFiles = getRegionFileSystem().getStoreFiles(getColumnFamilyName());
refreshStoreFilesInternal(newFiles);
}
@@ -658,7 +667,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
public void refreshStoreFiles(Collection<String> newFiles) throws IOException {
List<StoreFileInfo> storeFiles = new ArrayList<>(newFiles.size());
for (String file : newFiles) {
- storeFiles.add(fs.getStoreFileInfo(getColumnFamilyName(), file));
+ storeFiles.add(getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file));
}
refreshStoreFilesInternal(storeFiles);
}
@@ -735,7 +744,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
private HStoreFile createStoreFileAndReader(StoreFileInfo info) throws IOException {
info.setRegionCoprocessorHost(this.region.getCoprocessorHost());
- HStoreFile storeFile = new HStoreFile(info, this.family.getBloomFilterType(), this.cacheConf);
+ HStoreFile storeFile = new HStoreFile(info, getColumnFamilyDescriptor().getBloomFilterType(),
+ getCacheConfig());
storeFile.initReader();
return storeFile;
}
@@ -818,7 +828,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
LOG.info("Validating hfile at " + srcPath + " for inclusion in " + this);
FileSystem srcFs = srcPath.getFileSystem(conf);
srcFs.access(srcPath, FsAction.READ_WRITE);
- reader = HFile.createReader(srcFs, srcPath, cacheConf, isPrimaryReplicaStore(), conf);
+ reader = HFile.createReader(srcFs, srcPath, getCacheConfig(), isPrimaryReplicaStore(), conf);
Optional<byte[]> firstKey = reader.getFirstRowKey();
Preconditions.checkState(firstKey.isPresent(), "First key can not be null");
@@ -855,7 +865,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
do {
Cell cell = scanner.getCell();
if (prevCell != null) {
- if (comparator.compareRows(prevCell, cell) > 0) {
+ if (getComparator().compareRows(prevCell, cell) > 0) {
throw new InvalidHFileException("Previous row is greater than"
+ " current row: path=" + srcPath + " previous="
+ CellUtil.getCellKeyAsString(prevCell) + " current="
@@ -892,13 +902,13 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
*/
public Pair<Path, Path> preBulkLoadHFile(String srcPathStr, long seqNum) throws IOException {
Path srcPath = new Path(srcPathStr);
- return fs.bulkLoadStoreFile(getColumnFamilyName(), srcPath, seqNum);
+ return getRegionFileSystem().bulkLoadStoreFile(getColumnFamilyName(), srcPath, seqNum);
}
public Path bulkLoadHFile(byte[] family, String srcPathStr, Path dstPath) throws IOException {
Path srcPath = new Path(srcPathStr);
try {
- fs.commitStoreFile(srcPath, dstPath);
+ getRegionFileSystem().commitStoreFile(srcPath, dstPath);
} finally {
if (this.getCoprocessorHost() != null) {
this.getCoprocessorHost().postCommitStoreFile(family, srcPath, dstPath);
@@ -964,8 +974,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
storeEngine.getStoreFileManager().clearCompactedFiles();
// clear the compacted files
if (CollectionUtils.isNotEmpty(compactedfiles)) {
- removeCompactedfiles(compactedfiles, cacheConf != null ?
- cacheConf.shouldEvictOnClose() : true);
+ removeCompactedfiles(compactedfiles, getCacheConfig() != null ?
+ getCacheConfig().shouldEvictOnClose() : true);
}
if (!result.isEmpty()) {
// initialize the thread pool for closing store files in parallel.
@@ -981,7 +991,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
@Override
public Void call() throws IOException {
boolean evictOnClose =
- cacheConf != null? cacheConf.shouldEvictOnClose(): true;
+ getCacheConfig() != null? getCacheConfig().shouldEvictOnClose(): true;
f.closeStoreFile(evictOnClose);
return null;
}
@@ -1092,7 +1102,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
FileSystem srcFs = path.getFileSystem(conf);
srcFs.access(path, FsAction.READ_WRITE);
try (HFile.Reader reader =
- HFile.createReader(srcFs, path, cacheConf, isPrimaryReplicaStore(), conf)) {
+ HFile.createReader(srcFs, path, getCacheConfig(), isPrimaryReplicaStore(), conf)) {
Optional<byte[]> firstKey = reader.getFirstRowKey();
Preconditions.checkState(firstKey.isPresent(), "First key can not be null");
Optional<Cell> lk = reader.getLastKey();
@@ -1104,7 +1114,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
}
}
- Path dstPath = fs.commitStoreFile(getColumnFamilyName(), path);
+ Path dstPath = getRegionFileSystem().commitStoreFile(getColumnFamilyName(), path);
HStoreFile sf = createStoreFileAndReader(dstPath);
StoreFileReader r = sf.getReader();
this.storeSize.addAndGet(r.length());
@@ -1129,7 +1139,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
private HStoreFile commitFile(Path path, long logCacheFlushId, MonitoredTask status)
throws IOException {
// Write-out finished successfully, move into the right spot
- Path dstPath = fs.commitStoreFile(getColumnFamilyName(), path);
+ Path dstPath = getRegionFileSystem().commitStoreFile(getColumnFamilyName(), path);
status.setStatus("Flushing " + this + ": reopening flushed file");
HStoreFile sf = createStoreFileAndReader(dstPath);
@@ -1167,12 +1177,13 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
boolean shouldDropBehind, long totalCompactedFilesSize, String fileStoragePolicy)
throws IOException {
// creating new cache config for each new writer
+ final CacheConfig cacheConf = getCacheConfig();
final CacheConfig writerCacheConf = new CacheConfig(cacheConf);
if (isCompaction) {
// Don't cache data on write on compactions, unless specifically configured to do so
// Cache only when total file size remains lower than configured threshold
final boolean cacheCompactedBlocksOnWrite =
- cacheConf.shouldCacheCompactedBlocksOnWrite();
+ getCacheConfig().shouldCacheCompactedBlocksOnWrite();
// if data blocks are to be cached on write
// during compaction, we should forcefully
// cache index and bloom blocks as well
@@ -1212,12 +1223,12 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
region.getRegionInfo().getEncodedName());
}
HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag,
- cryptoContext);
- Path familyTempDir = new Path(fs.getTempDir(), family.getNameAsString());
+ getCryptoContext());
+ Path familyTempDir = new Path(getRegionFileSystem().getTempDir(), getColumnFamilyName());
StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf,
this.getFileSystem())
.withOutputDir(familyTempDir)
- .withBloomType(family.getBloomFilterType())
+ .withBloomType(storeContext.getBloomFilterType())
.withMaxKeyCount(maxKeyCount)
.withFavoredNodes(favoredNodes)
.withFileContext(hFileContext)
@@ -1236,18 +1247,19 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
.withIncludesMvcc(includeMVCCReadpoint)
.withIncludesTags(includesTag)
.withCompression(compression)
- .withCompressTags(family.isCompressTags())
- .withChecksumType(checksumType)
- .withBytesPerCheckSum(bytesPerChecksum)
+ .withCompressTags(getColumnFamilyDescriptor().isCompressTags())
+ .withChecksumType(StoreUtils.getChecksumType(conf))
+ .withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf))
.withBlockSize(blocksize)
.withHBaseCheckSum(true)
- .withDataBlockEncoding(family.getDataBlockEncoding())
+ .withDataBlockEncoding(getColumnFamilyDescriptor()
+ .getDataBlockEncoding())
.withEncryptionContext(cryptoContext)
.withCreateTime(EnvironmentEdgeManager.currentTime())
- .withColumnFamily(family.getName())
+ .withColumnFamily(getColumnFamilyDescriptor().getName())
.withTableName(region.getTableDescriptor()
.getTableName().getName())
- .withCellComparator(this.comparator)
+ .withCellComparator(getComparator())
.build();
return hFileContext;
}
@@ -1529,7 +1541,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
// Ready to go. Have list of files to compact.
LOG.info("Starting compaction of " + filesToCompact +
- " into tmpdir=" + fs.getTempDir() + ", totalSize=" +
+ " into tmpdir=" + getRegionFileSystem().getTempDir() + ", totalSize=" +
TraditionalBinaryPrefix.long2String(cr.getSize(), "", 1));
return doCompaction(cr, filesToCompact, user, compactionStartTime,
@@ -1579,7 +1591,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
String prefix = HConstants.STORAGE_POLICY_PREFIX;
for (Path newFile : newFiles) {
if (newFile.getParent().getName().startsWith(prefix)) {
- CommonFSUtils.setStoragePolicy(fs.getFileSystem(), newFile,
+ CommonFSUtils.setStoragePolicy(getRegionFileSystem().getFileSystem(), newFile,
newFile.getParent().getName().substring(prefix.length()));
}
}
@@ -1604,7 +1616,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
HStoreFile moveFileIntoPlace(Path newFile) throws IOException {
validateStoreFile(newFile);
// Move the file into the right spot
- Path destPath = fs.commitStoreFile(getColumnFamilyName(), newFile);
+ Path destPath = getRegionFileSystem().commitStoreFile(getColumnFamilyName(), newFile);
return createStoreFileAndReader(destPath);
}
@@ -1624,8 +1636,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
newFiles.stream().map(HStoreFile::getPath).collect(Collectors.toList());
RegionInfo info = this.region.getRegionInfo();
CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
- family.getName(), inputPaths, outputPaths,
- fs.getStoreDir(getColumnFamilyDescriptor().getNameAsString()));
+ getColumnFamilyDescriptor().getName(), inputPaths, outputPaths,
+ getRegionFileSystem().getStoreDir(getColumnFamilyDescriptor().getNameAsString()));
// Fix reaching into Region to get the maxWaitForSeqId.
// Does this method belong in Region altogether given it is making so many references up there?
// Could be Region#writeCompactionMarker(compactionDescriptor);
@@ -1752,7 +1764,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
String familyName = this.getColumnFamilyName();
Set<String> inputFiles = new HashSet<>();
for (String compactionInput : compactionInputs) {
- Path inputPath = fs.getStoreFilePath(familyName, compactionInput);
+ Path inputPath = getRegionFileSystem().getStoreFilePath(familyName, compactionInput);
inputFiles.add(inputPath.getName());
}
@@ -1772,7 +1784,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
compactionOutputs.remove(sf.getPath().getName());
}
for (String compactionOutput : compactionOutputs) {
- StoreFileInfo storeFileInfo = fs.getStoreFileInfo(getColumnFamilyName(), compactionOutput);
+ StoreFileInfo storeFileInfo =
+ getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), compactionOutput);
HStoreFile storeFile = createStoreFileAndReader(storeFileInfo);
outputStoreFiles.add(storeFile);
}
@@ -2092,7 +2105,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
throw new IllegalArgumentException("Number of versions must be > 0");
}
// Make sure we do not return more than maximum versions for this store.
- int maxVersions = this.family.getMaxVersions();
+ int maxVersions = getColumnFamilyDescriptor().getMaxVersions();
return wantedVersions > maxVersions ? maxVersions: wantedVersions;
}
@@ -2367,7 +2380,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
@Override
public RegionInfo getRegionInfo() {
- return this.fs.getRegionInfo();
+ return getRegionFileSystem().getRegionInfo();
}
@Override
@@ -2509,7 +2522,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
List<HStoreFile> storeFiles = new ArrayList<>(fileNames.size());
for (String file : fileNames) {
// open the file as a store file (hfile link, etc)
- StoreFileInfo storeFileInfo = fs.getStoreFileInfo(getColumnFamilyName(), file);
+ StoreFileInfo storeFileInfo = getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file);
HStoreFile storeFile = createStoreFileAndReader(storeFileInfo);
storeFiles.add(storeFile);
HStore.this.storeSize.addAndGet(storeFile.getReader().length());
@@ -2559,7 +2572,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
* @return cache configuration for this Store.
*/
public CacheConfig getCacheConfig() {
- return this.cacheConf;
+ return storeContext.getCacheConf();
}
public static final long FIXED_OVERHEAD = ClassSize.estimateBase(HStore.class, false);
@@ -2578,7 +2591,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
@Override
public CellComparator getComparator() {
- return comparator;
+ return storeContext.getComparator();
}
public ScanInfo getScanInfo() {
@@ -2652,7 +2665,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
public void onConfigurationChange(Configuration conf) {
this.conf = new CompoundConfiguration()
.add(conf)
- .addBytesMap(family.getValues());
+ .addBytesMap(getColumnFamilyDescriptor().getValues());
this.storeEngine.compactionPolicy.setConf(conf);
this.offPeakHours = OffPeakHours.getInstance(conf);
}
@@ -2784,8 +2797,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
LOG.debug("Moving the files {} to archive", filesToRemove);
// Only if this is successful it has to be removed
try {
- this.fs.removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(),
- filesToRemove);
+ getRegionFileSystem().removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(),
+ filesToRemove);
} catch (FailedArchiveException fae) {
// Even if archiving some files failed, we still need to clear out any of the
// files which were successfully archived. Otherwise we will receive a
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreContext.java
new file mode 100644
index 0000000..4a0c7af
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreContext.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.CellComparator;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFileContext;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.function.Supplier;
+
+/**
+ * This carries the information on some of the meta data about the HStore. This
+ * meta data can be used across the HFileWriter/Readers and other HStore consumers without the
+ * need of passing around the complete store.
+ */
+@InterfaceAudience.Private
+public class HStoreContext {
+ private final HFileContext defaultFileContext;
+ private final CacheConfig cacheConf;
+ private final HRegionFileSystem regionFileSystem;
+ private final CellComparator comparator;
+ private final BloomType bloomFilterType;
+ private final Supplier<Collection<HStoreFile>> compactedFilesSupplier;
+ private final Supplier<InetSocketAddress[]> favoredNodesSupplier;
+ private final ColumnFamilyDescriptor family;
+ private final Path familyStoreDirectoryPath;
+ private final RegionCoprocessorHost coprocessorHost;
+
+ private HStoreContext(Builder builder) {
+ this.defaultFileContext = builder.defaultFileContext;
+ this.cacheConf = builder.cacheConf;
+ this.regionFileSystem = builder.regionFileSystem;
+ this.comparator = builder.comparator;
+ this.bloomFilterType = builder.bloomFilterType;
+ this.compactedFilesSupplier = builder.compactedFilesSupplier;
+ this.favoredNodesSupplier = builder.favoredNodesSupplier;
+ this.family = builder.family;
+ this.familyStoreDirectoryPath = builder.familyStoreDirectoryPath;
+ this.coprocessorHost = builder.coprocessorHost;
+ }
+
+ public HFileContext getDefaultFileContext() {
+ return defaultFileContext;
+ }
+
+ public CacheConfig getCacheConf() {
+ return cacheConf;
+ }
+
+ public HRegionFileSystem getRegionFileSystem() {
+ return regionFileSystem;
+ }
+
+ public CellComparator getComparator() {
+ return comparator;
+ }
+
+ public BloomType getBloomFilterType() {
+ return bloomFilterType;
+ }
+
+ public Supplier<Collection<HStoreFile>> getCompactedFilesSupplier() {
+ return compactedFilesSupplier;
+ }
+
+ public Supplier<InetSocketAddress[]> getFavoredNodesSupplier() {
+ return favoredNodesSupplier;
+ }
+
+ public ColumnFamilyDescriptor getFamily() {
+ return family;
+ }
+
+ public Path getFamilyStoreDirectoryPath() {
+ return familyStoreDirectoryPath;
+ }
+
+ public RegionCoprocessorHost getCoprocessorHost() {
+ return coprocessorHost;
+ }
+
+ public static Builder getBuilder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private HFileContext defaultFileContext;
+ private CacheConfig cacheConf;
+ private HRegionFileSystem regionFileSystem;
+ private CellComparator comparator;
+ private BloomType bloomFilterType;
+ private Supplier<Collection<HStoreFile>> compactedFilesSupplier;
+ private Supplier<InetSocketAddress[]> favoredNodesSupplier;
+ private ColumnFamilyDescriptor family;
+ private Path familyStoreDirectoryPath;
+ private RegionCoprocessorHost coprocessorHost;
+
+ public Builder withDefaultHFileContext(HFileContext defaultFileContext) {
+ this.defaultFileContext = defaultFileContext;
+ return this;
+ }
+
+ public Builder withCacheConfig(CacheConfig cacheConf) {
+ this.cacheConf = cacheConf;
+ return this;
+ }
+
+ public Builder withRegionFileSystem(HRegionFileSystem regionFileSystem) {
+ this.regionFileSystem = regionFileSystem;
+ return this;
+ }
+
+ public Builder withCellComparator(CellComparator comparator) {
+ this.comparator = comparator;
+ return this;
+ }
+
+ public Builder withBloomType(BloomType bloomFilterType) {
+ this.bloomFilterType = bloomFilterType;
+ return this;
+ }
+
+ public Builder withCompactedFilesSupplier(Supplier<Collection<HStoreFile>>
+ compactedFilesSupplier) {
+ this.compactedFilesSupplier = compactedFilesSupplier;
+ return this;
+ }
+
+ public Builder withFavoredNodesSupplier(Supplier<InetSocketAddress[]> favoredNodesSupplier) {
+ this.favoredNodesSupplier = favoredNodesSupplier;
+ return this;
+ }
+
+ public Builder withColumnFamilyDescriptor(ColumnFamilyDescriptor family) {
+ this.family = family;
+ return this;
+ }
+
+ public Builder withFamilyStoreDirectoryPath(Path familyStoreDirectoryPath) {
+ this.familyStoreDirectoryPath = familyStoreDirectoryPath;
+ return this;
+ }
+
+ public Builder withRegionCoprocessorHost(RegionCoprocessorHost coprocessorHost) {
+ this.coprocessorHost = coprocessorHost;
+ return this;
+ }
+
+ public HStoreContext build () {
+ return new HStoreContext(this);
+ }
+ }
+
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java
index 0e4f6c2..6d383e8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreUtils.java
@@ -24,9 +24,13 @@ import java.util.Optional;
import java.util.OptionalInt;
import java.util.OptionalLong;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -136,4 +140,26 @@ public class StoreUtils {
return largestFile.isPresent() ? StoreUtils.getFileSplitPoint(largestFile.get(), comparator)
: Optional.empty();
}
+
+ /**
+ * Returns the configured checksum algorithm.
+ * @param conf The configuration
+ * @return The checksum algorithm that is set in the configuration
+ */
+ public static ChecksumType getChecksumType(Configuration conf) {
+ String checksumName = conf.get(HConstants.CHECKSUM_TYPE_NAME);
+ return checksumName == null ? ChecksumType.getDefaultChecksumType() :
+ ChecksumType.nameToType(checksumName);
+ }
+
+ /**
+ * Returns the configured bytesPerChecksum value.
+ * @param conf The configuration
+ * @return The bytesPerChecksum that is set in the configuration
+ */
+ public static int getBytesPerChecksum(Configuration conf) {
+ return conf.getInt(HConstants.BYTES_PER_CHECKSUM,
+ HFile.DEFAULT_BYTES_PER_CHECKSUM);
+ }
+
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
index ec9a59c..342565d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
@@ -86,6 +86,7 @@ import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
+import org.apache.hadoop.hbase.regionserver.StoreUtils;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.security.token.FsDelegationToken;
import org.apache.hadoop.hbase.util.Bytes;
@@ -749,8 +750,8 @@ public class BulkLoadHFilesTool extends Configured implements BulkLoadHFiles, To
Algorithm compression = familyDescriptor.getCompressionType();
BloomType bloomFilterType = familyDescriptor.getBloomFilterType();
HFileContext hFileContext = new HFileContextBuilder().withCompression(compression)
- .withChecksumType(HStore.getChecksumType(conf))
- .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blocksize)
+ .withChecksumType(StoreUtils.getChecksumType(conf))
+ .withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf)).withBlockSize(blocksize)
.withDataBlockEncoding(familyDescriptor.getDataBlockEncoding()).withIncludesTags(true)
.build();
halfWriter = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
new file mode 100644
index 0000000..e69de29
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java
index 9623bd1..50bc5fe 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java
@@ -39,8 +39,8 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.regionserver.CellSet;
-import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
+import org.apache.hadoop.hbase.regionserver.StoreUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.EntryBuffers.RegionEntryBuffer;
import org.apache.hadoop.hbase.wal.WAL.Entry;
@@ -199,8 +199,8 @@ public class BoundedRecoveredHFilesOutputSink extends OutputSink {
new StoreFileWriter.Builder(walSplitter.conf, CacheConfig.DISABLED, walSplitter.rootFS)
.withOutputDir(outputDir);
HFileContext hFileContext = new HFileContextBuilder().
- withChecksumType(HStore.getChecksumType(walSplitter.conf)).
- withBytesPerCheckSum(HStore.getBytesPerChecksum(walSplitter.conf)).
+ withChecksumType(StoreUtils.getChecksumType(walSplitter.conf)).
+ withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(walSplitter.conf)).
withCellComparator(isMetaTable?
MetaCellComparator.META_COMPARATOR: CellComparatorImpl.COMPARATOR).build();
return writerBuilder.withFileContext(hFileContext).build();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index fcbc718..bd1715b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -7657,7 +7657,7 @@ public class TestHRegion {
LOG.warn("hbase.hstore.compaction.complete is set to false");
List<HStoreFile> sfs = new ArrayList<>(newFiles.size());
final boolean evictOnClose =
- cacheConf != null? cacheConf.shouldEvictOnClose(): true;
+ getCacheConfig() != null? getCacheConfig().shouldEvictOnClose(): true;
for (Path newFile : newFiles) {
// Create storefile around what we wrote with a reader on it.
HStoreFile sf = createStoreFileAndReader(newFile);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkLoadManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkLoadManager.java
index 12cf576..88f201e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkLoadManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkLoadManager.java
@@ -221,8 +221,8 @@ public class TestSecureBulkLoadManager {
.withIncludesTags(true)
.withCompression(compression)
.withCompressTags(family.isCompressTags())
- .withChecksumType(HStore.getChecksumType(conf))
- .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf))
+ .withChecksumType(StoreUtils.getChecksumType(conf))
+ .withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf))
.withBlockSize(family.getBlocksize())
.withHBaseCheckSum(true)
.withDataBlockEncoding(family.getDataBlockEncoding())