You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2017/09/25 01:58:53 UTC
[11/11] hbase git commit: HBASE-18825 Use HStoreFile instead of
StoreFile in our own code base and remove unnecessary methods in StoreFile
interface
HBASE-18825 Use HStoreFile instead of StoreFile in our own code base and remove unnecessary methods in StoreFile interface
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a5f84430
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a5f84430
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a5f84430
Branch: refs/heads/master
Commit: a5f84430a305db6a1eddd45f82d19babf43a8d01
Parents: 1540483
Author: zhangduo <zh...@apache.org>
Authored: Sun Sep 24 19:22:16 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Mon Sep 25 09:35:39 2017 +0800
----------------------------------------------------------------------
.../hadoop/hbase/backup/util/RestoreTool.java | 4 +-
.../example/ZooKeeperScanPolicyObserver.java | 68 +-
.../hbase/mapreduce/HFileOutputFormat2.java | 30 +-
.../hbase/regionserver/CompactionTool.java | 6 +-
.../hbase/mapreduce/TestHFileOutputFormat2.java | 4 +-
.../hadoop/hbase/backup/HFileArchiver.java | 30 +-
.../hbase/coprocessor/RegionObserver.java | 4 +-
.../hadoop/hbase/io/HalfStoreFileReader.java | 23 +-
.../org/apache/hadoop/hbase/io/hfile/HFile.java | 11 +-
.../hbase/io/hfile/HFilePrettyPrinter.java | 2 +-
.../hadoop/hbase/io/hfile/HFileReaderImpl.java | 36 +-
.../assignment/SplitTableRegionProcedure.java | 31 +-
.../master/balancer/StochasticLoadBalancer.java | 5 +-
.../apache/hadoop/hbase/mob/CachedMobFile.java | 7 +-
.../hbase/mob/DefaultMobStoreCompactor.java | 2 +-
.../hbase/mob/DefaultMobStoreFlusher.java | 9 +-
.../org/apache/hadoop/hbase/mob/MobFile.java | 15 +-
.../apache/hadoop/hbase/mob/MobStoreEngine.java | 8 +-
.../org/apache/hadoop/hbase/mob/MobUtils.java | 11 +-
.../PartitionedMobCompactionRequest.java | 13 +-
.../compactions/PartitionedMobCompactor.java | 58 +-
.../regionserver/ChangedReadersObserver.java | 4 +-
.../hbase/regionserver/CompactingMemStore.java | 4 +-
.../regionserver/DateTieredStoreEngine.java | 8 +-
.../hbase/regionserver/DefaultStoreEngine.java | 20 +-
.../regionserver/DefaultStoreFileManager.java | 155 +-
.../hbase/regionserver/DefaultStoreFlusher.java | 2 +-
.../hadoop/hbase/regionserver/HMobStore.java | 8 +-
.../hadoop/hbase/regionserver/HRegion.java | 90 +-
.../hbase/regionserver/HRegionFileSystem.java | 44 +-
.../hbase/regionserver/HRegionServer.java | 78 +-
.../hadoop/hbase/regionserver/HStore.java | 524 +++---
.../hadoop/hbase/regionserver/HStoreFile.java | 170 +-
.../hbase/regionserver/MemStoreCompactor.java | 31 +-
.../MemStoreCompactorSegmentsIterator.java | 8 +-
.../hbase/regionserver/MobStoreScanner.java | 4 +-
.../regionserver/RegionCoprocessorHost.java | 54 +-
.../hbase/regionserver/RegionSplitPolicy.java | 15 +-
.../regionserver/ReversedMobStoreScanner.java | 4 +-
.../regionserver/ReversedStoreScanner.java | 4 +-
.../apache/hadoop/hbase/regionserver/Store.java | 121 +-
.../hadoop/hbase/regionserver/StoreEngine.java | 16 +-
.../hadoop/hbase/regionserver/StoreFile.java | 126 +-
.../regionserver/StoreFileComparators.java | 28 +-
.../hbase/regionserver/StoreFileManager.java | 42 +-
.../hbase/regionserver/StoreFileReader.java | 46 +-
.../hbase/regionserver/StoreFileScanner.java | 24 +-
.../hbase/regionserver/StoreFileWriter.java | 42 +-
.../hadoop/hbase/regionserver/StoreFlusher.java | 6 +-
.../hadoop/hbase/regionserver/StoreScanner.java | 29 +-
.../hadoop/hbase/regionserver/StoreUtils.java | 86 +-
.../hbase/regionserver/StripeStoreEngine.java | 8 +-
.../regionserver/StripeStoreFileManager.java | 225 +--
.../hbase/regionserver/StripeStoreFlusher.java | 6 +-
.../AbstractMultiOutputCompactor.java | 6 +-
.../compactions/CompactionContext.java | 11 +-
.../compactions/CompactionPolicy.java | 8 +-
.../compactions/CompactionRequest.java | 12 +-
.../regionserver/compactions/Compactor.java | 26 +-
.../compactions/DateTieredCompactionPolicy.java | 38 +-
.../DateTieredCompactionRequest.java | 4 +-
.../compactions/DateTieredCompactor.java | 6 +-
.../compactions/DefaultCompactor.java | 10 +-
.../compactions/ExploringCompactionPolicy.java | 40 +-
.../compactions/FIFOCompactionPolicy.java | 29 +-
.../compactions/RatioBasedCompactionPolicy.java | 18 +-
.../compactions/SortedCompactionPolicy.java | 60 +-
.../compactions/StripeCompactionPolicy.java | 92 +-
.../compactions/StripeCompactor.java | 6 +-
.../hadoop/hbase/snapshot/SnapshotManifest.java | 21 +-
.../hbase/tool/LoadIncrementalHFiles.java | 25 +-
.../hadoop/hbase/util/BloomFilterFactory.java | 14 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 17 +-
.../hadoop/hbase/util/RowBloomContext.java | 7 +-
.../hadoop/hbase/util/RowColBloomContext.java | 7 +-
.../hbase-webapps/regionserver/region.jsp | 2 +-
.../org/apache/hadoop/hbase/TestIOFencing.java | 12 +-
.../apache/hadoop/hbase/client/TestAdmin1.java | 17 +-
...estAvoidCellReferencesIntoShippedBlocks.java | 13 +-
.../hadoop/hbase/client/TestFromClientSide.java | 7 +-
.../hbase/coprocessor/SimpleRegionObserver.java | 15 +-
.../TestRegionObserverScannerOpenHook.java | 3 +-
.../hbase/io/TestHalfStoreFileReader.java | 4 +-
.../apache/hadoop/hbase/io/hfile/TestHFile.java | 4 +-
.../hbase/io/hfile/TestHFileBlockIndex.java | 8 +-
.../hadoop/hbase/io/hfile/TestHFileSeek.java | 4 +-
.../hbase/mob/compactions/TestMobCompactor.java | 6 +-
.../TestPartitionedMobCompactor.java | 74 +-
.../hbase/quotas/SpaceQuotaHelperForTests.java | 12 +-
.../AbstractTestDateTieredCompactionPolicy.java | 24 +-
.../regionserver/DataBlockEncodingTool.java | 2 +-
.../EncodedSeekPerformanceTest.java | 8 +-
.../hbase/regionserver/MockHStoreFile.java | 229 +++
.../hbase/regionserver/MockStoreFile.java | 220 ---
.../regionserver/NoOpScanPolicyObserver.java | 15 +-
.../regionserver/TestCacheOnWriteInSchema.java | 2 +-
.../hbase/regionserver/TestCompaction.java | 41 +-
.../TestCompactionArchiveConcurrentClose.java | 15 +-
.../TestCompactionArchiveIOException.java | 18 +-
.../regionserver/TestCompactionPolicy.java | 26 +-
.../regionserver/TestCompoundBloomFilter.java | 2 +-
.../TestDefaultCompactSelection.java | 10 +-
.../regionserver/TestDefaultStoreEngine.java | 10 +-
.../regionserver/TestEncryptionKeyRotation.java | 24 +-
.../TestEncryptionRandomKeying.java | 4 +-
.../hbase/regionserver/TestFSErrorsExposed.java | 6 +-
.../hbase/regionserver/TestHMobStore.java | 6 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 135 +-
.../regionserver/TestHRegionReplayEvents.java | 8 +-
.../regionserver/TestHRegionServerBulkLoad.java | 16 +-
.../hadoop/hbase/regionserver/TestHStore.java | 1743 ++++++++++++++++++
.../hbase/regionserver/TestHStoreFile.java | 15 +-
.../hbase/regionserver/TestMajorCompaction.java | 8 +-
.../regionserver/TestMobStoreCompaction.java | 16 +-
.../hbase/regionserver/TestRegionReplicas.java | 15 +-
.../regionserver/TestRegionSplitPolicy.java | 13 +-
.../regionserver/TestReversibleScanners.java | 45 +-
.../regionserver/TestScannerWithBulkload.java | 9 +-
.../TestSplitTransactionOnCluster.java | 17 +-
.../hadoop/hbase/regionserver/TestStore.java | 1740 -----------------
.../regionserver/TestStripeStoreEngine.java | 12 +-
.../TestStripeStoreFileManager.java | 155 +-
.../regionserver/TestSwitchToStreamRead.java | 11 +-
.../ConstantSizeFileListGenerator.java | 11 +-
.../compactions/EverythingPolicy.java | 12 +-
.../compactions/ExplicitFileListGenerator.java | 11 +-
.../compactions/GaussianFileListGenerator.java | 11 +-
.../compactions/MockStoreFileGenerator.java | 24 +-
.../compactions/PerfTestCompactionPolicies.java | 38 +-
.../SemiConstantSizeFileListGenerator.java | 11 +-
.../SinusoidalFileListGenerator.java | 13 +-
.../compactions/SpikyFileListGenerator.java | 11 +-
.../compactions/StoreFileListGenerator.java | 9 +-
.../TestCompactedHFilesDischarger.java | 63 +-
.../regionserver/compactions/TestCompactor.java | 6 +-
.../compactions/TestDateTieredCompactor.java | 16 +-
.../compactions/TestStripeCompactionPolicy.java | 89 +-
.../compactions/TestStripeCompactor.java | 8 +-
.../regionserver/wal/AbstractTestWALReplay.java | 17 +-
.../visibility/TestVisibilityLabels.java | 10 +-
.../apache/hadoop/hbase/util/HFileTestUtil.java | 4 +-
.../hbase/util/TestCoprocessorScanPolicy.java | 13 +-
.../hbase/util/TestHBaseFsckEncryption.java | 13 +-
.../hadoop/hbase/spark/HBaseContext.scala | 10 +-
144 files changed, 3920 insertions(+), 4147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java
index ca0d026..b00351b 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java
@@ -440,8 +440,8 @@ public class RestoreTool {
final byte[] first, last;
try {
reader.loadFileInfo();
- first = reader.getFirstRowKey();
- last = reader.getLastRowKey();
+ first = reader.getFirstRowKey().get();
+ last = reader.getLastRowKey().get();
LOG.debug("Trying to figure out region boundaries hfile=" + hfile + " first="
+ Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last));
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
index 6b31664..a730403 100644
--- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
+++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ZooKeeperScanPolicyObserver.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreScanner;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.zookeeper.KeeperException;
@@ -130,32 +129,32 @@ public class ZooKeeperScanPolicyObserver implements RegionObserver {
@Override
public void process(WatchedEvent event) {
- switch(event.getType()) {
- case NodeDataChanged:
- case NodeCreated:
- try {
- // get data and re-watch
- data = zk.getData(node, this, null);
- LOG.debug("Read asynchronously: "+(data == null ? "null" : Bytes.toLong(data)));
- } catch (InterruptedException ix) {
- } catch (KeeperException kx) {
- needSetup = true;
- }
- break;
+ switch (event.getType()) {
+ case NodeDataChanged:
+ case NodeCreated:
+ try {
+ // get data and re-watch
+ data = zk.getData(node, this, null);
+ LOG.debug("Read asynchronously: " + (data == null ? "null" : Bytes.toLong(data)));
+ } catch (InterruptedException ix) {
+ } catch (KeeperException kx) {
+ needSetup = true;
+ }
+ break;
- case NodeDeleted:
- try {
- // just re-watch
- zk.exists(node, this);
- data = null;
- } catch (InterruptedException ix) {
- } catch (KeeperException kx) {
- needSetup = true;
- }
- break;
+ case NodeDeleted:
+ try {
+ // just re-watch
+ zk.exists(node, this);
+ data = null;
+ } catch (InterruptedException ix) {
+ } catch (KeeperException kx) {
+ needSetup = true;
+ }
+ break;
- default:
- // ignore
+ default:
+ // ignore
}
}
}
@@ -166,15 +165,13 @@ public class ZooKeeperScanPolicyObserver implements RegionObserver {
if (!re.getSharedData().containsKey(zkkey)) {
// there is a short race here
// in the worst case we create a watcher that will be notified once
- re.getSharedData().putIfAbsent(
- zkkey,
- new ZKWatcher(re.getRegionServerServices().getZooKeeper()
- .getRecoverableZooKeeper().getZooKeeper()));
+ re.getSharedData().putIfAbsent(zkkey, new ZKWatcher(
+ re.getRegionServerServices().getZooKeeper().getRecoverableZooKeeper().getZooKeeper()));
}
}
protected ScanInfo getScanInfo(Store store, RegionCoprocessorEnvironment e) {
- byte[] data = ((ZKWatcher)e.getSharedData().get(zkkey)).getData();
+ byte[] data = ((ZKWatcher) e.getSharedData().get(zkkey)).getData();
if (data == null) {
return null;
}
@@ -182,8 +179,7 @@ public class ZooKeeperScanPolicyObserver implements RegionObserver {
if (oldSI.getTtl() == Long.MAX_VALUE) {
return null;
}
- long ttl = Math.max(EnvironmentEdgeManager.currentTime() -
- Bytes.toLong(data), oldSI.getTtl());
+ long ttl = Math.max(EnvironmentEdgeManager.currentTime() - Bytes.toLong(data), oldSI.getTtl());
return new ScanInfo(oldSI.getConfiguration(), store.getColumnFamilyDescriptor(), ttl,
oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
}
@@ -197,7 +193,7 @@ public class ZooKeeperScanPolicyObserver implements RegionObserver {
// take default action
return null;
}
- return new StoreScanner(store, scanInfo, OptionalInt.empty(), scanners,
+ return new StoreScanner((HStore) store, scanInfo, OptionalInt.empty(), scanners,
ScanType.COMPACT_RETAIN_DELETES, store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP);
}
@@ -210,7 +206,7 @@ public class ZooKeeperScanPolicyObserver implements RegionObserver {
// take default action
return null;
}
- return new StoreScanner(store, scanInfo, OptionalInt.empty(), scanners, scanType,
+ return new StoreScanner((HStore) store, scanInfo, OptionalInt.empty(), scanners, scanType,
store.getSmallestReadPoint(), earliestPutTs);
}
@@ -223,7 +219,7 @@ public class ZooKeeperScanPolicyObserver implements RegionObserver {
// take default action
return null;
}
- return new StoreScanner(store, scanInfo, scan, targetCols,
- ((HStore)store).getHRegion().getReadPoint(IsolationLevel.READ_COMMITTED));
+ return new StoreScanner((HStore) store, scanInfo, scan, targetCols,
+ ((HStore) store).getHRegion().getReadPoint(IsolationLevel.READ_COMMITTED));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
index 41a9839..e8b7d11 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
@@ -17,6 +17,11 @@
*/
package org.apache.hadoop.hbase.mapreduce;
+import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TASK_KEY;
+import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TIME_KEY;
+import static org.apache.hadoop.hbase.regionserver.HStoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY;
+import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY;
+
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.InetSocketAddress;
@@ -43,8 +48,12 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
@@ -52,10 +61,8 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.fs.HFileSystem;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
@@ -65,13 +72,9 @@ import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.io.hfile.HFileWriterImpl;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HStore;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -85,6 +88,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner;
+import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
@@ -413,13 +417,13 @@ public class HFileOutputFormat2
private void close(final StoreFileWriter w) throws IOException {
if (w != null) {
- w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY,
+ w.appendFileInfo(BULKLOAD_TIME_KEY,
Bytes.toBytes(System.currentTimeMillis()));
- w.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY,
+ w.appendFileInfo(BULKLOAD_TASK_KEY,
Bytes.toBytes(context.getTaskAttemptID().toString()));
- w.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY,
+ w.appendFileInfo(MAJOR_COMPACTION_KEY,
Bytes.toBytes(true));
- w.appendFileInfo(StoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY,
+ w.appendFileInfo(EXCLUDE_FROM_MINOR_COMPACTION_KEY,
Bytes.toBytes(compactionExclude));
w.appendTrackedTimestampsToMetadata();
w.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
index bb01459..7912340 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.mapreduce.JobUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
@@ -62,6 +61,7 @@ import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.util.LineReader;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.yetus.audience.InterfaceAudience;
/*
* The CompactionTool allows to execute a compaction specifying a:
@@ -165,11 +165,11 @@ public class CompactionTool extends Configured implements Tool {
if (!compaction.isPresent()) {
break;
}
- List<StoreFile> storeFiles =
+ List<HStoreFile> storeFiles =
store.compact(compaction.get(), NoLimitThroughputController.INSTANCE);
if (storeFiles != null && !storeFiles.isEmpty()) {
if (keepCompactedFiles && deleteCompacted) {
- for (StoreFile storeFile: storeFiles) {
+ for (HStoreFile storeFile: storeFiles) {
fs.delete(storeFile.getPath(), false);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
index cbff2de..0b5a929 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
@@ -18,6 +18,7 @@
*/
package org.apache.hadoop.hbase.mapreduce;
+import static org.apache.hadoop.hbase.regionserver.HStoreFile.BLOOM_FILTER_TYPE_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -85,7 +86,6 @@ import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.TestHRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -1162,7 +1162,7 @@ public class TestHFileOutputFormat2 {
Reader reader = HFile.createReader(fs, dataFilePath, new CacheConfig(conf), true, conf);
Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
- byte[] bloomFilter = fileInfo.get(StoreFile.BLOOM_FILTER_TYPE_KEY);
+ byte[] bloomFilter = fileInfo.get(BLOOM_FILTER_TYPE_KEY);
if (bloomFilter == null) bloomFilter = Bytes.toBytes("NONE");
assertEquals("Incorrect bloom filter used for column family " + familyStr +
"(reader: " + reader + ")",
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
index d17546e..4321dc8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
@@ -33,14 +33,14 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.io.MultipleIOException;
+import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.shaded.com.google.common.base.Function;
import org.apache.hadoop.hbase.shaded.com.google.common.base.Preconditions;
@@ -48,7 +48,7 @@ import org.apache.hadoop.hbase.shaded.com.google.common.collect.Collections2;
import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
/**
- * Utility class to handle the removal of HFiles (or the respective {@link StoreFile StoreFiles})
+ * Utility class to handle the removal of HFiles (or the respective {@link HStoreFile StoreFiles})
* for a HRegion from the {@link FileSystem}. The hfiles will be archived or deleted, depending on
* the state of the system.
*/
@@ -226,7 +226,7 @@ public class HFileArchiver {
* @throws IOException if the files could not be correctly disposed.
*/
public static void archiveStoreFiles(Configuration conf, FileSystem fs, HRegionInfo regionInfo,
- Path tableDir, byte[] family, Collection<StoreFile> compactedFiles)
+ Path tableDir, byte[] family, Collection<HStoreFile> compactedFiles)
throws IOException, FailedArchiveException {
// sometimes in testing, we don't have rss, so we need to check for that
@@ -479,13 +479,13 @@ public class HFileArchiver {
* @throws IOException if a file cannot be deleted. All files will be attempted to deleted before
* throwing the exception, rather than failing at the first file.
*/
- private static void deleteStoreFilesWithoutArchiving(Collection<StoreFile> compactedFiles)
+ private static void deleteStoreFilesWithoutArchiving(Collection<HStoreFile> compactedFiles)
throws IOException {
LOG.debug("Deleting store files without archiving.");
List<IOException> errors = new ArrayList<>(0);
- for (StoreFile hsf : compactedFiles) {
+ for (HStoreFile hsf : compactedFiles) {
try {
- hsf.deleteReader();
+ hsf.deleteStoreFile();
} catch (IOException e) {
LOG.error("Failed to delete store file:" + hsf.getPath());
errors.add(e);
@@ -524,16 +524,16 @@ public class HFileArchiver {
}
/**
- * Convert the {@link StoreFile} into something we can manage in the archive
+ * Convert the {@link HStoreFile} into something we can manage in the archive
* methods
*/
- private static class StoreToFile extends FileConverter<StoreFile> {
+ private static class StoreToFile extends FileConverter<HStoreFile> {
public StoreToFile(FileSystem fs) {
super(fs);
}
@Override
- public File apply(StoreFile input) {
+ public File apply(HStoreFile input) {
return new FileableStoreFile(fs, input);
}
}
@@ -656,20 +656,20 @@ public class HFileArchiver {
}
/**
- * {@link File} adapter for a {@link StoreFile} living on a {@link FileSystem}
+ * {@link File} adapter for a {@link HStoreFile} living on a {@link FileSystem}
* .
*/
private static class FileableStoreFile extends File {
- StoreFile file;
+ HStoreFile file;
- public FileableStoreFile(FileSystem fs, StoreFile store) {
+ public FileableStoreFile(FileSystem fs, HStoreFile store) {
super(fs);
this.file = store;
}
@Override
public void delete() throws IOException {
- file.deleteReader();
+ file.deleteStoreFile();
}
@Override
@@ -690,7 +690,7 @@ public class HFileArchiver {
@Override
public void close() throws IOException {
- file.closeReader(true);
+ file.closeStoreFile(true);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index 9318b9a..60e5f40 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -189,7 +189,7 @@ public interface RegionObserver extends Coprocessor {
* @param tracker tracker used to track the life cycle of a compaction
*/
default void preCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- List<StoreFile> candidates, CompactionLifeCycleTracker tracker) throws IOException {}
+ List<? extends StoreFile> candidates, CompactionLifeCycleTracker tracker) throws IOException {}
/**
* Called after the {@link StoreFile}s to compact have been selected from the available
@@ -200,7 +200,7 @@ public interface RegionObserver extends Coprocessor {
* @param tracker tracker used to track the life cycle of a compaction
*/
default void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- ImmutableList<StoreFile> selected, CompactionLifeCycleTracker tracker) {}
+ ImmutableList<? extends StoreFile> selected, CompactionLifeCycleTracker tracker) {}
/**
* Called prior to writing the {@link StoreFile}s selected for compaction into a new
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
index 0c915d1..18ddb6a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.io;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@@ -48,7 +49,7 @@ import org.apache.hadoop.hbase.util.Bytes;
* <p>This type works in tandem with the {@link Reference} type. This class
* is used reading while Reference is used writing.
*
- * <p>This file is not splitable. Calls to {@link #midkey()} return null.
+ * <p>This file is not splitable. Calls to {@link #midKey()} return null.
*/
@InterfaceAudience.Private
public class HalfStoreFileReader extends StoreFileReader {
@@ -60,7 +61,7 @@ public class HalfStoreFileReader extends StoreFileReader {
protected final Cell splitCell;
- private Cell firstKey = null;
+ private Optional<Cell> firstKey = null;
private boolean firstKeySeeked = false;
@@ -258,8 +259,8 @@ public class HalfStoreFileReader extends StoreFileReader {
@Override
public boolean seekBefore(Cell key) throws IOException {
if (top) {
- Cell fk = getFirstKey();
- if (getComparator().compareKeyIgnoresMvcc(key, fk) <= 0) {
+ Optional<Cell> fk = getFirstKey();
+ if (getComparator().compareKeyIgnoresMvcc(key, fk.get()) <= 0) {
return false;
}
} else {
@@ -303,7 +304,7 @@ public class HalfStoreFileReader extends StoreFileReader {
}
@Override
- public Cell getLastKey() {
+ public Optional<Cell> getLastKey() {
if (top) {
return super.getLastKey();
}
@@ -311,7 +312,7 @@ public class HalfStoreFileReader extends StoreFileReader {
HFileScanner scanner = getScanner(true, true);
try {
if (scanner.seekBefore(this.splitCell)) {
- return scanner.getKey();
+ return Optional.ofNullable(scanner.getKey());
}
} catch (IOException e) {
LOG.warn("Failed seekBefore " + Bytes.toStringBinary(this.splitkey), e);
@@ -320,22 +321,22 @@ public class HalfStoreFileReader extends StoreFileReader {
scanner.close();
}
}
- return null;
+ return Optional.empty();
}
@Override
- public Cell midkey() throws IOException {
+ public Optional<Cell> midKey() throws IOException {
// Returns null to indicate file is not splitable.
- return null;
+ return Optional.empty();
}
@Override
- public Cell getFirstKey() {
+ public Optional<Cell> getFirstKey() {
if (!firstKeySeeked) {
HFileScanner scanner = getScanner(true, true, false);
try {
if (scanner.seekTo()) {
- this.firstKey = scanner.getKey();
+ this.firstKey = Optional.ofNullable(scanner.getKey());
}
firstKeySeeked = true;
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index 3d6cdaf..9cf00b3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -31,6 +31,7 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
@@ -434,21 +435,21 @@ public class HFile {
Map<byte[], byte[]> loadFileInfo() throws IOException;
- Cell getLastKey();
+ Optional<Cell> getLastKey();
- Cell midkey() throws IOException;
+ Optional<Cell> midKey() throws IOException;
long length();
long getEntries();
- Cell getFirstKey();
+ Optional<Cell> getFirstKey();
long indexSize();
- byte[] getFirstRowKey();
+ Optional<byte[]> getFirstRowKey();
- byte[] getLastRowKey();
+ Optional<byte[]> getLastRowKey();
FixedFileTrailer getTrailer();
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
index dcfffb8..4524350 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
@@ -544,7 +544,7 @@ public class HFilePrettyPrinter extends Configured implements Tool {
}
try {
- out.println("Mid-key: " + (CellUtil.getCellKeyAsString(reader.midkey())));
+ out.println("Mid-key: " + reader.midKey().map(CellUtil::getCellKeyAsString));
} catch (Exception e) {
out.println ("Unable to retrieve the midkey");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
index bf722de..381279a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import java.security.Key;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@@ -338,14 +339,12 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
}
}
- private String toStringFirstKey() {
- if(getFirstKey() == null)
- return null;
- return CellUtil.getCellKeyAsString(getFirstKey());
+ private Optional<String> toStringFirstKey() {
+ return getFirstKey().map(CellUtil::getCellKeyAsString);
}
- private String toStringLastKey() {
- return CellUtil.toString(getLastKey(), false);
+ private Optional<String> toStringLastKey() {
+ return getLastKey().map(CellUtil::getCellKeyAsString);
}
@Override
@@ -382,12 +381,12 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
* first KeyValue.
*/
@Override
- public Cell getFirstKey() {
+ public Optional<Cell> getFirstKey() {
if (dataBlockIndexReader == null) {
throw new BlockIndexNotLoadedException();
}
- return dataBlockIndexReader.isEmpty() ? null
- : dataBlockIndexReader.getRootBlockKey(0);
+ return dataBlockIndexReader.isEmpty() ? Optional.empty()
+ : Optional.of(dataBlockIndexReader.getRootBlockKey(0));
}
/**
@@ -397,10 +396,9 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
* @return the first row key, or null if the file is empty.
*/
@Override
- public byte[] getFirstRowKey() {
- Cell firstKey = getFirstKey();
+ public Optional<byte[]> getFirstRowKey() {
// We have to copy the row part to form the row key alone
- return firstKey == null? null: CellUtil.cloneRow(firstKey);
+ return getFirstKey().map(CellUtil::cloneRow);
}
/**
@@ -410,9 +408,9 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
* @return the last row key, or null if the file is empty.
*/
@Override
- public byte[] getLastRowKey() {
- Cell lastKey = getLastKey();
- return lastKey == null? null: CellUtil.cloneRow(lastKey);
+ public Optional<byte[]> getLastRowKey() {
+ // We have to copy the row part to form the row key alone
+ return getLastKey().map(CellUtil::cloneRow);
}
/** @return number of KV entries in this HFile */
@@ -1550,8 +1548,8 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
* key
*/
@Override
- public Cell getLastKey() {
- return dataBlockIndexReader.isEmpty() ? null : lastKeyCell;
+ public Optional<Cell> getLastKey() {
+ return dataBlockIndexReader.isEmpty() ? Optional.empty() : Optional.of(lastKeyCell);
}
/**
@@ -1560,8 +1558,8 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
* @throws IOException
*/
@Override
- public Cell midkey() throws IOException {
- return dataBlockIndexReader.midkey();
+ public Optional<Cell> midKey() throws IOException {
+ return Optional.ofNullable(dataBlockIndexReader.midkey());
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
index 98cd16d..f67aa5b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java
@@ -18,12 +18,14 @@
package org.apache.hadoop.hbase.master.assignment;
-import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
-
import java.io.IOException;
import java.io.InterruptedIOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -41,7 +43,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
@@ -58,19 +60,21 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.quotas.QuotaExceededException;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.SplitTableRegionState;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.SplitTableRegionState;
/**
* The procedure to split a region in a table.
@@ -636,8 +640,8 @@ public class SplitTableRegionProcedure
}
}
- private Pair<Path, Path> splitStoreFile(final HRegionFileSystem regionFs,
- final byte[] family, final StoreFile sf) throws IOException {
+ private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, byte[] family, HStoreFile sf)
+ throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("pid=" + getProcId() + " splitting started for store file: " +
sf.getPath() + " for region: " + getParentRegion().getShortNameToLog());
@@ -663,7 +667,7 @@ public class SplitTableRegionProcedure
private class StoreFileSplitter implements Callable<Pair<Path,Path>> {
private final HRegionFileSystem regionFs;
private final byte[] family;
- private final StoreFile sf;
+ private final HStoreFile sf;
/**
* Constructor that takes what it needs to split
@@ -671,8 +675,7 @@ public class SplitTableRegionProcedure
* @param family Family that contains the store file
* @param sf which file
*/
- public StoreFileSplitter(final HRegionFileSystem regionFs, final byte[] family,
- final StoreFile sf) {
+ public StoreFileSplitter(HRegionFileSystem regionFs, byte[] family, HStoreFile sf) {
this.regionFs = regionFs;
this.sf = sf;
this.family = family;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index c1fef92..957c182 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.master.balancer;
import com.google.common.annotations.VisibleForTesting;
+
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
@@ -42,7 +43,6 @@ import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.Action;
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.MoveRegi
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.SwapRegionsAction;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.shaded.com.google.common.base.Optional;
import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
@@ -1241,7 +1242,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
/**
* Compute a cost of a potential cluster configuration based upon where
- * {@link org.apache.hadoop.hbase.regionserver.StoreFile}s are located.
+ * {@link org.apache.hadoop.hbase.regionserver.HStoreFile}s are located.
*/
static abstract class LocalityBasedCostFunction extends CostFunction {
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java
index 9dc32be..397570c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java
@@ -24,11 +24,10 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* Cached mob file.
@@ -39,7 +38,7 @@ public class CachedMobFile extends MobFile implements Comparable<CachedMobFile>
private long accessCount;
private AtomicLong referenceCount = new AtomicLong(0);
- public CachedMobFile(StoreFile sf) {
+ public CachedMobFile(HStoreFile sf) {
super(sf);
}
@@ -47,7 +46,7 @@ public class CachedMobFile extends MobFile implements Comparable<CachedMobFile>
CacheConfig cacheConf) throws IOException {
// XXX: primaryReplica is only used for constructing the key of block cache so it is not a
// critical problem if we pass the wrong value, so here we always pass true. Need to fix later.
- StoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true);
+ HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true);
return new CachedMobFile(sf);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
index 1badeb2..e8ada97 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
@@ -91,7 +91,7 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
}
};
- public DefaultMobStoreCompactor(Configuration conf, Store store) {
+ public DefaultMobStoreCompactor(Configuration conf, HStore store) {
super(conf, store);
// The mob cells reside in the mob-enabled column family which is held by HMobStore.
// During the compaction, the compactor reads the cells from the mob files and
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
index bb0301e..bef73f2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
@@ -32,19 +32,19 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
import org.apache.hadoop.hbase.regionserver.HMobStore;
+import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.MemStoreSnapshot;
import org.apache.hadoop.hbase.regionserver.ScannerContext;
-import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
import org.apache.hadoop.hbase.regionserver.throttle.ThroughputControlUtil;
import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.util.StringUtils;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* An implementation of the StoreFlusher. It extends the DefaultStoreFlusher.
@@ -70,8 +70,11 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
private Path targetPath;
private HMobStore mobStore;
- public DefaultMobStoreFlusher(Configuration conf, Store store) throws IOException {
+ public DefaultMobStoreFlusher(Configuration conf, HStore store) throws IOException {
super(conf, store);
+ if (!(store instanceof HMobStore)) {
+ throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
+ }
mobCellValueSizeThreshold = store.getColumnFamilyDescriptor().getMobThreshold();
this.targetPath = MobUtils.getMobFamilyPath(conf, store.getTableName(),
store.getColumnFamilyName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java
index 929bfd2..91702c1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java
@@ -26,12 +26,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* The mob file.
@@ -39,13 +38,13 @@ import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
@InterfaceAudience.Private
public class MobFile {
- private StoreFile sf;
+ private HStoreFile sf;
// internal use only for sub classes
protected MobFile() {
}
- protected MobFile(StoreFile sf) {
+ protected MobFile(HStoreFile sf) {
this.sf = sf;
}
@@ -56,7 +55,7 @@ public class MobFile {
* @throws IOException
*/
public StoreFileScanner getScanner() throws IOException {
- List<StoreFile> sfs = new ArrayList<>();
+ List<HStoreFile> sfs = new ArrayList<>();
sfs.add(sf);
List<StoreFileScanner> sfScanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true,
false, false, sf.getMaxMemstoreTS());
@@ -86,7 +85,7 @@ public class MobFile {
public Cell readCell(Cell search, boolean cacheMobBlocks, long readPt) throws IOException {
Cell result = null;
StoreFileScanner scanner = null;
- List<StoreFile> sfs = new ArrayList<>();
+ List<HStoreFile> sfs = new ArrayList<>();
sfs.add(sf);
try {
List<StoreFileScanner> sfScanners = StoreFileScanner.getScannersForStoreFiles(sfs,
@@ -129,7 +128,7 @@ public class MobFile {
*/
public void close() throws IOException {
if (sf != null) {
- sf.closeReader(false);
+ sf.closeStoreFile(false);
sf = null;
}
}
@@ -147,7 +146,7 @@ public class MobFile {
throws IOException {
// XXX: primaryReplica is only used for constructing the key of block cache so it is not a
// critical problem if we pass the wrong value, so here we always pass true. Need to fix later.
- StoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true);
+ HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true);
return new MobFile(sf);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java
index bdd336f..ee1fe7d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java
@@ -20,9 +20,9 @@ package org.apache.hadoop.hbase.mob;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
-import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.HStore;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* MobStoreEngine creates the mob specific compactor, and store flusher.
@@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.regionserver.Store;
public class MobStoreEngine extends DefaultStoreEngine {
@Override
- protected void createStoreFlusher(Configuration conf, Store store) throws IOException {
+ protected void createStoreFlusher(Configuration conf, HStore store) throws IOException {
// When using MOB, we use DefaultMobStoreFlusher always
// Just use the compactor and compaction policy as that in DefaultStoreEngine. We can have MOB
// specific compactor and policy when that is implemented.
@@ -42,7 +42,7 @@ public class MobStoreEngine extends DefaultStoreEngine {
* Creates the DefaultMobCompactor.
*/
@Override
- protected void createCompactor(Configuration conf, Store store) throws IOException {
+ protected void createCompactor(Configuration conf, HStore store) throws IOException {
compactor = new DefaultMobStoreCompactor(conf, store);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index 2592b72..54f1373 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.TagType;
import org.apache.hadoop.hbase.TagUtil;
import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.MobCompactPartitionPolicy;
import org.apache.hadoop.hbase.client.Scan;
@@ -70,7 +69,6 @@ import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ChecksumType;
@@ -78,6 +76,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.hadoop.hbase.util.Threads;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* The mob utilities
@@ -315,7 +314,7 @@ public final class MobUtils {
// no file found
return;
}
- List<StoreFile> filesToClean = new ArrayList<>();
+ List<HStoreFile> filesToClean = new ArrayList<>();
int deletedFileCount = 0;
for (FileStatus file : stats) {
String fileName = file.getPath().getName();
@@ -467,7 +466,7 @@ public final class MobUtils {
* @throws IOException
*/
public static void removeMobFiles(Configuration conf, FileSystem fs, TableName tableName,
- Path tableDir, byte[] family, Collection<StoreFile> storeFiles) throws IOException {
+ Path tableDir, byte[] family, Collection<HStoreFile> storeFiles) throws IOException {
HFileArchiver.archiveStoreFiles(conf, fs, getMobRegionInfo(tableName), tableDir, family,
storeFiles);
}
@@ -721,7 +720,7 @@ public final class MobUtils {
*/
private static void validateMobFile(Configuration conf, FileSystem fs, Path path,
CacheConfig cacheConfig, boolean primaryReplica) throws IOException {
- StoreFile storeFile = null;
+ HStoreFile storeFile = null;
try {
storeFile = new HStoreFile(fs, path, conf, cacheConfig, BloomType.NONE, primaryReplica);
storeFile.initReader();
@@ -730,7 +729,7 @@ public final class MobUtils {
throw e;
} finally {
if (storeFile != null) {
- storeFile.closeReader(false);
+ storeFile.closeStoreFile(false);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
index 7916779..aaf545b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
@@ -25,10 +25,10 @@ import java.util.List;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* An implementation of {@link MobCompactionRequest} that is used in
@@ -106,8 +106,7 @@ public class PartitionedMobCompactionRequest extends MobCompactionRequest {
* Set start key of this partition, only if the input startKey is less than
* the current start key.
*/
- public void setStartKey(final byte[] startKey)
- {
+ public void setStartKey(final byte[] startKey) {
if ((this.startKey == null) || (Bytes.compareTo(startKey, this.startKey) < 0)) {
this.startKey = startKey;
}
@@ -227,7 +226,7 @@ public class PartitionedMobCompactionRequest extends MobCompactionRequest {
*/
protected static class CompactionDelPartition {
private List<Path> delFiles = new ArrayList<Path>();
- private List<StoreFile> storeFiles = new ArrayList<>();
+ private List<HStoreFile> storeFiles = new ArrayList<>();
private CompactionDelPartitionId id;
public CompactionDelPartition(CompactionDelPartitionId id) {
@@ -241,11 +240,11 @@ public class PartitionedMobCompactionRequest extends MobCompactionRequest {
void addDelFile(FileStatus file) {
delFiles.add(file.getPath());
}
- public void addStoreFile(final StoreFile file) {
+ public void addStoreFile(HStoreFile file) {
storeFiles.add(file);
}
- public List<StoreFile> getStoreFiles() {
+ public List<HStoreFile> getStoreFiles() {
return storeFiles;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
index e40f3a7..1fc2902 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
@@ -18,6 +18,10 @@
*/
package org.apache.hadoop.hbase.mob.compactions;
+import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TIME_KEY;
+import static org.apache.hadoop.hbase.regionserver.HStoreFile.MOB_CELLS_COUNT;
+import static org.apache.hadoop.hbase.regionserver.HStoreFile.SKIP_RESET_SEQ_ID;
+
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
@@ -50,7 +54,6 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.TagType;
import org.apache.hadoop.hbase.TagUtil;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
@@ -75,7 +78,6 @@ import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.ScanInfo;
import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.ScannerContext;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
@@ -85,6 +87,7 @@ import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
@@ -225,8 +228,8 @@ public class PartitionedMobCompactor extends MobCompactor {
// Get delId from the file
try (Reader reader = HFile.createReader(fs, linkedFile.getPath(), conf)) {
- delId.setStartKey(reader.getFirstRowKey());
- delId.setEndKey(reader.getLastRowKey());
+ delId.setStartKey(reader.getFirstRowKey().get());
+ delId.setEndKey(reader.getLastRowKey().get());
}
CompactionDelPartition delPartition = delFilesToCompact.get(delId);
if (delPartition == null) {
@@ -266,8 +269,8 @@ public class PartitionedMobCompactor extends MobCompactor {
// get startKey and endKey from the file and update partition
// TODO: is it possible to skip read of most hfiles?
try (Reader reader = HFile.createReader(fs, linkedFile.getPath(), conf)) {
- compactionPartition.setStartKey(reader.getFirstRowKey());
- compactionPartition.setEndKey(reader.getLastRowKey());
+ compactionPartition.setStartKey(reader.getFirstRowKey().get());
+ compactionPartition.setEndKey(reader.getLastRowKey().get());
}
}
@@ -335,7 +338,7 @@ public class PartitionedMobCompactor extends MobCompactor {
try {
for (CompactionDelPartition delPartition : request.getDelPartitions()) {
for (Path newDelPath : delPartition.listDelFiles()) {
- StoreFile sf =
+ HStoreFile sf =
new HStoreFile(fs, newDelPath, conf, compactionCacheConfig, BloomType.NONE, true);
// pre-create reader of a del file to avoid race condition when opening the reader in each
// partition.
@@ -361,7 +364,8 @@ public class PartitionedMobCompactor extends MobCompactor {
for (CompactionDelPartition delPartition : request.getDelPartitions()) {
LOG.info(delPartition.listDelFiles());
try {
- MobUtils.removeMobFiles(conf, fs, tableName, mobTableDir, column.getName(), delPartition.getStoreFiles());
+ MobUtils.removeMobFiles(conf, fs, tableName, mobTableDir, column.getName(),
+ delPartition.getStoreFiles());
} catch (IOException e) {
LOG.error("Failed to archive the del files " + delPartition.getStoreFiles(), e);
}
@@ -397,11 +401,11 @@ public class PartitionedMobCompactor extends MobCompactor {
}
@VisibleForTesting
- List<StoreFile> getListOfDelFilesForPartition(final CompactionPartition partition,
+ List<HStoreFile> getListOfDelFilesForPartition(final CompactionPartition partition,
final List<CompactionDelPartition> delPartitions) {
// Binary search for startKey and endKey
- List<StoreFile> result = new ArrayList<>();
+ List<HStoreFile> result = new ArrayList<>();
DelPartitionComparator comparator = new DelPartitionComparator(false);
CompactionDelPartitionId id = new CompactionDelPartitionId(null, partition.getStartKey());
@@ -473,7 +477,7 @@ public class PartitionedMobCompactor extends MobCompactor {
// Search the delPartitions and collect all the delFiles for the partition
// One optimization can do is that if there is no del file, we do not need to
// come up with startKey/endKey.
- List<StoreFile> delFiles = getListOfDelFilesForPartition(partition,
+ List<HStoreFile> delFiles = getListOfDelFilesForPartition(partition,
request.getDelPartitions());
results.put(partition.getPartitionId(), pool.submit(new Callable<List<Path>>() {
@@ -521,7 +525,7 @@ public class PartitionedMobCompactor extends MobCompactor {
*/
private List<Path> compactMobFilePartition(PartitionedMobCompactionRequest request,
CompactionPartition partition,
- List<StoreFile> delFiles,
+ List<HStoreFile> delFiles,
Connection connection,
Table table) throws IOException {
if (MobUtils.isMobFileExpired(column, EnvironmentEdgeManager.currentTime(),
@@ -550,9 +554,9 @@ public class PartitionedMobCompactor extends MobCompactor {
// clean the bulkload directory to avoid loading old files.
fs.delete(bulkloadPathOfPartition, true);
// add the selected mob files and del files into filesToCompact
- List<StoreFile> filesToCompact = new ArrayList<>();
+ List<HStoreFile> filesToCompact = new ArrayList<>();
for (int i = offset; i < batch + offset; i++) {
- StoreFile sf = new HStoreFile(fs, files.get(i).getPath(), conf, compactionCacheConfig,
+ HStoreFile sf = new HStoreFile(fs, files.get(i).getPath(), conf, compactionCacheConfig,
BloomType.NONE, true);
filesToCompact.add(sf);
}
@@ -572,10 +576,10 @@ public class PartitionedMobCompactor extends MobCompactor {
* Closes the readers of store files.
* @param storeFiles The store files to be closed.
*/
- private void closeStoreFileReaders(List<StoreFile> storeFiles) {
- for (StoreFile storeFile : storeFiles) {
+ private void closeStoreFileReaders(List<HStoreFile> storeFiles) {
+ for (HStoreFile storeFile : storeFiles) {
try {
- storeFile.closeReader(true);
+ storeFile.closeStoreFile(true);
} catch (IOException e) {
LOG.warn("Failed to close the reader on store file " + storeFile.getPath(), e);
}
@@ -600,14 +604,14 @@ public class PartitionedMobCompactor extends MobCompactor {
private void compactMobFilesInBatch(PartitionedMobCompactionRequest request,
CompactionPartition partition,
Connection connection, Table table,
- List<StoreFile> filesToCompact, int batch,
+ List<HStoreFile> filesToCompact, int batch,
Path bulkloadPathOfPartition, Path bulkloadColumnPath,
List<Path> newFiles)
throws IOException {
// open scanner to the selected mob files and del files.
StoreScanner scanner = createScanner(filesToCompact, ScanType.COMPACT_DROP_DELETES);
// the mob files to be compacted, not include the del files.
- List<StoreFile> mobFilesToCompact = filesToCompact.subList(0, batch);
+ List<HStoreFile> mobFilesToCompact = filesToCompact.subList(0, batch);
// Pair(maxSeqId, cellsCount)
Pair<Long, Long> fileInfo = getFileInfo(mobFilesToCompact);
// open writers for the mob files and new ref store files.
@@ -726,7 +730,7 @@ public class PartitionedMobCompactor extends MobCompactor {
if (delFilePaths.size() - offset < compactionBatchSize) {
batch = delFilePaths.size() - offset;
}
- List<StoreFile> batchedDelFiles = new ArrayList<>();
+ List<HStoreFile> batchedDelFiles = new ArrayList<>();
if (batch == 1) {
// only one file left, do not compact it, directly add it to the new files.
paths.add(delFilePaths.get(offset));
@@ -753,7 +757,7 @@ public class PartitionedMobCompactor extends MobCompactor {
* @throws IOException if IO failure is encountered
*/
private Path compactDelFilesInBatch(PartitionedMobCompactionRequest request,
- List<StoreFile> delFiles) throws IOException {
+ List<HStoreFile> delFiles) throws IOException {
// create a scanner for the del files.
StoreScanner scanner = createScanner(delFiles, ScanType.COMPACT_RETAIN_DELETES);
StoreFileWriter writer = null;
@@ -803,7 +807,7 @@ public class PartitionedMobCompactor extends MobCompactor {
* @return The store scanner.
* @throws IOException if IO failure is encountered
*/
- private StoreScanner createScanner(List<StoreFile> filesToCompact, ScanType scanType)
+ private StoreScanner createScanner(List<HStoreFile> filesToCompact, ScanType scanType)
throws IOException {
List<StoreFileScanner> scanners = StoreFileScanner.getScannersForStoreFiles(filesToCompact,
false, true, false, false, HConstants.LATEST_TIMESTAMP);
@@ -864,8 +868,8 @@ public class PartitionedMobCompactor extends MobCompactor {
throws IOException {
if (writer != null) {
writer.appendMetadata(maxSeqId, false);
- writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(bulkloadTime));
- writer.appendFileInfo(StoreFile.SKIP_RESET_SEQ_ID, Bytes.toBytes(true));
+ writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(bulkloadTime));
+ writer.appendFileInfo(SKIP_RESET_SEQ_ID, Bytes.toBytes(true));
try {
writer.close();
} catch (IOException e) {
@@ -880,14 +884,14 @@ public class PartitionedMobCompactor extends MobCompactor {
* @return The pair of the max seqId and number of cells of the store files.
* @throws IOException if IO failure is encountered
*/
- private Pair<Long, Long> getFileInfo(List<StoreFile> storeFiles) throws IOException {
+ private Pair<Long, Long> getFileInfo(List<HStoreFile> storeFiles) throws IOException {
long maxSeqId = 0;
long maxKeyCount = 0;
- for (StoreFile sf : storeFiles) {
+ for (HStoreFile sf : storeFiles) {
// the readers will be closed later after the merge.
maxSeqId = Math.max(maxSeqId, sf.getMaxSequenceId());
sf.initReader();
- byte[] count = sf.getReader().loadFileInfo().get(StoreFile.MOB_CELLS_COUNT);
+ byte[] count = sf.getReader().loadFileInfo().get(MOB_CELLS_COUNT);
if (count != null) {
maxKeyCount += Bytes.toLong(count);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
index 07b72e2..065fd37 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
@@ -16,11 +16,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import java.util.List;
+
import org.apache.yetus.audience.InterfaceAudience;
/**
@@ -40,5 +40,5 @@ public interface ChangedReadersObserver {
* @param memStoreScanners scanner of current memstore
* @throws IOException e
*/
- void updateReaders(List<StoreFile> sfs, List<KeyValueScanner> memStoreScanners) throws IOException;
+ void updateReaders(List<HStoreFile> sfs, List<KeyValueScanner> memStoreScanners) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index c31a1cf..da502c2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -69,7 +69,7 @@ public class CompactingMemStore extends AbstractMemStore {
private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT = 0.25;
private static final Log LOG = LogFactory.getLog(CompactingMemStore.class);
- private Store store;
+ private HStore store;
private RegionServicesForStores regionServices;
private CompactionPipeline pipeline;
private MemStoreCompactor compactor;
@@ -341,7 +341,7 @@ public class CompactingMemStore extends AbstractMemStore {
return store.getSmallestReadPoint();
}
- public Store getStore() {
+ public HStore getStore() {
return store;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
index 25e1609..f7c18f9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hbase.security.User;
public class DateTieredStoreEngine extends StoreEngine<DefaultStoreFlusher,
DateTieredCompactionPolicy, DateTieredCompactor, DefaultStoreFileManager> {
@Override
- public boolean needsCompaction(List<StoreFile> filesCompacting) {
+ public boolean needsCompaction(List<HStoreFile> filesCompacting) {
return compactionPolicy.needsCompaction(storeFileManager.getStorefiles(),
filesCompacting);
}
@@ -54,7 +54,7 @@ public class DateTieredStoreEngine extends StoreEngine<DefaultStoreFlusher,
}
@Override
- protected void createComponents(Configuration conf, Store store, CellComparator kvComparator)
+ protected void createComponents(Configuration conf, HStore store, CellComparator kvComparator)
throws IOException {
this.compactionPolicy = new DateTieredCompactionPolicy(conf, store);
this.storeFileManager =
@@ -67,13 +67,13 @@ public class DateTieredStoreEngine extends StoreEngine<DefaultStoreFlusher,
private final class DateTieredCompactionContext extends CompactionContext {
@Override
- public List<StoreFile> preSelect(List<StoreFile> filesCompacting) {
+ public List<HStoreFile> preSelect(List<HStoreFile> filesCompacting) {
return compactionPolicy.preSelectCompactionForCoprocessor(storeFileManager.getStorefiles(),
filesCompacting);
}
@Override
- public boolean select(List<StoreFile> filesCompacting, boolean isUserCompaction,
+ public boolean select(List<HStoreFile> filesCompacting, boolean isUserCompaction,
boolean mayUseOffPeak, boolean forceMajor) throws IOException {
request = compactionPolicy.selectCompaction(storeFileManager.getStorefiles(), filesCompacting,
isUserCompaction, mayUseOffPeak, forceMajor);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f84430/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java
index 70421e1..58f8bbb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy;
@@ -33,6 +32,7 @@ import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPoli
import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.ReflectionUtils;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* Default StoreEngine creates the default compactor, policy, and store file manager, or
@@ -57,14 +57,14 @@ public class DefaultStoreEngine extends StoreEngine<
DEFAULT_COMPACTION_POLICY_CLASS = ExploringCompactionPolicy.class;
@Override
- public boolean needsCompaction(List<StoreFile> filesCompacting) {
+ public boolean needsCompaction(List<HStoreFile> filesCompacting) {
return compactionPolicy.needsCompaction(
this.storeFileManager.getStorefiles(), filesCompacting);
}
@Override
protected void createComponents(
- Configuration conf, Store store, CellComparator kvComparator) throws IOException {
+ Configuration conf, HStore store, CellComparator kvComparator) throws IOException {
createCompactor(conf, store);
createCompactionPolicy(conf, store);
createStoreFlusher(conf, store);
@@ -73,17 +73,17 @@ public class DefaultStoreEngine extends StoreEngine<
compactionPolicy.getConf());
}
- protected void createCompactor(Configuration conf, Store store) throws IOException {
+ protected void createCompactor(Configuration conf, HStore store) throws IOException {
String className = conf.get(DEFAULT_COMPACTOR_CLASS_KEY, DEFAULT_COMPACTOR_CLASS.getName());
try {
compactor = ReflectionUtils.instantiateWithCustomCtor(className,
- new Class[] { Configuration.class, Store.class }, new Object[] { conf, store });
+ new Class[] { Configuration.class, HStore.class }, new Object[] { conf, store });
} catch (Exception e) {
throw new IOException("Unable to load configured compactor '" + className + "'", e);
}
}
- protected void createCompactionPolicy(Configuration conf, Store store) throws IOException {
+ protected void createCompactionPolicy(Configuration conf, HStore store) throws IOException {
String className = conf.get(
DEFAULT_COMPACTION_POLICY_CLASS_KEY, DEFAULT_COMPACTION_POLICY_CLASS.getName());
try {
@@ -95,12 +95,12 @@ public class DefaultStoreEngine extends StoreEngine<
}
}
- protected void createStoreFlusher(Configuration conf, Store store) throws IOException {
+ protected void createStoreFlusher(Configuration conf, HStore store) throws IOException {
String className = conf.get(
DEFAULT_STORE_FLUSHER_CLASS_KEY, DEFAULT_STORE_FLUSHER_CLASS.getName());
try {
storeFlusher = ReflectionUtils.instantiateWithCustomCtor(className,
- new Class[] { Configuration.class, Store.class }, new Object[] { conf, store });
+ new Class[] { Configuration.class, HStore.class }, new Object[] { conf, store });
} catch (Exception e) {
throw new IOException("Unable to load configured store flusher '" + className + "'", e);
}
@@ -113,7 +113,7 @@ public class DefaultStoreEngine extends StoreEngine<
private class DefaultCompactionContext extends CompactionContext {
@Override
- public boolean select(List<StoreFile> filesCompacting, boolean isUserCompaction,
+ public boolean select(List<HStoreFile> filesCompacting, boolean isUserCompaction,
boolean mayUseOffPeak, boolean forceMajor) throws IOException {
request = compactionPolicy.selectCompaction(storeFileManager.getStorefiles(),
filesCompacting, isUserCompaction, mayUseOffPeak, forceMajor);
@@ -127,7 +127,7 @@ public class DefaultStoreEngine extends StoreEngine<
}
@Override
- public List<StoreFile> preSelect(List<StoreFile> filesCompacting) {
+ public List<HStoreFile> preSelect(List<HStoreFile> filesCompacting) {
return compactionPolicy.preSelectCompactionForCoprocessor(
storeFileManager.getStorefiles(), filesCompacting);
}