You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2015/05/20 01:14:45 UTC
[50/50] [abbrv] hbase git commit: Merge remote-tracking branch
'apache/master' (5/19/15) into hbase-11339
Merge remote-tracking branch 'apache/master' (5/19/15) into hbase-11339
Patches that caused deltas:
HBASE-10810 - around HColumnDiscriptor should vs is api.
HBASE-11677 - LOG made private
HBASE-11927 - Checksum constant changed
HBASE-10800 - CellComparator instead of KVComparator
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c5644742
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c5644742
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c5644742
Branch: refs/heads/hbase-11339
Commit: c56447421764329edbc4ef007dd63fa6d3fdca87
Parents: bbc7b87 74e5b26
Author: Jonathan M Hsieh <jm...@apache.org>
Authored: Tue May 19 14:10:31 2015 -0700
Committer: Jonathan M Hsieh <jm...@apache.org>
Committed: Tue May 19 14:10:48 2015 -0700
----------------------------------------------------------------------
.gitignore | 1 +
bin/draining_servers.rb | 2 +-
bin/hbase-daemon.sh | 2 +-
bin/hbase.cmd | 12 +-
bin/region_mover.rb | 69 +-
bin/region_status.rb | 4 +-
conf/hbase-env.cmd | 4 +
conf/hbase-env.sh | 4 +
dev-support/check_compatibility.sh | 2 +-
dev-support/test-patch.properties | 4 +-
dev-support/test-patch.sh | 2 +-
hbase-assembly/pom.xml | 3 -
.../src/main/assembly/hadoop-two-compat.xml | 18 +-
hbase-assembly/src/main/assembly/src.xml | 26 +
.../org/apache/hadoop/hbase/ClusterStatus.java | 20 -
.../apache/hadoop/hbase/HColumnDescriptor.java | 312 ++-----
.../org/apache/hadoop/hbase/HRegionInfo.java | 59 +-
.../apache/hadoop/hbase/HTableDescriptor.java | 6 +-
.../org/apache/hadoop/hbase/ServerLoad.java | 4 +-
.../org/apache/hadoop/hbase/client/Admin.java | 160 +++-
.../hadoop/hbase/client/AsyncProcess.java | 2 +-
.../client/ClientAsyncPrefetchScanner.java | 234 ++++++
.../hadoop/hbase/client/ClientIdGenerator.java | 2 +-
.../hadoop/hbase/client/ClientScanner.java | 143 +++-
.../hbase/client/ClientSimpleScanner.java | 54 ++
.../hadoop/hbase/client/ClientSmallScanner.java | 17 +-
.../hbase/client/ConnectionImplementation.java | 4 +-
.../apache/hadoop/hbase/client/HBaseAdmin.java | 785 ++++++++++--------
.../org/apache/hadoop/hbase/client/HTable.java | 41 +-
.../hadoop/hbase/client/HTableInterface.java | 10 +-
.../apache/hadoop/hbase/client/MetaCache.java | 7 +-
.../client/PreemptiveFastFailInterceptor.java | 2 +-
.../hbase/client/RegionServerCallable.java | 2 +-
.../org/apache/hadoop/hbase/client/Result.java | 13 +-
.../hadoop/hbase/client/ResultScanner.java | 6 +
.../hbase/client/ReversedClientScanner.java | 2 +-
.../hbase/client/RpcRetryingCallerImpl.java | 1 +
.../RpcRetryingCallerWithReadReplicas.java | 2 +-
.../org/apache/hadoop/hbase/client/Scan.java | 30 +-
.../hadoop/hbase/client/ScannerCallable.java | 26 +-
.../client/ScannerCallableWithReplicas.java | 12 +-
.../hadoop/hbase/client/TableConfiguration.java | 24 +-
.../client/UnmodifyableHTableDescriptor.java | 7 +-
.../hadoop/hbase/client/ZooKeeperRegistry.java | 2 +-
.../apache/hadoop/hbase/filter/FilterList.java | 6 +-
.../hadoop/hbase/filter/FuzzyRowFilter.java | 5 +-
.../hbase/filter/MultiRowRangeFilter.java | 10 +-
.../apache/hadoop/hbase/filter/ParseFilter.java | 6 +-
.../hbase/filter/SingleColumnValueFilter.java | 2 +-
.../hadoop/hbase/ipc/AbstractRpcClient.java | 3 +-
.../org/apache/hadoop/hbase/ipc/AsyncCall.java | 4 +-
.../hadoop/hbase/ipc/AsyncRpcChannel.java | 2 +-
.../apache/hadoop/hbase/ipc/AsyncRpcClient.java | 7 +-
.../hbase/ipc/AsyncServerResponseHandler.java | 4 +-
.../hadoop/hbase/ipc/CoprocessorRpcChannel.java | 2 +-
.../org/apache/hadoop/hbase/ipc/IPCUtil.java | 1 +
.../hbase/ipc/MasterCoprocessorRpcChannel.java | 2 +-
.../hbase/ipc/RegionCoprocessorRpcChannel.java | 2 +-
.../ipc/RegionServerCoprocessorRpcChannel.java | 2 +-
.../apache/hadoop/hbase/ipc/RpcClientImpl.java | 4 +
.../hadoop/hbase/protobuf/ProtobufUtil.java | 2 +-
.../hadoop/hbase/protobuf/RequestConverter.java | 3 +
.../hbase/protobuf/ResponseConverter.java | 22 +-
.../hbase/security/HBaseSaslRpcClient.java | 2 +-
.../hbase/security/SaslClientHandler.java | 4 +-
.../hbase/security/SecureBulkLoadUtil.java | 2 +-
.../hbase/security/access/TablePermission.java | 2 +-
.../hbase/security/access/UserPermission.java | 2 +-
.../token/AuthenticationTokenSelector.java | 2 +-
.../hadoop/hbase/security/token/TokenUtil.java | 2 +-
.../security/visibility/VisibilityClient.java | 212 +++--
.../hbase/zookeeper/MasterAddressTracker.java | 30 +-
.../hbase/zookeeper/RecoverableZooKeeper.java | 6 +-
.../hadoop/hbase/zookeeper/ZKLeaderManager.java | 2 +-
.../hbase/zookeeper/ZooKeeperNodeTracker.java | 3 +-
.../hadoop/hbase/TestHTableDescriptor.java | 2 +-
.../hbase/TestInterfaceAudienceAnnotations.java | 27 +-
.../hadoop/hbase/client/TestAsyncProcess.java | 3 +
.../hbase/client/TestClientNoCluster.java | 4 +-
.../hadoop/hbase/client/TestClientScanner.java | 10 +
.../client/TestClientSmallReversedScanner.java | 32 +-
.../hbase/client/TestClientSmallScanner.java | 34 +-
.../hadoop/hbase/client/TestOperation.java | 9 +-
.../hbase/client/TestProcedureFuture.java | 17 +-
.../apache/hadoop/hbase/ipc/TestIPCUtil.java | 2 +-
.../java/org/apache/hadoop/hbase/AuthUtil.java | 2 +-
.../org/apache/hadoop/hbase/CellComparator.java | 827 ++++++++++++-------
.../java/org/apache/hadoop/hbase/CellUtil.java | 146 +++-
.../org/apache/hadoop/hbase/ChoreService.java | 8 +-
.../org/apache/hadoop/hbase/HConstants.java | 17 +-
.../java/org/apache/hadoop/hbase/KeyValue.java | 50 +-
.../apache/hadoop/hbase/KeyValueTestUtil.java | 2 +-
.../org/apache/hadoop/hbase/KeyValueUtil.java | 123 ++-
.../org/apache/hadoop/hbase/NoTagsKeyValue.java | 37 +
.../org/apache/hadoop/hbase/ScheduledChore.java | 8 +-
.../java/org/apache/hadoop/hbase/TableName.java | 6 +-
.../hadoop/hbase/codec/KeyValueCodec.java | 3 +-
.../hbase/codec/KeyValueCodecWithTags.java | 3 +-
.../hadoop/hbase/io/BoundedByteBufferPool.java | 4 +-
.../hadoop/hbase/io/compress/Compression.java | 2 +-
.../io/encoding/BufferedDataBlockEncoder.java | 162 ++--
.../io/encoding/CopyKeyDataBlockEncoder.java | 4 +-
.../hbase/io/encoding/DataBlockEncoder.java | 10 +-
.../hbase/io/encoding/DiffKeyDeltaEncoder.java | 4 +-
.../hbase/io/encoding/FastDiffDeltaEncoder.java | 4 +-
.../io/encoding/PrefixKeyDeltaEncoder.java | 4 +-
.../hadoop/hbase/io/hfile/HFileContext.java | 3 +-
.../hbase/io/hfile/HFileContextBuilder.java | 3 +-
.../hadoop/hbase/util/ByteBufferArray.java | 2 +-
.../org/apache/hadoop/hbase/util/Bytes.java | 222 ++++-
.../apache/hadoop/hbase/util/ChecksumType.java | 95 +--
.../org/apache/hadoop/hbase/util/ClassSize.java | 2 +-
.../java/org/apache/hadoop/hbase/util/JVM.java | 9 +
.../org/apache/hadoop/hbase/util/Methods.java | 2 +-
.../org/apache/hadoop/hbase/util/Sleeper.java | 2 +-
.../org/apache/hadoop/hbase/util/Threads.java | 2 +-
.../hbase/util/test/RedundantKVGenerator.java | 3 +-
.../src/main/resources/hbase-default.xml | 28 +-
.../apache/hadoop/hbase/TestCellComparator.java | 100 +--
.../apache/hadoop/hbase/TestChoreService.java | 764 +++++++++--------
.../org/apache/hadoop/hbase/TestKeyValue.java | 147 +---
.../hadoop/hbase/codec/TestCellCodec.java | 7 +-
.../hbase/codec/TestCellCodecWithTags.java | 8 +-
.../hbase/codec/TestKeyValueCodecWithTags.java | 8 +-
.../io/crypto/TestKeyStoreKeyProvider.java | 2 +-
.../org/apache/hadoop/hbase/util/TestBytes.java | 41 +-
.../apache/hadoop/hbase/types/TestPBCell.java | 3 +-
.../hbase/ipc/MetricsHBaseServerSource.java | 22 +
.../hbase/master/MetricsMasterSource.java | 4 +-
.../MetricsHBaseServerSourceFactoryImpl.java | 6 +-
.../hbase/ipc/MetricsHBaseServerSourceImpl.java | 71 +-
.../hbase/master/MetricsMasterSourceImpl.java | 3 +-
.../hadoop/hbase/IntegrationTestIngest.java | 1 +
.../IntegrationTestIngestWithEncryption.java | 6 +-
...tegrationTestIngestWithVisibilityLabels.java | 4 +-
.../hbase/IntegrationTestMetaReplicas.java | 2 +-
.../hadoop/hbase/chaos/actions/Action.java | 2 +-
.../actions/RollingBatchRestartRsAction.java | 2 +-
.../hadoop/hbase/chaos/policies/Policy.java | 2 +-
.../mapreduce/IntegrationTestImportTsv.java | 7 +-
.../test/IntegrationTestBigLinkedList.java | 2 +-
...egrationTestBigLinkedListWithVisibility.java | 21 +-
...tionTestWithCellVisibilityLoadAndVerify.java | 16 +-
.../hbase/codec/prefixtree/PrefixTreeCodec.java | 18 +-
.../codec/prefixtree/PrefixTreeSeeker.java | 12 +-
.../decode/PrefixTreeArrayScanner.java | 4 +-
.../codec/prefixtree/decode/PrefixTreeCell.java | 25 +-
.../codec/prefixtree/row/BaseTestRowData.java | 4 +-
.../prefixtree/row/TestPrefixTreeSearcher.java | 6 +-
.../row/data/TestRowDataNumberStrings.java | 2 +-
.../row/data/TestRowDataSearcherRowMiss.java | 14 +-
.../prefixtree/row/data/TestRowDataSimple.java | 14 +-
.../hbase/procedure2/OnePhaseProcedure.java | 2 +-
.../procedure2/ProcedureAbortedException.java | 2 +-
.../hbase/procedure2/ProcedureException.java | 2 +-
.../hbase/procedure2/ProcedureExecutor.java | 68 +-
.../hbase/procedure2/ProcedureResult.java | 2 +-
.../procedure2/ProcedureYieldException.java | 2 +-
.../procedure2/RemoteProcedureException.java | 2 +-
.../hbase/procedure2/SequentialProcedure.java | 2 +-
.../hbase/procedure2/StateMachineProcedure.java | 2 +-
.../hbase/procedure2/TwoPhaseProcedure.java | 2 +-
.../hbase/procedure2/store/ProcedureStore.java | 2 +-
.../procedure2/store/ProcedureStoreTracker.java | 21 +-
.../CorruptedWALProcedureStoreException.java | 2 +-
.../procedure2/store/wal/WALProcedureStore.java | 134 ++-
.../hbase/procedure2/util/StringUtils.java | 4 +
.../store/TestProcedureStoreTracker.java | 47 ++
.../hbase/protobuf/generated/ClientProtos.java | 349 ++++++--
.../protobuf/generated/ClusterStatusProtos.java | 100 +--
.../hbase/protobuf/generated/MasterProtos.java | 589 ++++++++-----
.../apache/hadoop/hbase/util/ByteStringer.java | 2 +-
hbase-protocol/src/main/protobuf/Client.proto | 7 +
.../src/main/protobuf/ClusterStatus.proto | 4 +-
hbase-protocol/src/main/protobuf/Master.proto | 2 +
.../apache/hadoop/hbase/rest/RESTServer.java | 1 +
.../apache/hadoop/hbase/rest/RESTServlet.java | 2 +-
.../hadoop/hbase/rest/SchemaResource.java | 4 +-
.../hadoop/hbase/rest/client/RemoteHTable.java | 5 +
.../rest/model/StorageClusterStatusModel.java | 30 +-
.../generated/StorageClusterStatusMessage.java | 96 +--
.../protobuf/StorageClusterStatusMessage.proto | 4 +-
.../apache/hadoop/hbase/rest/DummyFilter.java | 2 +-
.../hbase/rest/HBaseRESTTestingUtility.java | 2 +-
.../hbase/rest/TestScannersWithLabels.java | 10 +-
.../hadoop/hbase/rest/TestStatusResource.java | 2 +-
hbase-server/pom.xml | 1 +
.../tmpl/master/BackupMasterStatusTmpl.jamon | 51 +-
.../hbase/tmpl/master/MasterStatusTmpl.jamon | 5 +-
.../tmpl/regionserver/RegionListTmpl.jamon | 6 +-
.../apache/hadoop/hbase/HealthCheckChore.java | 2 +-
.../org/apache/hadoop/hbase/HealthChecker.java | 2 +-
.../org/apache/hadoop/hbase/JMXListener.java | 18 +-
.../apache/hadoop/hbase/LocalHBaseCluster.java | 2 +-
.../apache/hadoop/hbase/SplitLogCounters.java | 5 +-
.../org/apache/hadoop/hbase/TagRewriteCell.java | 5 +
.../apache/hadoop/hbase/ZKNamespaceManager.java | 2 +-
.../org/apache/hadoop/hbase/ZNodeClearer.java | 2 +-
.../hbase/client/ClientSideRegionScanner.java | 9 +-
.../hbase/client/TableSnapshotScanner.java | 5 +
.../hadoop/hbase/conf/ConfigurationManager.java | 2 +-
.../ForeignExceptionDispatcher.java | 2 +-
.../hadoop/hbase/http/HttpRequestLog.java | 2 +-
.../apache/hadoop/hbase/http/HttpServer.java | 13 +-
.../hadoop/hbase/io/HalfStoreFileReader.java | 26 +-
.../hadoop/hbase/io/hfile/ChecksumUtil.java | 110 +--
.../hadoop/hbase/io/hfile/FixedFileTrailer.java | 76 +-
.../org/apache/hadoop/hbase/io/hfile/HFile.java | 31 +-
.../hadoop/hbase/io/hfile/HFileBlockIndex.java | 50 +-
.../hbase/io/hfile/HFilePrettyPrinter.java | 14 +-
.../hadoop/hbase/io/hfile/HFileReaderImpl.java | 270 ++++--
.../hbase/io/hfile/HFileWriterFactory.java | 4 +-
.../hadoop/hbase/io/hfile/HFileWriterImpl.java | 141 +++-
.../hadoop/hbase/io/hfile/LruBlockCache.java | 2 +-
.../hbase/io/hfile/bucket/BucketAllocator.java | 2 +-
.../hbase/io/hfile/bucket/BucketCache.java | 2 +-
.../org/apache/hadoop/hbase/ipc/CallRunner.java | 14 +-
.../hadoop/hbase/ipc/MetricsHBaseServer.java | 34 +
.../apache/hadoop/hbase/ipc/QosPriority.java | 32 +
.../hadoop/hbase/ipc/RWQueueRpcExecutor.java | 3 +
.../org/apache/hadoop/hbase/ipc/RpcServer.java | 23 +-
.../hadoop/hbase/ipc/SimpleRpcScheduler.java | 2 +-
.../hbase/mapred/TableRecordReaderImpl.java | 2 +-
.../hbase/mapreduce/HFileOutputFormat.java | 2 +-
.../hbase/mapreduce/HFileOutputFormat2.java | 9 +-
.../hbase/mapreduce/KeyValueSerialization.java | 5 +-
.../hbase/mapreduce/KeyValueSortReducer.java | 3 +-
.../hbase/mapreduce/LoadIncrementalHFiles.java | 82 +-
.../mapreduce/MultiTableInputFormatBase.java | 2 +-
.../hadoop/hbase/mapreduce/PutSortReducer.java | 3 +-
.../mapreduce/SimpleTotalOrderPartitioner.java | 2 +-
.../hbase/mapreduce/TableInputFormatBase.java | 2 +-
.../hbase/mapreduce/TableMapReduceUtil.java | 7 +-
.../hbase/mapreduce/TableRecordReaderImpl.java | 2 +-
.../hadoop/hbase/mapreduce/TableSplit.java | 24 +-
.../hadoop/hbase/mapreduce/TextSortReducer.java | 3 +-
.../hadoop/hbase/mapreduce/WALPlayer.java | 2 +-
.../hadoop/hbase/master/AssignmentManager.java | 90 +-
.../master/AssignmentVerificationReport.java | 2 +-
.../hbase/master/GeneralBulkAssigner.java | 29 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 26 +-
.../hadoop/hbase/master/HMasterCommandLine.java | 27 +-
.../hadoop/hbase/master/MasterRpcServices.java | 15 +-
.../hadoop/hbase/master/MasterServices.java | 4 +-
.../hadoop/hbase/master/MetricsMaster.java | 4 +-
.../apache/hadoop/hbase/master/RackManager.java | 2 +-
.../hbase/master/balancer/FavoredNodesPlan.java | 2 +-
.../master/balancer/RegionLocationFinder.java | 2 +-
.../hbase/master/cleaner/HFileLinkCleaner.java | 13 +-
.../hadoop/hbase/master/cleaner/LogCleaner.java | 2 +-
.../master/cleaner/TimeToLiveHFileCleaner.java | 2 +-
.../master/cleaner/TimeToLiveLogCleaner.java | 2 +-
.../master/handler/ModifyTableHandler.java | 149 ----
.../master/handler/TableAddFamilyHandler.java | 89 --
.../handler/TableDeleteFamilyHandler.java | 90 --
.../handler/TableModifyFamilyHandler.java | 84 --
.../master/handler/TruncateTableHandler.java | 142 ----
.../master/procedure/CreateTableProcedure.java | 5 +
.../master/procedure/DeleteTableProcedure.java | 14 +-
.../master/procedure/ProcedureSyncWait.java | 12 +
.../hbase/master/snapshot/SnapshotManager.java | 37 +-
.../org/apache/hadoop/hbase/mob/MobUtils.java | 23 +-
.../PartitionedMobFileCompactor.java | 11 +-
.../hbase/namespace/NamespaceAuditor.java | 18 +-
.../hbase/namespace/NamespaceStateManager.java | 29 +-
.../hadoop/hbase/procedure/ProcedureMember.java | 22 +-
.../hadoop/hbase/procedure/Subprocedure.java | 9 +-
.../procedure/ZKProcedureCoordinatorRpcs.java | 2 +-
.../hbase/procedure/ZKProcedureMemberRpcs.java | 18 +-
.../hadoop/hbase/quotas/MasterQuotaManager.java | 6 +
.../AnnotationReadingPriorityFunction.java | 52 +-
.../hbase/regionserver/CellSkipListSet.java | 4 +-
.../hbase/regionserver/CompactSplitThread.java | 5 +-
.../hbase/regionserver/DefaultMemStore.java | 7 +-
.../hbase/regionserver/DefaultStoreEngine.java | 4 +-
.../regionserver/DefaultStoreFileManager.java | 8 +-
.../hbase/regionserver/DefaultStoreFlusher.java | 2 +-
.../GetClosestRowBeforeTracker.java | 17 +-
.../hadoop/hbase/regionserver/HMobStore.java | 20 +-
.../hadoop/hbase/regionserver/HRegion.java | 257 ++++--
.../hbase/regionserver/HRegionFileSystem.java | 50 +-
.../hbase/regionserver/HRegionServer.java | 32 +-
.../hadoop/hbase/regionserver/HStore.java | 27 +-
...IncreasingToUpperBoundRegionSplitPolicy.java | 2 +-
.../hadoop/hbase/regionserver/KeyValueHeap.java | 12 +-
.../hadoop/hbase/regionserver/LogRoller.java | 2 +-
.../hadoop/hbase/regionserver/LruHashMap.java | 2 +-
.../hbase/regionserver/MemStoreFlusher.java | 2 +-
.../MetricsRegionServerWrapperImpl.java | 2 +-
.../regionserver/NoLimitScannerContext.java | 28 +-
.../hbase/regionserver/RSDumpServlet.java | 22 +-
.../hbase/regionserver/RSRpcServices.java | 189 ++++-
.../hadoop/hbase/regionserver/Region.java | 4 +
.../regionserver/RegionCoprocessorHost.java | 8 +-
.../hbase/regionserver/RegionMergeRequest.java | 2 +-
.../hbase/regionserver/RegionSplitPolicy.java | 11 +
.../regionserver/ReversedKeyValueHeap.java | 10 +-
.../regionserver/ReversedRegionScannerImpl.java | 6 +-
.../regionserver/ReversedStoreScanner.java | 9 +-
.../hadoop/hbase/regionserver/ScanInfo.java | 10 +-
.../hbase/regionserver/ScanQueryMatcher.java | 45 +-
.../hbase/regionserver/ScannerContext.java | 128 ++-
.../hadoop/hbase/regionserver/SplitRequest.java | 2 +-
.../apache/hadoop/hbase/regionserver/Store.java | 4 +-
.../hadoop/hbase/regionserver/StoreEngine.java | 8 +-
.../hadoop/hbase/regionserver/StoreFile.java | 137 +--
.../hbase/regionserver/StoreFileInfo.java | 7 +-
.../hbase/regionserver/StoreFileScanner.java | 22 +-
.../hadoop/hbase/regionserver/StoreScanner.java | 41 +-
.../regionserver/StripeMultiFileWriter.java | 60 +-
.../hbase/regionserver/StripeStoreConfig.java | 2 +-
.../hbase/regionserver/StripeStoreEngine.java | 6 +-
.../regionserver/StripeStoreFileManager.java | 47 +-
.../hbase/regionserver/StripeStoreFlusher.java | 2 +-
.../compactions/CompactionConfiguration.java | 2 +-
.../compactions/CompactionRequest.java | 2 +-
.../regionserver/compactions/Compactor.java | 2 +-
.../compactions/DefaultCompactor.java | 11 +-
.../compactions/RatioBasedCompactionPolicy.java | 4 +-
.../compactions/StripeCompactionPolicy.java | 2 +-
.../compactions/StripeCompactor.java | 6 +-
.../hadoop/hbase/regionserver/wal/FSHLog.java | 8 +-
.../hadoop/hbase/regionserver/wal/HLogKey.java | 2 +-
.../hbase/regionserver/wal/MetricsWAL.java | 2 +-
.../regionserver/wal/MetricsWALEditsReplay.java | 2 +-
.../regionserver/wal/ProtobufLogWriter.java | 2 +-
.../hbase/regionserver/wal/WALCellCodec.java | 3 +-
.../hadoop/hbase/regionserver/wal/WALEdit.java | 11 +-
.../hadoop/hbase/regionserver/wal/WALUtil.java | 2 +-
.../HBaseInterClusterReplicationEndpoint.java | 1 +
.../replication/regionserver/MetricsSource.java | 2 +-
.../regionserver/ReplicationSinkManager.java | 10 +
.../regionserver/ReplicationSource.java | 2 +-
.../regionserver/ReplicationSyncUp.java | 2 +-
.../hbase/security/HBaseSaslRpcServer.java | 2 +-
.../security/access/AccessControlLists.java | 2 +-
.../hbase/security/access/AccessController.java | 4 +-
.../HbaseObjectWritableFor96Migration.java | 2 +-
.../security/access/SecureBulkLoadEndpoint.java | 2 +-
.../hbase/security/access/TableAuthManager.java | 2 +-
.../security/access/ZKPermissionWatcher.java | 2 +-
.../token/AuthenticationTokenSecretManager.java | 2 +-
.../hbase/security/token/TokenProvider.java | 2 +-
.../hbase/security/token/ZKSecretWatcher.java | 2 +-
.../visibility/VisibilityController.java | 3 +-
.../hbase/snapshot/SnapshotReferenceUtil.java | 2 +-
.../apache/hadoop/hbase/util/BloomFilter.java | 61 +-
.../hadoop/hbase/util/BloomFilterBase.java | 13 -
.../hadoop/hbase/util/BloomFilterChunk.java | 322 ++++++++
.../hadoop/hbase/util/BloomFilterFactory.java | 12 +-
.../hadoop/hbase/util/BloomFilterUtil.java | 269 ++++++
.../hadoop/hbase/util/BloomFilterWriter.java | 4 -
.../util/BoundedConcurrentLinkedQueue.java | 122 +++
.../hadoop/hbase/util/ByteBloomFilter.java | 662 ---------------
.../hbase/util/CollectionBackedScanner.java | 14 +-
.../hadoop/hbase/util/CompoundBloomFilter.java | 54 +-
.../hbase/util/CompoundBloomFilterBase.java | 33 +-
.../hbase/util/CompoundBloomFilterWriter.java | 39 +-
.../hadoop/hbase/util/CompressionTest.java | 4 +-
.../hadoop/hbase/util/ConnectionCache.java | 3 +-
.../hadoop/hbase/util/EncryptionTest.java | 2 +-
.../org/apache/hadoop/hbase/util/FSUtils.java | 61 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 85 +-
.../hadoop/hbase/util/HBaseFsckRepair.java | 8 +-
.../org/apache/hadoop/hbase/util/HMerge.java | 11 +-
.../org/apache/hadoop/hbase/util/Merge.java | 2 +-
.../hadoop/hbase/util/RegionSizeCalculator.java | 9 +-
.../hbase/util/RegionSplitCalculator.java | 3 +-
.../hadoop/hbase/util/RegionSplitter.java | 4 +-
.../org/apache/hadoop/hbase/wal/WALFactory.java | 11 +-
.../org/apache/hadoop/hbase/wal/WALKey.java | 2 +-
.../apache/hadoop/hbase/wal/WALSplitter.java | 2 +-
.../native/src/mlockall_agent/mlockall_agent.c | 64 +-
.../hbase-webapps/master/tablesDetailed.jsp | 13 +-
.../org/apache/hadoop/hbase/HBaseCluster.java | 1 +
.../hadoop/hbase/HBaseTestingUtility.java | 2 +-
.../hbase/HFilePerformanceEvaluation.java | 4 +-
.../hbase/MapFilePerformanceEvaluation.java | 2 +-
.../apache/hadoop/hbase/MetaMockingUtil.java | 2 +-
.../apache/hadoop/hbase/MiniHBaseCluster.java | 2 +-
.../hadoop/hbase/MultithreadedTestUtil.java | 2 +-
.../hadoop/hbase/PerformanceEvaluation.java | 2 +-
.../hbase/PerformanceEvaluationCommons.java | 2 +-
.../hadoop/hbase/TestGlobalMemStoreSize.java | 2 +-
.../hadoop/hbase/TestHBaseTestingUtility.java | 2 +-
.../TestHColumnDescriptorDefaultVersions.java | 8 +-
.../org/apache/hadoop/hbase/TestIOFencing.java | 2 +-
.../apache/hadoop/hbase/TestInfoServers.java | 2 +-
.../hadoop/hbase/TestMovedRegionsCleaner.java | 2 +-
.../org/apache/hadoop/hbase/TestNamespace.java | 2 +-
.../hbase/TestPartialResultsFromClientSide.java | 3 +-
.../hadoop/hbase/TestRegionRebalancing.java | 2 +-
.../apache/hadoop/hbase/TestSerialization.java | 8 +-
.../hadoop/hbase/TestTableDescriptor.java | 2 +-
.../apache/hadoop/hbase/TestTagRewriteCell.java | 48 ++
.../org/apache/hadoop/hbase/TestZooKeeper.java | 2 +-
.../hadoop/hbase/backup/TestHFileArchiving.java | 2 +-
.../apache/hadoop/hbase/client/TestAdmin1.java | 32 +-
.../apache/hadoop/hbase/client/TestAdmin2.java | 3 +-
.../client/TestClientScannerRPCTimeout.java | 2 +-
.../hadoop/hbase/client/TestClientTimeouts.java | 2 +-
.../client/TestCloneSnapshotFromClient.java | 3 +-
.../hadoop/hbase/client/TestFastFail.java | 9 +-
.../hadoop/hbase/client/TestFromClientSide.java | 39 +-
.../hbase/client/TestFromClientSide3.java | 28 +-
.../hbase/client/TestHTableMultiplexer.java | 2 +-
.../client/TestHTableMultiplexerFlushCache.java | 2 +-
.../hbase/client/TestMetaWithReplicas.java | 2 +-
.../hbase/client/TestMultipleTimestamps.java | 2 +-
.../hbase/client/TestReplicaWithCluster.java | 2 +-
.../client/TestRestoreSnapshotFromClient.java | 4 +-
.../apache/hadoop/hbase/client/TestResult.java | 17 +-
.../hadoop/hbase/client/TestScannerTimeout.java | 2 +-
.../client/TestScannersFromClientSide.java | 99 +++
.../hadoop/hbase/client/TestSizeFailures.java | 2 +-
.../client/TestSnapshotCloneIndependence.java | 73 +-
.../hbase/client/TestSnapshotMetadata.java | 2 +-
.../hbase/client/TestTimestampsFilter.java | 2 +-
.../replication/TestReplicationAdmin.java | 16 +-
.../hadoop/hbase/codec/CodecPerformance.java | 2 +
.../hbase/codec/TestCellMessageCodec.java | 9 +-
.../hbase/conf/TestConfigurationManager.java | 2 +-
.../coprocessor/ColumnAggregationEndpoint.java | 2 +-
.../ColumnAggregationEndpointNullResponse.java | 2 +-
.../ColumnAggregationEndpointWithErrors.java | 2 +-
.../coprocessor/SampleRegionWALObserver.java | 2 +-
.../hbase/coprocessor/SimpleRegionObserver.java | 2 +-
.../coprocessor/TestCoprocessorEndpoint.java | 1 +
.../coprocessor/TestCoprocessorInterface.java | 3 +-
.../hbase/coprocessor/TestMasterObserver.java | 10 +-
.../TestRegionObserverInterface.java | 2 +-
.../TestRegionServerCoprocessorEndpoint.java | 38 +-
...gionServerCoprocessorExceptionWithAbort.java | 5 +-
.../coprocessor/TestRowProcessorEndpoint.java | 2 +-
.../DummyRegionServerEndpointProtos.java | 81 +-
.../hbase/filter/FilterTestingCluster.java | 10 +-
.../hbase/filter/TestColumnRangeFilter.java | 2 +-
.../hbase/filter/TestDependentColumnFilter.java | 5 +-
.../apache/hadoop/hbase/filter/TestFilter.java | 9 +-
.../hadoop/hbase/filter/TestFilterList.java | 15 +-
.../TestFuzzyRowAndColumnRangeFilter.java | 2 +-
.../hadoop/hbase/filter/TestFuzzyRowFilter.java | 6 +-
.../hbase/filter/TestMultiRowRangeFilter.java | 4 +-
.../TestSingleColumnValueExcludeFilter.java | 5 +-
.../hadoop/hbase/http/TestGlobalFilter.java | 2 +-
.../hadoop/hbase/http/TestHttpServer.java | 2 +-
.../hadoop/hbase/http/TestPathFilter.java | 2 +-
.../hadoop/hbase/http/TestServletFilter.java | 2 +-
.../hbase/http/resource/JerseyResource.java | 2 +-
.../hbase/io/TestHalfStoreFileReader.java | 7 +-
.../apache/hadoop/hbase/io/TestHeapSize.java | 2 +-
.../encoding/TestBufferedDataBlockEncoder.java | 33 +
.../hbase/io/encoding/TestChangingEncoding.java | 2 +-
.../io/encoding/TestDataBlockEncoders.java | 5 +-
.../encoding/TestLoadAndSwitchEncodeOnDisk.java | 2 +-
.../io/encoding/TestPrefixTreeEncoding.java | 15 +-
.../encoding/TestSeekToBlockWithEncoders.java | 3 +-
.../hadoop/hbase/io/hfile/TestCacheOnWrite.java | 3 +-
.../hadoop/hbase/io/hfile/TestChecksum.java | 83 +-
.../hbase/io/hfile/TestFixedFileTrailer.java | 25 +-
.../apache/hadoop/hbase/io/hfile/TestHFile.java | 158 +++-
.../hadoop/hbase/io/hfile/TestHFileBlock.java | 6 +-
.../hbase/io/hfile/TestHFileBlockIndex.java | 24 +-
.../hadoop/hbase/io/hfile/TestHFileSeek.java | 6 +-
.../hbase/io/hfile/TestHFileWriterV2.java | 6 +-
.../hbase/io/hfile/TestHFileWriterV3.java | 9 +-
.../hadoop/hbase/io/hfile/TestPrefetch.java | 3 +-
.../hadoop/hbase/io/hfile/TestReseekTo.java | 3 +-
.../hadoop/hbase/io/hfile/TestSeekTo.java | 3 +-
.../apache/hadoop/hbase/ipc/TestAsyncIPC.java | 2 +-
.../hbase/ipc/TestRpcHandlerException.java | 2 +-
.../apache/hadoop/hbase/ipc/TestRpcMetrics.java | 18 +
.../hbase/ipc/TestSimpleRpcScheduler.java | 2 +-
.../hbase/mapreduce/TestHFileOutputFormat.java | 15 +-
.../hbase/mapreduce/TestHFileOutputFormat2.java | 9 +-
.../hbase/mapreduce/TestImportExport.java | 9 +-
.../TestImportTSVWithOperationAttributes.java | 2 +-
.../TestImportTSVWithVisibilityLabels.java | 8 +-
.../hadoop/hbase/mapreduce/TestImportTsv.java | 2 +-
.../TestLoadIncrementalHFilesSplitRecovery.java | 4 +-
.../mapreduce/TestMultiTableInputFormat.java | 2 +-
.../hadoop/hbase/mapreduce/TestRowCounter.java | 2 +-
.../mapreduce/TestTableInputFormatScanBase.java | 2 +-
.../hbase/mapreduce/TestWALRecordReader.java | 2 +-
.../hadoop/hbase/master/TestCatalogJanitor.java | 11 +-
.../master/TestDistributedLogSplitting.java | 3 +-
.../TestMasterOperationsForRegionReplicas.java | 2 +-
.../hbase/master/TestMasterPriorityRpc.java | 94 +++
.../hadoop/hbase/master/TestMasterShutdown.java | 4 +-
.../hbase/master/TestRegionPlacement.java | 2 +-
.../hbase/master/TestRegionPlacement2.java | 2 +-
.../hadoop/hbase/master/TestRollingRestart.java | 2 +
.../hbase/master/TestSplitLogManager.java | 8 +-
.../hbase/master/TestTableLockManager.java | 2 +-
.../hadoop/hbase/master/TestWarmupRegion.java | 2 +-
.../handler/TestTableDeleteFamilyHandler.java | 6 +-
.../TestTableDescriptorModification.java | 16 +-
.../TestPartitionedMobFileCompactor.java | 9 +-
.../hbase/namespace/TestNamespaceAuditor.java | 193 +++++
.../hbase/procedure/TestProcedureManager.java | 2 +-
.../procedure/TestZKProcedureControllers.java | 2 +-
.../hadoop/hbase/quotas/TestQuotaAdmin.java | 4 +-
.../hadoop/hbase/quotas/TestQuotaTableUtil.java | 4 +-
.../hbase/regionserver/KeyValueScanFixture.java | 12 +-
.../hbase/regionserver/MockStoreFile.java | 4 +-
.../hbase/regionserver/TestAtomicOperation.java | 2 +-
.../hbase/regionserver/TestBlocksRead.java | 2 +-
.../hbase/regionserver/TestCellSkipListSet.java | 3 +-
.../hbase/regionserver/TestColumnSeeking.java | 2 +-
.../hbase/regionserver/TestCompaction.java | 2 +-
.../hbase/regionserver/TestCompactionState.java | 2 +-
.../regionserver/TestCompoundBloomFilter.java | 32 +-
.../TestCorruptedRegionStoreFile.java | 250 ++++++
.../hbase/regionserver/TestDefaultMemStore.java | 16 +-
.../regionserver/TestDefaultStoreEngine.java | 4 +-
.../regionserver/TestEncryptionKeyRotation.java | 2 +-
.../regionserver/TestFlushRegionEntry.java | 37 +-
.../regionserver/TestGetClosestAtOrBefore.java | 4 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 14 +-
.../regionserver/TestHRegionReplayEvents.java | 82 +-
.../regionserver/TestHRegionServerBulkLoad.java | 2 +-
.../hbase/regionserver/TestJoinedScanners.java | 2 +-
.../hbase/regionserver/TestKeyValueHeap.java | 9 +-
.../regionserver/TestKeyValueScanFixture.java | 3 +-
.../hbase/regionserver/TestMajorCompaction.java | 2 +-
.../regionserver/TestMasterAddressTracker.java | 107 ++-
.../hbase/regionserver/TestMinorCompaction.java | 2 +-
.../hbase/regionserver/TestMobCompaction.java | 10 +-
.../regionserver/TestMultiColumnScanner.java | 4 +-
.../hbase/regionserver/TestParallelPut.java | 2 +-
.../hbase/regionserver/TestQosFunction.java | 54 +-
.../hbase/regionserver/TestQueryMatcher.java | 6 +-
.../hbase/regionserver/TestRecoveredEdits.java | 6 +-
.../regionserver/TestRegionFavoredNodes.java | 5 +-
.../regionserver/TestRegionReplicaFailover.java | 1 +
.../regionserver/TestRegionServerHostname.java | 8 +-
.../TestRegionServerReportForDuty.java | 18 +-
.../regionserver/TestRegionSplitPolicy.java | 4 +-
.../regionserver/TestReversibleScanners.java | 5 +-
.../hadoop/hbase/regionserver/TestScanner.java | 2 +-
.../TestScannerHeartbeatMessages.java | 539 ++++++++++++
.../TestScannerRetriableFailure.java | 171 ++++
.../regionserver/TestSeekOptimizations.java | 5 +-
.../TestSplitTransactionOnCluster.java | 22 +-
.../hadoop/hbase/regionserver/TestStore.java | 16 +-
.../hbase/regionserver/TestStoreFile.java | 30 +-
.../hbase/regionserver/TestStoreScanner.java | 13 +-
.../hbase/regionserver/TestStripeCompactor.java | 6 +-
.../regionserver/TestStripeStoreEngine.java | 4 +-
.../TestStripeStoreFileManager.java | 5 +-
.../hadoop/hbase/regionserver/TestTags.java | 2 +-
.../hbase/regionserver/TestWideScanner.java | 2 +-
.../compactions/TestStripeCompactionPolicy.java | 15 +-
.../regionserver/wal/SequenceFileLogWriter.java | 2 +-
.../hbase/regionserver/wal/TestFSHLog.java | 2 +-
.../hbase/regionserver/wal/TestProtobufLog.java | 2 +-
.../wal/TestWALActionsListener.java | 4 +-
.../hbase/regionserver/wal/TestWALReplay.java | 2 +-
.../replication/TestMasterReplication.java | 13 +-
.../replication/TestReplicationEndpoint.java | 2 +-
.../TestReplicationWALEntryFilters.java | 3 +-
.../TestReplicationSinkManager.java | 30 +-
.../TestReplicationSourceManager.java | 2 +-
.../regionserver/TestReplicationThrottler.java | 27 +-
.../apache/hadoop/hbase/security/TestUser.java | 2 +-
.../security/access/TestAccessController.java | 7 +-
.../security/token/TestTokenAuthentication.java | 4 +-
.../security/token/TestZKSecretWatcher.java | 2 +-
.../TestDefaultScanLabelGeneratorStack.java | 6 +-
.../TestEnforcingScanLabelGenerator.java | 6 +-
.../visibility/TestVisibilityLabels.java | 36 +-
...sibilityLabelsOpWithDifferentUsersNoACL.java | 23 +-
.../TestVisibilityLabelsReplication.java | 10 +-
.../visibility/TestVisibilityLabelsWithACL.java | 43 +-
...ibilityLabelsWithDefaultVisLabelService.java | 21 +-
.../TestVisibilityLabelsWithDeletes.java | 18 +-
.../TestVisibilityLabelsWithSLGStack.java | 6 +-
.../TestVisibilityLablesWithGroups.java | 19 +-
.../TestVisibilityWithCheckAuths.java | 12 +-
.../TestWithDisabledAuthorization.java | 27 +-
.../hbase/snapshot/TestExportSnapshot.java | 3 +-
.../TestRestoreFlushSnapshotFromClient.java | 2 +-
.../snapshot/TestRestoreSnapshotHelper.java | 2 +-
.../apache/hadoop/hbase/util/LoadTestTool.java | 88 +-
.../apache/hadoop/hbase/util/MockServer.java | 2 +-
.../hadoop/hbase/util/TestBloomFilterChunk.java | 185 +++++
.../util/TestBoundedConcurrentLinkedQueue.java | 161 ++++
.../hadoop/hbase/util/TestByteBloomFilter.java | 169 ----
.../hadoop/hbase/util/TestCompressionTest.java | 2 +-
.../hadoop/hbase/util/TestConnectionCache.java | 54 ++
.../hbase/util/TestCoprocessorScanPolicy.java | 2 +-
.../apache/hadoop/hbase/util/TestFSUtils.java | 45 +-
.../apache/hadoop/hbase/util/TestFSVisitor.java | 2 +-
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 15 +-
.../apache/hadoop/hbase/util/TestMergeTool.java | 2 +-
.../util/hbck/OfflineMetaRebuildTestCore.java | 2 +-
.../util/hbck/TestOfflineMetaRebuildBase.java | 5 +-
.../util/hbck/TestOfflineMetaRebuildHole.java | 5 +-
.../hbck/TestOfflineMetaRebuildOverlap.java | 5 +-
.../hbase/wal/TestDefaultWALProvider.java | 2 +-
.../apache/hadoop/hbase/wal/TestSecureWAL.java | 2 +-
.../apache/hadoop/hbase/wal/TestWALFactory.java | 8 +-
.../hbase/wal/TestWALReaderOnSecureWAL.java | 2 +-
.../hbase/wal/WALPerformanceEvaluation.java | 2 +-
.../hbase/zookeeper/TestZKLeaderManager.java | 2 +-
.../protobuf/DummyRegionServerEndpoint.proto | 1 +
hbase-shaded/hbase-shaded-client/pom.xml | 61 ++
hbase-shaded/hbase-shaded-server/pom.xml | 61 ++
hbase-shaded/pom.xml | 121 +++
hbase-shell/src/main/ruby/hbase/admin.rb | 15 +-
.../src/main/ruby/hbase/visibility_labels.rb | 12 +-
.../src/main/ruby/shell/commands/create.rb | 6 +-
hbase-shell/src/test/ruby/hbase/admin_test.rb | 16 +
.../apache/hadoop/hbase/thrift/CallQueue.java | 2 +-
.../hbase/thrift/HbaseHandlerMetricsProxy.java | 2 +-
.../hadoop/hbase/thrift/IncrementCoalescer.java | 2 +-
.../hadoop/hbase/thrift/ThriftHttpServlet.java | 2 +-
.../hadoop/hbase/thrift/ThriftServerRunner.java | 2 +-
.../hadoop/hbase/thrift/ThriftUtilities.java | 2 +-
.../hadoop/hbase/thrift/TestCallQueue.java | 2 +-
.../hbase/thrift/TestThriftHttpServer.java | 2 +-
.../hbase/thrift/TestThriftServerCmdLine.java | 2 +-
.../thrift2/TestThriftHBaseServiceHandler.java | 2 +-
...TestThriftHBaseServiceHandlerWithLabels.java | 10 +-
pom.xml | 27 +-
.../asciidoc/_chapters/appendix_acl_matrix.adoc | 94 +--
src/main/asciidoc/_chapters/architecture.adoc | 14 +
src/main/asciidoc/_chapters/configuration.adoc | 38 +-
src/main/asciidoc/_chapters/datamodel.adoc | 2 +-
src/main/asciidoc/_chapters/developer.adoc | 20 +-
.../asciidoc/_chapters/getting_started.adoc | 10 +-
src/main/asciidoc/_chapters/hbase_apis.adoc | 109 ++-
src/main/asciidoc/_chapters/mapreduce.adoc | 27 +-
src/main/asciidoc/_chapters/ops_mgt.adoc | 187 ++++-
src/main/asciidoc/_chapters/schema_design.adoc | 96 ++-
src/main/asciidoc/_chapters/upgrading.adoc | 8 +-
636 files changed, 11702 insertions(+), 6145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5644742/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5644742/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5644742/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5644742/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5644742/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5644742/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5644742/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5644742/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5644742/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --cc hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 17c1ee3,6790c7e..4b8d6b8
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@@ -41,9 -40,9 +41,10 @@@ import org.apache.hadoop.hbase.classifi
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.exceptions.MergeRegionException;
import org.apache.hadoop.hbase.exceptions.UnknownProtocolException;
+ import org.apache.hadoop.hbase.ipc.QosPriority;
import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
+import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureResult;
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5644742/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5644742/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
----------------------------------------------------------------------
diff --cc hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index 2159597,0000000..9f08c6e
mode 100644,000000..100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@@ -1,751 -1,0 +1,742 @@@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
- import org.apache.hadoop.hbase.Cell;
- import org.apache.hadoop.hbase.HColumnDescriptor;
- import org.apache.hadoop.hbase.HConstants;
- import org.apache.hadoop.hbase.HRegionInfo;
- import org.apache.hadoop.hbase.KeyValue;
- import org.apache.hadoop.hbase.TableName;
- import org.apache.hadoop.hbase.Tag;
- import org.apache.hadoop.hbase.TagType;
++import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.compress.Compression;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileContext;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
+import org.apache.hadoop.hbase.master.TableLockManager;
+import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
+import org.apache.hadoop.hbase.mob.filecompactions.MobFileCompactor;
+import org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactor;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.regionserver.HStore;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
- import org.apache.hadoop.hbase.util.Bytes;
- import org.apache.hadoop.hbase.util.FSUtils;
- import org.apache.hadoop.hbase.util.ReflectionUtils;
- import org.apache.hadoop.hbase.util.Threads;
++import org.apache.hadoop.hbase.util.*;
+
+/**
+ * The mob utilities
+ */
+@InterfaceAudience.Private
+public class MobUtils {
+
+ private static final Log LOG = LogFactory.getLog(MobUtils.class);
+
+ private static final ThreadLocal<SimpleDateFormat> LOCAL_FORMAT =
+ new ThreadLocal<SimpleDateFormat>() {
+ @Override
+ protected SimpleDateFormat initialValue() {
+ return new SimpleDateFormat("yyyyMMdd");
+ }
+ };
+
+ /**
+ * Formats a date to a string.
+ * @param date The date.
+ * @return The string format of the date, it's yyyymmdd.
+ */
+ public static String formatDate(Date date) {
+ return LOCAL_FORMAT.get().format(date);
+ }
+
+ /**
+ * Parses the string to a date.
+ * @param dateString The string format of a date, it's yyyymmdd.
+ * @return A date.
+ * @throws ParseException
+ */
+ public static Date parseDate(String dateString) throws ParseException {
+ return LOCAL_FORMAT.get().parse(dateString);
+ }
+
+ /**
+ * Whether the current cell is a mob reference cell.
+ * @param cell The current cell.
+ * @return True if the cell has a mob reference tag, false if it doesn't.
+ */
+ public static boolean isMobReferenceCell(Cell cell) {
+ if (cell.getTagsLength() > 0) {
+ Tag tag = Tag.getTag(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength(),
+ TagType.MOB_REFERENCE_TAG_TYPE);
+ return tag != null;
+ }
+ return false;
+ }
+
+ /**
+ * Gets the table name tag.
+ * @param cell The current cell.
+ * @return The table name tag.
+ */
+ public static Tag getTableNameTag(Cell cell) {
+ if (cell.getTagsLength() > 0) {
+ Tag tag = Tag.getTag(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength(),
+ TagType.MOB_TABLE_NAME_TAG_TYPE);
+ return tag;
+ }
+ return null;
+ }
+
+ /**
+ * Whether the tag list has a mob reference tag.
+ * @param tags The tag list.
+ * @return True if the list has a mob reference tag, false if it doesn't.
+ */
+ public static boolean hasMobReferenceTag(List<Tag> tags) {
+ if (!tags.isEmpty()) {
+ for (Tag tag : tags) {
+ if (tag.getType() == TagType.MOB_REFERENCE_TAG_TYPE) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Indicates whether it's a raw scan.
+ * The information is set in the attribute "hbase.mob.scan.raw" of scan.
+ * For a mob cell, in a normal scan the scanners retrieves the mob cell from the mob file.
+ * In a raw scan, the scanner directly returns cell in HBase without retrieve the one in
+ * the mob file.
+ * @param scan The current scan.
+ * @return True if it's a raw scan.
+ */
+ public static boolean isRawMobScan(Scan scan) {
+ byte[] raw = scan.getAttribute(MobConstants.MOB_SCAN_RAW);
+ try {
+ return raw != null && Bytes.toBoolean(raw);
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Indicates whether it's a reference only scan.
+ * The information is set in the attribute "hbase.mob.scan.ref.only" of scan.
+ * If it's a ref only scan, only the cells with ref tag are returned.
+ * @param scan The current scan.
+ * @return True if it's a ref only scan.
+ */
+ public static boolean isRefOnlyScan(Scan scan) {
+ byte[] refOnly = scan.getAttribute(MobConstants.MOB_SCAN_REF_ONLY);
+ try {
+ return refOnly != null && Bytes.toBoolean(refOnly);
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Indicates whether the scan contains the information of caching blocks.
+ * The information is set in the attribute "hbase.mob.cache.blocks" of scan.
+ * @param scan The current scan.
+ * @return True when the Scan attribute specifies to cache the MOB blocks.
+ */
+ public static boolean isCacheMobBlocks(Scan scan) {
+ byte[] cache = scan.getAttribute(MobConstants.MOB_CACHE_BLOCKS);
+ try {
+ return cache != null && Bytes.toBoolean(cache);
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Sets the attribute of caching blocks in the scan.
+ *
+ * @param scan
+ * The current scan.
+ * @param cacheBlocks
+ * True, set the attribute of caching blocks into the scan, the scanner with this scan
+ * caches blocks.
+ * False, the scanner doesn't cache blocks for this scan.
+ */
+ public static void setCacheMobBlocks(Scan scan, boolean cacheBlocks) {
+ scan.setAttribute(MobConstants.MOB_CACHE_BLOCKS, Bytes.toBytes(cacheBlocks));
+ }
+
+ /**
+ * Cleans the expired mob files.
+ * Cleans the files whose creation date is older than (current - columnFamily.ttl), and
+ * the minVersions of that column family is 0.
+ * @param fs The current file system.
+ * @param conf The current configuration.
+ * @param tableName The current table name.
+ * @param columnDescriptor The descriptor of the current column family.
+ * @param cacheConfig The cacheConfig that disables the block cache.
+ * @param current The current time.
+ * @throws IOException
+ */
+ public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName,
+ HColumnDescriptor columnDescriptor, CacheConfig cacheConfig, long current)
+ throws IOException {
+ long timeToLive = columnDescriptor.getTimeToLive();
+ if (Integer.MAX_VALUE == timeToLive) {
+ // no need to clean, because the TTL is not set.
+ return;
+ }
+
+ Date expireDate = new Date(current - timeToLive * 1000);
+ expireDate = new Date(expireDate.getYear(), expireDate.getMonth(), expireDate.getDate());
+ LOG.info("MOB HFiles older than " + expireDate.toGMTString() + " will be deleted!");
+
+ FileStatus[] stats = null;
+ Path mobTableDir = FSUtils.getTableDir(getMobHome(conf), tableName);
+ Path path = getMobFamilyPath(conf, tableName, columnDescriptor.getNameAsString());
+ try {
+ stats = fs.listStatus(path);
+ } catch (FileNotFoundException e) {
+ LOG.warn("Fail to find the mob file " + path, e);
+ }
+ if (null == stats) {
+ // no file found
+ return;
+ }
+ List<StoreFile> filesToClean = new ArrayList<StoreFile>();
+ int deletedFileCount = 0;
+ for (FileStatus file : stats) {
+ String fileName = file.getPath().getName();
+ try {
+ MobFileName mobFileName = null;
+ if (!HFileLink.isHFileLink(file.getPath())) {
+ mobFileName = MobFileName.create(fileName);
+ } else {
+ HFileLink hfileLink = HFileLink.buildFromHFileLinkPattern(conf, file.getPath());
+ mobFileName = MobFileName.create(hfileLink.getOriginPath().getName());
+ }
+ Date fileDate = parseDate(mobFileName.getDate());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Checking file " + fileName);
+ }
+ if (fileDate.getTime() < expireDate.getTime()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(fileName + " is an expired file");
+ }
+ filesToClean.add(new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE));
+ }
+ } catch (Exception e) {
+ LOG.error("Cannot parse the fileName " + fileName, e);
+ }
+ }
+ if (!filesToClean.isEmpty()) {
+ try {
+ removeMobFiles(conf, fs, tableName, mobTableDir, columnDescriptor.getName(),
+ filesToClean);
+ deletedFileCount = filesToClean.size();
+ } catch (IOException e) {
+ LOG.error("Fail to delete the mob files " + filesToClean, e);
+ }
+ }
+ LOG.info(deletedFileCount + " expired mob files are deleted");
+ }
+
+ /**
+ * Gets the root dir of the mob files.
+ * It's {HBASE_DIR}/mobdir.
+ * @param conf The current configuration.
+ * @return the root dir of the mob file.
+ */
+ public static Path getMobHome(Configuration conf) {
+ Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
+ return new Path(hbaseDir, MobConstants.MOB_DIR_NAME);
+ }
+
+ /**
+ * Gets the qualified root dir of the mob files.
+ * @param conf The current configuration.
+ * @return The qualified root dir.
+ * @throws IOException
+ */
+ public static Path getQualifiedMobRootDir(Configuration conf) throws IOException {
+ Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
+ Path mobRootDir = new Path(hbaseDir, MobConstants.MOB_DIR_NAME);
+ FileSystem fs = mobRootDir.getFileSystem(conf);
+ return mobRootDir.makeQualified(fs);
+ }
+
+ /**
+ * Gets the region dir of the mob files.
+ * It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}.
+ * @param conf The current configuration.
+ * @param tableName The current table name.
+ * @return The region dir of the mob files.
+ */
+ public static Path getMobRegionPath(Configuration conf, TableName tableName) {
+ Path tablePath = FSUtils.getTableDir(getMobHome(conf), tableName);
+ HRegionInfo regionInfo = getMobRegionInfo(tableName);
+ return new Path(tablePath, regionInfo.getEncodedName());
+ }
+
+ /**
+ * Gets the family dir of the mob files.
+ * It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}/{columnFamilyName}.
+ * @param conf The current configuration.
+ * @param tableName The current table name.
+ * @param familyName The current family name.
+ * @return The family dir of the mob files.
+ */
+ public static Path getMobFamilyPath(Configuration conf, TableName tableName, String familyName) {
+ return new Path(getMobRegionPath(conf, tableName), familyName);
+ }
+
+ /**
+ * Gets the family dir of the mob files.
+ * It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}/{columnFamilyName}.
+ * @param regionPath The path of mob region which is a dummy one.
+ * @param familyName The current family name.
+ * @return The family dir of the mob files.
+ */
+ public static Path getMobFamilyPath(Path regionPath, String familyName) {
+ return new Path(regionPath, familyName);
+ }
+
+ /**
+ * Gets the HRegionInfo of the mob files.
+ * This is a dummy region. The mob files are not saved in a region in HBase.
+ * This is only used in mob snapshot. It's internally used only.
+ * @param tableName
+ * @return A dummy mob region info.
+ */
+ public static HRegionInfo getMobRegionInfo(TableName tableName) {
+ HRegionInfo info = new HRegionInfo(tableName, MobConstants.MOB_REGION_NAME_BYTES,
+ HConstants.EMPTY_END_ROW, false, 0);
+ return info;
+ }
+
+ /**
+ * Gets whether the current HRegionInfo is a mob one.
+ * @param regionInfo The current HRegionInfo.
+ * @return If true, the current HRegionInfo is a mob one.
+ */
+ public static boolean isMobRegionInfo(HRegionInfo regionInfo) {
+ return regionInfo == null ? false : getMobRegionInfo(regionInfo.getTable()).getEncodedName()
+ .equals(regionInfo.getEncodedName());
+ }
+
+ /**
+ * Gets whether the current region name follows the pattern of a mob region name.
+ * @param tableName The current table name.
+ * @param regionName The current region name.
+ * @return True if the current region name follows the pattern of a mob region name.
+ */
+ public static boolean isMobRegionName(TableName tableName, byte[] regionName) {
+ return Bytes.equals(regionName, getMobRegionInfo(tableName).getRegionName());
+ }
+
+ /**
+ * Gets the working directory of the mob compaction.
+ * @param root The root directory of the mob compaction.
+ * @param jobName The current job name.
+ * @return The directory of the mob compaction for the current job.
+ */
+ public static Path getCompactionWorkingPath(Path root, String jobName) {
+ return new Path(root, jobName);
+ }
+
+ /**
+ * Archives the mob files.
+ * @param conf The current configuration.
+ * @param fs The current file system.
+ * @param tableName The table name.
+ * @param tableDir The table directory.
+ * @param family The name of the column family.
+ * @param storeFiles The files to be deleted.
+ * @throws IOException
+ */
+ public static void removeMobFiles(Configuration conf, FileSystem fs, TableName tableName,
+ Path tableDir, byte[] family, Collection<StoreFile> storeFiles) throws IOException {
+ HFileArchiver.archiveStoreFiles(conf, fs, getMobRegionInfo(tableName), tableDir, family,
+ storeFiles);
+ }
+
+ /**
+ * Creates a mob reference KeyValue.
+ * The value of the mob reference KeyValue is mobCellValueSize + mobFileName.
+ * @param cell The original Cell.
+ * @param fileName The mob file name where the mob reference KeyValue is written.
+ * @param tableNameTag The tag of the current table name. It's very important in
+ * cloning the snapshot.
+ * @return The mob reference KeyValue.
+ */
+ public static KeyValue createMobRefKeyValue(Cell cell, byte[] fileName, Tag tableNameTag) {
+ // Append the tags to the KeyValue.
+ // The key is same, the value is the filename of the mob file
+ List<Tag> tags = new ArrayList<Tag>();
+ // Add the ref tag as the 1st one.
+ tags.add(MobConstants.MOB_REF_TAG);
+ // Add the tag of the source table name, this table is where this mob file is flushed
+ // from.
+ // It's very useful in cloning the snapshot. When reading from the cloning table, we need to
+ // find the original mob files by this table name. For details please see cloning
+ // snapshot for mob files.
+ tags.add(tableNameTag);
+ // Add the existing tags.
+ tags.addAll(Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()));
+ int valueLength = cell.getValueLength();
+ byte[] refValue = Bytes.add(Bytes.toBytes(valueLength), fileName);
+ KeyValue reference = new KeyValue(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
+ cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
+ cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
+ cell.getTimestamp(), KeyValue.Type.Put, refValue, 0, refValue.length, tags);
+ reference.setSequenceId(cell.getSequenceId());
+ return reference;
+ }
+
+ /**
+ * Creates a writer for the mob file in temp directory.
+ * @param conf The current configuration.
+ * @param fs The current file system.
+ * @param family The descriptor of the current column family.
+ * @param date The date string, its format is yyyymmmdd.
+ * @param basePath The basic path for a temp directory.
+ * @param maxKeyCount The key count.
+ * @param compression The compression algorithm.
+ * @param startKey The hex string of the start key.
+ * @param cacheConfig The current cache config.
+ * @return The writer for the mob file.
+ * @throws IOException
+ */
+ public static StoreFile.Writer createWriter(Configuration conf, FileSystem fs,
+ HColumnDescriptor family, String date, Path basePath, long maxKeyCount,
+ Compression.Algorithm compression, String startKey, CacheConfig cacheConfig)
+ throws IOException {
+ MobFileName mobFileName = MobFileName.create(startKey, date, UUID.randomUUID().toString()
+ .replaceAll("-", ""));
+ return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression,
+ cacheConfig);
+ }
+
+ /**
+ * Creates a writer for the ref file in temp directory.
+ * @param conf The current configuration.
+ * @param fs The current file system.
+ * @param family The descriptor of the current column family.
+ * @param basePath The basic path for a temp directory.
+ * @param maxKeyCount The key count.
+ * @param cacheConfig The current cache config.
+ * @return The writer for the mob file.
+ * @throws IOException
+ */
+ public static StoreFile.Writer createRefFileWriter(Configuration conf, FileSystem fs,
+ HColumnDescriptor family, Path basePath, long maxKeyCount, CacheConfig cacheConfig)
+ throws IOException {
+ HFileContext hFileContext = new HFileContextBuilder().withIncludesMvcc(true)
+ .withIncludesTags(true).withCompression(family.getCompactionCompression())
- .withCompressTags(family.shouldCompressTags()).withChecksumType(HStore.getChecksumType(conf))
++ .withCompressTags(family.isCompressTags()).withChecksumType(HStore.getChecksumType(conf))
+ .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(family.getBlocksize())
+ .withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding()).build();
+ Path tempPath = new Path(basePath, UUID.randomUUID().toString().replaceAll("-", ""));
+ StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConfig, fs).withFilePath(tempPath)
- .withComparator(KeyValue.COMPARATOR).withBloomType(family.getBloomFilterType())
++ .withComparator(CellComparator.COMPARATOR).withBloomType(family.getBloomFilterType())
+ .withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build();
+ return w;
+ }
+
+ /**
+ * Creates a writer for the mob file in temp directory.
+ * @param conf The current configuration.
+ * @param fs The current file system.
+ * @param family The descriptor of the current column family.
+ * @param date The date string, its format is yyyymmmdd.
+ * @param basePath The basic path for a temp directory.
+ * @param maxKeyCount The key count.
+ * @param compression The compression algorithm.
+ * @param startKey The start key.
+ * @param cacheConfig The current cache config.
+ * @return The writer for the mob file.
+ * @throws IOException
+ */
+ public static StoreFile.Writer createWriter(Configuration conf, FileSystem fs,
+ HColumnDescriptor family, String date, Path basePath, long maxKeyCount,
+ Compression.Algorithm compression, byte[] startKey, CacheConfig cacheConfig)
+ throws IOException {
+ MobFileName mobFileName = MobFileName.create(startKey, date, UUID.randomUUID().toString()
+ .replaceAll("-", ""));
+ return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression,
+ cacheConfig);
+ }
+
+ /**
+ * Creates a writer for the del file in temp directory.
+ * @param conf The current configuration.
+ * @param fs The current file system.
+ * @param family The descriptor of the current column family.
+ * @param date The date string, its format is yyyymmmdd.
+ * @param basePath The basic path for a temp directory.
+ * @param maxKeyCount The key count.
+ * @param compression The compression algorithm.
+ * @param startKey The start key.
+ * @param cacheConfig The current cache config.
+ * @return The writer for the del file.
+ * @throws IOException
+ */
+ public static StoreFile.Writer createDelFileWriter(Configuration conf, FileSystem fs,
+ HColumnDescriptor family, String date, Path basePath, long maxKeyCount,
+ Compression.Algorithm compression, byte[] startKey, CacheConfig cacheConfig)
+ throws IOException {
+ String suffix = UUID
+ .randomUUID().toString().replaceAll("-", "") + "_del";
+ MobFileName mobFileName = MobFileName.create(startKey, date, suffix);
+ return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression,
+ cacheConfig);
+ }
+
+ /**
+ * Creates a writer for the del file in temp directory.
+ * @param conf The current configuration.
+ * @param fs The current file system.
+ * @param family The descriptor of the current column family.
+ * @param mobFileName The mob file name.
+ * @param basePath The basic path for a temp directory.
+ * @param maxKeyCount The key count.
+ * @param compression The compression algorithm.
+ * @param cacheConfig The current cache config.
+ * @return The writer for the mob file.
+ * @throws IOException
+ */
+ private static StoreFile.Writer createWriter(Configuration conf, FileSystem fs,
+ HColumnDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount,
+ Compression.Algorithm compression, CacheConfig cacheConfig) throws IOException {
+ HFileContext hFileContext = new HFileContextBuilder().withCompression(compression)
- .withIncludesMvcc(true).withIncludesTags(true).withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
++ .withIncludesMvcc(true).withIncludesTags(true)
++ .withChecksumType(ChecksumType.getDefaultChecksumType())
+ .withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM).withBlockSize(family.getBlocksize())
+ .withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding()).build();
+
+ StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConfig, fs)
+ .withFilePath(new Path(basePath, mobFileName.getFileName()))
- .withComparator(KeyValue.COMPARATOR).withBloomType(BloomType.NONE)
++ .withComparator(CellComparator.COMPARATOR).withBloomType(BloomType.NONE)
+ .withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build();
+ return w;
+ }
+
+ /**
+ * Commits the mob file.
+ * @param conf The current configuration.
+ * @param fs The current file system.
+ * @param sourceFile The path where the mob file is saved.
+ * @param targetPath The directory path where the source file is renamed to.
+ * @param cacheConfig The current cache config.
+ * @return The target file path the source file is renamed to.
+ * @throws IOException
+ */
+ public static Path commitFile(Configuration conf, FileSystem fs, final Path sourceFile,
+ Path targetPath, CacheConfig cacheConfig) throws IOException {
+ if (sourceFile == null) {
+ return null;
+ }
+ Path dstPath = new Path(targetPath, sourceFile.getName());
+ validateMobFile(conf, fs, sourceFile, cacheConfig);
+ String msg = "Renaming flushed file from " + sourceFile + " to " + dstPath;
+ LOG.info(msg);
+ Path parent = dstPath.getParent();
+ if (!fs.exists(parent)) {
+ fs.mkdirs(parent);
+ }
+ if (!fs.rename(sourceFile, dstPath)) {
+ throw new IOException("Failed rename of " + sourceFile + " to " + dstPath);
+ }
+ return dstPath;
+ }
+
+ /**
+ * Validates a mob file by opening and closing it.
+ * @param conf The current configuration.
+ * @param fs The current file system.
+ * @param path The path where the mob file is saved.
+ * @param cacheConfig The current cache config.
+ */
+ private static void validateMobFile(Configuration conf, FileSystem fs, Path path,
+ CacheConfig cacheConfig) throws IOException {
+ StoreFile storeFile = null;
+ try {
+ storeFile = new StoreFile(fs, path, conf, cacheConfig, BloomType.NONE);
+ storeFile.createReader();
+ } catch (IOException e) {
+ LOG.error("Fail to open mob file[" + path + "], keep it in temp directory.", e);
+ throw e;
+ } finally {
+ if (storeFile != null) {
+ storeFile.closeReader(false);
+ }
+ }
+ }
+
+ /**
+ * Indicates whether the current mob ref cell has a valid value.
+ * A mob ref cell has a mob reference tag.
+ * The value of a mob ref cell consists of two parts, real mob value length and mob file name.
+ * The real mob value length takes 4 bytes.
+ * The remaining part is the mob file name.
+ * @param cell The mob ref cell.
+ * @return True if the cell has a valid value.
+ */
+ public static boolean hasValidMobRefCellValue(Cell cell) {
+ return cell.getValueLength() > Bytes.SIZEOF_INT;
+ }
+
+ /**
+ * Gets the mob value length from the mob ref cell.
+ * A mob ref cell has a mob reference tag.
+ * The value of a mob ref cell consists of two parts, real mob value length and mob file name.
+ * The real mob value length takes 4 bytes.
+ * The remaining part is the mob file name.
+ * @param cell The mob ref cell.
+ * @return The real mob value length.
+ */
+ public static int getMobValueLength(Cell cell) {
+ return Bytes.toInt(cell.getValueArray(), cell.getValueOffset(), Bytes.SIZEOF_INT);
+ }
+
+ /**
+ * Gets the mob file name from the mob ref cell.
+ * A mob ref cell has a mob reference tag.
+ * The value of a mob ref cell consists of two parts, real mob value length and mob file name.
+ * The real mob value length takes 4 bytes.
+ * The remaining part is the mob file name.
+ * @param cell The mob ref cell.
+ * @return The mob file name.
+ */
+ public static String getMobFileName(Cell cell) {
+ return Bytes.toString(cell.getValueArray(), cell.getValueOffset() + Bytes.SIZEOF_INT,
+ cell.getValueLength() - Bytes.SIZEOF_INT);
+ }
+
+ /**
+ * Gets the table name used in the table lock.
+ * The table lock name is a dummy one, it's not a table name. It's tableName + ".mobLock".
+ * @param tn The table name.
+ * @return The table name used in table lock.
+ */
+ public static TableName getTableLockName(TableName tn) {
+ byte[] tableName = tn.getName();
+ return TableName.valueOf(Bytes.add(tableName, MobConstants.MOB_TABLE_LOCK_SUFFIX));
+ }
+
+ /**
+ * Performs the mob file compaction.
+ * @param conf the Configuration
+ * @param fs the file system
+ * @param tableName the table the compact
+ * @param hcd the column descriptor
+ * @param pool the thread pool
+ * @param tableLockManager the tableLock manager
+ * @param isForceAllFiles Whether add all mob files into the compaction.
+ */
+ public static void doMobFileCompaction(Configuration conf, FileSystem fs, TableName tableName,
+ HColumnDescriptor hcd, ExecutorService pool, TableLockManager tableLockManager,
+ boolean isForceAllFiles) throws IOException {
+ String className = conf.get(MobConstants.MOB_FILE_COMPACTOR_CLASS_KEY,
+ PartitionedMobFileCompactor.class.getName());
+ // instantiate the mob file compactor.
+ MobFileCompactor compactor = null;
+ try {
+ compactor = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] {
+ Configuration.class, FileSystem.class, TableName.class, HColumnDescriptor.class,
+ ExecutorService.class }, new Object[] { conf, fs, tableName, hcd, pool });
+ } catch (Exception e) {
+ throw new IOException("Unable to load configured mob file compactor '" + className + "'", e);
+ }
+ // compact only for mob-enabled column.
+ // obtain a write table lock before performing compaction to avoid race condition
+ // with major compaction in mob-enabled column.
+ boolean tableLocked = false;
+ TableLock lock = null;
+ try {
+ // the tableLockManager might be null in testing. In that case, it is lock-free.
+ if (tableLockManager != null) {
+ lock = tableLockManager.writeLock(MobUtils.getTableLockName(tableName),
+ "Run MobFileCompaction");
+ lock.acquire();
+ }
+ tableLocked = true;
+ compactor.compact(isForceAllFiles);
+ } catch (Exception e) {
+ LOG.error("Fail to compact the mob files for the column " + hcd.getNameAsString()
+ + " in the table " + tableName.getNameAsString(), e);
+ } finally {
+ if (lock != null && tableLocked) {
+ try {
+ lock.release();
+ } catch (IOException e) {
+ LOG.error("Fail to release the write lock for the table " + tableName.getNameAsString(),
+ e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Creates a thread pool.
+ * @param conf the Configuration
+ * @return A thread pool.
+ */
+ public static ExecutorService createMobFileCompactorThreadPool(Configuration conf) {
+ int maxThreads = conf.getInt(MobConstants.MOB_FILE_COMPACTION_THREADS_MAX,
+ MobConstants.DEFAULT_MOB_FILE_COMPACTION_THREADS_MAX);
+ if (maxThreads == 0) {
+ maxThreads = 1;
+ }
+ final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, 60, TimeUnit.SECONDS, queue,
+ Threads.newDaemonThreadFactory("MobFileCompactor"), new RejectedExecutionHandler() {
+ @Override
+ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
+ try {
+ // waiting for a thread to pick up instead of throwing exceptions.
+ queue.put(r);
+ } catch (InterruptedException e) {
+ throw new RejectedExecutionException(e);
+ }
+ }
+ });
+ ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
+ return pool;
+ }
+}