You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2015/05/01 17:28:36 UTC
[50/50] [abbrv] hbase git commit: Merge branch 'apache/master'
(4/16/15) into hbase-11339
Merge branch 'apache/master' (4/16/15) into hbase-11339
API conflicts and test fixes
Update LoadTestTool.COLUMN_FAMILY -> DEFAULT_COLUMN_FAMILY due HBASE-11842
Use new 1.0+ api in some tests
Use updated Scanners internal api
Fix to take into account HBASE-13203 - procedure v2 table delete
Conflicts:
hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0e20bbf6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0e20bbf6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0e20bbf6
Branch: refs/heads/hbase-11339
Commit: 0e20bbf6a3f1b8390c040230cc8b28e777c1d1f2
Parents: eba8a70 ddab472
Author: Jonathan M Hsieh <jm...@apache.org>
Authored: Fri May 1 08:18:51 2015 -0700
Committer: Jonathan M Hsieh <jm...@apache.org>
Committed: Fri May 1 08:18:51 2015 -0700
----------------------------------------------------------------------
.gitignore | 1 +
bin/hbase | 22 +-
bin/hbase-cleanup.sh | 12 +-
bin/hbase-config.sh | 6 +-
bin/hbase.cmd | 7 +-
bin/local-master-backup.sh | 8 +-
bin/local-regionservers.sh | 8 +-
bin/region_mover.rb | 6 +-
bin/region_status.rb | 8 +-
conf/hbase-env.cmd | 5 +-
conf/hbase-env.sh | 12 +-
conf/log4j.properties | 6 +-
dev-support/check_compatibility.sh | 16 +-
dev-support/checkstyle_report.py | 0
dev-support/findHangingTests.py | 0
dev-support/jenkinsEnv.sh | 2 +-
dev-support/publish_hbase_website.sh | 32 +-
dev-support/test-patch.properties | 9 +-
dev-support/test-patch.sh | 167 +-
hbase-annotations/src/main/asciidoc/.gitignore | 0
hbase-assembly/src/main/asciidoc/.gitignore | 0
hbase-checkstyle/src/main/asciidoc/.gitignore | 0
hbase-client/pom.xml | 49 +-
hbase-client/src/main/asciidoc/.gitignore | 0
.../org/apache/hadoop/hbase/ClusterStatus.java | 67 +-
.../hadoop/hbase/CoprocessorEnvironment.java | 6 +-
.../apache/hadoop/hbase/HColumnDescriptor.java | 26 +-
.../apache/hadoop/hbase/HTableDescriptor.java | 36 +
.../apache/hadoop/hbase/MetaTableAccessor.java | 451 +-
.../org/apache/hadoop/hbase/RegionLoad.java | 10 +
.../apache/hadoop/hbase/RegionLocations.java | 28 +
.../org/apache/hadoop/hbase/ServerLoad.java | 27 +-
.../hbase/client/AbstractClientScanner.java | 15 +-
.../org/apache/hadoop/hbase/client/Admin.java | 7 +
.../hadoop/hbase/client/AsyncProcess.java | 6 +-
.../hadoop/hbase/client/BufferedMutator.java | 4 +-
.../hbase/client/BufferedMutatorImpl.java | 2 +-
.../hadoop/hbase/client/ClientScanner.java | 460 +-
.../client/ClientSmallReversedScanner.java | 182 +-
.../hadoop/hbase/client/ClientSmallScanner.java | 228 +-
.../hadoop/hbase/client/ClusterConnection.java | 8 +-
.../apache/hadoop/hbase/client/Connection.java | 13 +-
.../hadoop/hbase/client/ConnectionAdapter.java | 7 +-
.../hadoop/hbase/client/ConnectionFactory.java | 15 +-
.../hbase/client/ConnectionImplementation.java | 2248 +++
.../hadoop/hbase/client/ConnectionManager.java | 2644 ---
.../hadoop/hbase/client/ConnectionUtils.java | 9 +-
.../org/apache/hadoop/hbase/client/Delete.java | 4 +-
.../hbase/client/FlushRegionCallable.java | 102 +
.../org/apache/hadoop/hbase/client/Get.java | 1 +
.../apache/hadoop/hbase/client/HBaseAdmin.java | 890 +-
.../apache/hadoop/hbase/client/HConnection.java | 38 +-
.../hadoop/hbase/client/HConnectionKey.java | 146 -
.../hadoop/hbase/client/HConnectionManager.java | 324 -
.../hadoop/hbase/client/HRegionLocator.java | 27 +-
.../org/apache/hadoop/hbase/client/HTable.java | 198 +-
.../hadoop/hbase/client/HTableFactory.java | 51 -
.../hbase/client/HTableInterfaceFactory.java | 54 -
.../apache/hadoop/hbase/client/Increment.java | 20 +
.../apache/hadoop/hbase/client/MetaScanner.java | 425 -
.../apache/hadoop/hbase/client/MultiAction.java | 2 +-
.../org/apache/hadoop/hbase/client/Put.java | 46 +-
.../client/RegionAdminServiceCallable.java | 63 +-
.../hadoop/hbase/client/RegionReplicaUtil.java | 56 +
.../org/apache/hadoop/hbase/client/Result.java | 152 +-
.../RetryingCallerInterceptorFactory.java | 2 +-
.../hbase/client/ReversedClientScanner.java | 33 +-
.../hbase/client/ReversedScannerCallable.java | 2 -
.../hbase/client/RpcRetryingCallerImpl.java | 11 +-
.../org/apache/hadoop/hbase/client/Scan.java | 93 +-
.../hadoop/hbase/client/ScannerCallable.java | 46 +-
.../client/ScannerCallableWithReplicas.java | 54 +-
.../org/apache/hadoop/hbase/client/Table.java | 5 +
.../client/ZooKeeperKeepAliveConnection.java | 4 +-
.../hadoop/hbase/client/ZooKeeperRegistry.java | 8 +-
.../hbase/client/metrics/ScanMetrics.java | 11 +-
.../client/replication/ReplicationAdmin.java | 240 +-
.../exceptions/ConnectionClosingException.java | 2 +-
.../exceptions/PreemptiveFastFailException.java | 2 +-
.../apache/hadoop/hbase/executor/EventType.java | 31 +-
.../hadoop/hbase/executor/ExecutorType.java | 3 +-
.../hadoop/hbase/filter/ColumnRangeFilter.java | 2 +-
.../hadoop/hbase/filter/FamilyFilter.java | 2 +-
.../org/apache/hadoop/hbase/ipc/AsyncCall.java | 9 +-
.../hadoop/hbase/ipc/AsyncRpcChannel.java | 316 +-
.../apache/hadoop/hbase/ipc/AsyncRpcClient.java | 174 +-
.../hbase/ipc/AsyncServerResponseHandler.java | 16 +-
.../org/apache/hadoop/hbase/ipc/IPCUtil.java | 53 +-
.../apache/hadoop/hbase/ipc/RpcClientImpl.java | 7 +-
.../hadoop/hbase/protobuf/ProtobufUtil.java | 72 +-
.../hadoop/hbase/protobuf/RequestConverter.java | 36 +
.../hbase/protobuf/ResponseConverter.java | 12 +-
.../apache/hadoop/hbase/protobuf/package.html | 2 +-
.../hbase/quotas/ThrottlingException.java | 11 +-
.../hbase/replication/ReplicationLoadSink.java | 36 +
.../replication/ReplicationLoadSource.java | 53 +
.../hadoop/hbase/security/EncryptionUtil.java | 47 +-
.../hbase/security/SaslClientHandler.java | 59 +-
.../security/access/AccessControlClient.java | 121 +-
.../hbase/snapshot/SnapshotExistsException.java | 3 +
.../apache/hadoop/hbase/zookeeper/ZKConfig.java | 103 +-
.../apache/hadoop/hbase/zookeeper/ZKUtil.java | 107 +-
.../hadoop/hbase/zookeeper/ZkAclReset.java | 110 +
.../hbase/zookeeper/ZooKeeperWatcher.java | 7 +
.../hadoop/hbase/TestRegionLocations.java | 57 +-
.../hadoop/hbase/client/TestAsyncProcess.java | 4 +-
.../client/TestClientExponentialBackoff.java | 4 +
.../hbase/client/TestClientNoCluster.java | 57 +-
.../hadoop/hbase/client/TestClientScanner.java | 489 +
.../client/TestClientSmallReversedScanner.java | 349 +
.../hbase/client/TestClientSmallScanner.java | 339 +
.../org/apache/hadoop/hbase/client/TestGet.java | 7 +
.../hbase/client/TestProcedureFuture.java | 186 +
.../hbase/client/TestSnapshotFromAdmin.java | 8 +-
.../hadoop/hbase/filter/TestLongComparator.java | 4 +
.../hbase/security/TestEncryptionUtil.java | 49 +-
.../src/test/resources/log4j.properties | 2 +-
hbase-common/pom.xml | 49 +-
hbase-common/src/main/asciidoc/.gitignore | 0
.../java/org/apache/hadoop/hbase/AuthUtil.java | 6 +-
.../org/apache/hadoop/hbase/CellComparator.java | 4 +-
.../java/org/apache/hadoop/hbase/CellUtil.java | 19 +
.../org/apache/hadoop/hbase/ChoreService.java | 4 +-
.../apache/hadoop/hbase/HBaseConfiguration.java | 2 +-
.../org/apache/hadoop/hbase/HConstants.java | 51 +-
.../java/org/apache/hadoop/hbase/KeyValue.java | 163 +-
.../org/apache/hadoop/hbase/KeyValueUtil.java | 2 +-
.../org/apache/hadoop/hbase/ScheduledChore.java | 45 +-
.../java/org/apache/hadoop/hbase/Stoppable.java | 4 +-
.../java/org/apache/hadoop/hbase/TableName.java | 38 +-
.../hbase/exceptions/TimeoutIOException.java | 46 +
.../hadoop/hbase/io/BoundedByteBufferPool.java | 113 +
.../hadoop/hbase/io/ByteBufferOutputStream.java | 32 +-
.../org/apache/hadoop/hbase/io/TimeRange.java | 1 +
.../hadoop/hbase/io/crypto/Encryption.java | 31 +-
.../io/encoding/BufferedDataBlockEncoder.java | 5 -
.../hbase/io/encoding/DataBlockEncoder.java | 21 -
.../hadoop/hbase/io/util/StreamUtils.java | 12 +-
.../org/apache/hadoop/hbase/security/User.java | 2 +
.../org/apache/hadoop/hbase/util/Bytes.java | 16 +-
.../hadoop/hbase/util/ForeignExceptionUtil.java | 109 +
.../java/org/apache/hadoop/hbase/util/Hash.java | 2 +-
.../apache/hadoop/hbase/util/JenkinsHash.java | 2 +-
.../apache/hadoop/hbase/util/MurmurHash.java | 2 +-
.../apache/hadoop/hbase/util/MurmurHash3.java | 2 +-
.../apache/hadoop/hbase/util/PrettyPrinter.java | 2 +-
.../apache/hadoop/hbase/util/RetryCounter.java | 4 +-
.../org/apache/hadoop/hbase/util/Threads.java | 92 +-
.../src/main/resources/hbase-default.xml | 77 +-
.../apache/hadoop/hbase/TestChoreService.java | 71 +-
.../org/apache/hadoop/hbase/TestKeyValue.java | 235 +
.../hbase/io/TestBoundedByteBufferPool.java | 88 +
.../hbase/io/crypto/TestCipherProvider.java | 8 +-
.../hadoop/hbase/io/crypto/TestEncryption.java | 7 +-
.../org/apache/hadoop/hbase/util/TestBytes.java | 14 +
.../src/test/resources/log4j.properties | 2 +-
hbase-examples/pom.xml | 64 +-
hbase-examples/src/main/asciidoc/.gitignore | 0
.../coprocessor/example/BulkDeleteEndpoint.java | 13 +-
.../hadoop/hbase/thrift/HttpDoAsClient.java | 27 +-
hbase-hadoop-compat/pom.xml | 51 +-
.../src/main/asciidoc/.gitignore | 0
.../hbase/ipc/MetricsHBaseServerSource.java | 6 +-
.../regionserver/MetricsRegionServerSource.java | 12 +-
.../MetricsRegionServerWrapper.java | 6 +-
.../MetricsReplicationSinkSource.java | 1 +
.../MetricsReplicationSourceSource.java | 1 +
hbase-hadoop2-compat/pom.xml | 40 +-
.../src/main/asciidoc/.gitignore | 0
.../hbase/ipc/MetricsHBaseServerSourceImpl.java | 11 +-
.../hbase/master/MetricsMasterSourceImpl.java | 3 +-
.../MetricsRegionAggregateSourceImpl.java | 3 +-
.../MetricsRegionServerSourceImpl.java | 15 +-
.../MetricsReplicationGlobalSourceSource.java | 5 +
.../MetricsReplicationSinkSourceImpl.java | 5 +
.../MetricsReplicationSourceSourceImpl.java | 5 +
hbase-it/pom.xml | 63 +-
hbase-it/src/main/asciidoc/.gitignore | 0
.../hadoop/hbase/IntegrationTestBase.java | 15 +-
.../hadoop/hbase/IntegrationTestIngest.java | 35 +-
.../IntegrationTestIngestStripeCompactions.java | 11 +-
.../IntegrationTestIngestWithEncryption.java | 8 +-
.../hbase/IntegrationTestIngestWithMOB.java | 9 +-
...IntegrationTestRegionReplicaReplication.java | 226 +
.../hadoop/hbase/RESTApiClusterManager.java | 350 +
.../hbase/chaos/actions/RemoveColumnAction.java | 2 +-
.../hadoop/hbase/mttr/IntegrationTestMTTR.java | 17 +-
.../test/IntegrationTestBigLinkedList.java | 228 +-
...egrationTestBigLinkedListWithVisibility.java | 11 +-
.../hbase/test/IntegrationTestReplication.java | 417 +
hbase-prefix-tree/pom.xml | 49 +-
hbase-prefix-tree/src/main/asciidoc/.gitignore | 0
.../codec/prefixtree/PrefixTreeSeeker.java | 29 -
hbase-procedure/pom.xml | 181 +
.../hbase/procedure2/OnePhaseProcedure.java | 28 +
.../hadoop/hbase/procedure2/Procedure.java | 680 +
.../procedure2/ProcedureAbortedException.java | 42 +
.../hbase/procedure2/ProcedureException.java | 45 +
.../hbase/procedure2/ProcedureExecutor.java | 1077 ++
.../procedure2/ProcedureFairRunQueues.java | 174 +
.../hbase/procedure2/ProcedureResult.java | 95 +
.../hbase/procedure2/ProcedureRunnableSet.java | 78 +
.../procedure2/ProcedureSimpleRunQueue.java | 121 +
.../procedure2/ProcedureYieldException.java | 40 +
.../procedure2/RemoteProcedureException.java | 116 +
.../hbase/procedure2/RootProcedureState.java | 185 +
.../hbase/procedure2/SequentialProcedure.java | 81 +
.../hbase/procedure2/StateMachineProcedure.java | 166 +
.../hbase/procedure2/TwoPhaseProcedure.java | 28 +
.../hbase/procedure2/store/ProcedureStore.java | 121 +
.../procedure2/store/ProcedureStoreTracker.java | 548 +
.../CorruptedWALProcedureStoreException.java | 43 +
.../procedure2/store/wal/ProcedureWALFile.java | 152 +
.../store/wal/ProcedureWALFormat.java | 234 +
.../store/wal/ProcedureWALFormatReader.java | 166 +
.../procedure2/store/wal/WALProcedureStore.java | 721 +
.../hadoop/hbase/procedure2/util/ByteSlot.java | 111 +
.../hbase/procedure2/util/StringUtils.java | 80 +
.../procedure2/util/TimeoutBlockingQueue.java | 217 +
.../procedure2/ProcedureTestingUtility.java | 163 +
.../procedure2/TestProcedureExecution.java | 338 +
.../procedure2/TestProcedureFairRunQueues.java | 155 +
.../hbase/procedure2/TestProcedureRecovery.java | 488 +
.../procedure2/TestProcedureReplayOrder.java | 226 +
.../store/TestProcedureStoreTracker.java | 193 +
.../store/wal/TestWALProcedureStore.java | 267 +
.../util/TestTimeoutBlockingQueue.java | 137 +
hbase-protocol/README.txt | 2 +-
hbase-protocol/pom.xml | 2 +
hbase-protocol/src/main/asciidoc/.gitignore | 0
.../hbase/protobuf/generated/AdminProtos.java | 1557 +-
.../hbase/protobuf/generated/ClientProtos.java | 870 +-
.../protobuf/generated/ClusterStatusProtos.java | 10280 +++++++----
.../generated/MasterProcedureProtos.java | 11424 ++++++++++++
.../hbase/protobuf/generated/MasterProtos.java | 15506 ++++++++++-------
.../protobuf/generated/ProcedureProtos.java | 7219 ++++++++
.../hbase/protobuf/generated/RPCProtos.java | 1664 +-
.../generated/RegionServerStatusProtos.java | 809 +-
.../hbase/protobuf/generated/WALProtos.java | 317 +-
.../protobuf/generated/ZooKeeperProtos.java | 1417 +-
hbase-protocol/src/main/protobuf/Admin.proto | 13 +
hbase-protocol/src/main/protobuf/Client.proto | 22 +
.../src/main/protobuf/ClusterStatus.proto | 43 +
hbase-protocol/src/main/protobuf/Master.proto | 39 +
.../src/main/protobuf/MasterProcedure.proto | 185 +
.../src/main/protobuf/Procedure.proto | 114 +
hbase-protocol/src/main/protobuf/RPC.proto | 11 +
.../src/main/protobuf/RegionServerStatus.proto | 8 +-
hbase-protocol/src/main/protobuf/WAL.proto | 3 +
.../src/main/protobuf/ZooKeeper.proto | 17 -
hbase-rest/pom.xml | 30 +
hbase-rest/src/main/asciidoc/.gitignore | 0
.../apache/hadoop/hbase/rest/RESTServer.java | 6 +-
.../apache/hadoop/hbase/rest/RESTServlet.java | 4 +-
.../hadoop/hbase/rest/RESTServletContainer.java | 3 +-
.../hadoop/hbase/rest/RegionsResource.java | 6 +-
.../apache/hadoop/hbase/rest/RowResource.java | 4 +-
.../org/apache/hadoop/hbase/rest/RowSpec.java | 8 +-
.../hadoop/hbase/rest/SchemaResource.java | 27 +-
.../apache/hadoop/hbase/rest/TableResource.java | 3 +-
.../hadoop/hbase/rest/client/RemoteHTable.java | 79 +-
.../hbase/rest/model/ColumnSchemaModel.java | 10 +-
.../hadoop/hbase/rest/model/ScannerModel.java | 2 +-
.../rest/model/StorageClusterStatusModel.java | 4 +-
.../hbase/rest/model/TableRegionModel.java | 2 +-
.../hbase/rest/model/TableSchemaModel.java | 16 +-
.../hadoop/hbase/rest/RowResourceBase.java | 1 -
.../hadoop/hbase/rest/TestGzipFilter.java | 1 -
.../hbase/rest/TestScannersWithFilters.java | 1 -
.../hadoop/hbase/rest/TestStatusResource.java | 1 -
.../hadoop/hbase/rest/TestTableResource.java | 54 +-
hbase-rest/src/test/resources/log4j.properties | 2 +-
hbase-server/pom.xml | 71 +-
hbase-server/src/main/asciidoc/.gitignore | 0
.../hbase/tmpl/master/MasterStatusTmpl.jamon | 23 +-
.../tmpl/master/RegionServerListTmpl.jamon | 1 -
.../tmpl/regionserver/BlockCacheTmpl.jamon | 62 +-
.../tmpl/regionserver/RegionListTmpl.jamon | 4 +-
.../tmpl/regionserver/ServerMetricsTmpl.jamon | 2 +
.../apache/hadoop/hbase/LocalHBaseCluster.java | 17 +-
.../java/org/apache/hadoop/hbase/Server.java | 4 +-
.../hbase/client/ClientSideRegionScanner.java | 5 +-
.../hbase/client/CoprocessorHConnection.java | 13 +-
.../hadoop/hbase/client/HTableWrapper.java | 57 +-
.../hbase/client/TableSnapshotScanner.java | 4 +-
.../SplitLogWorkerCoordination.java | 4 +-
.../ZKSplitLogManagerCoordination.java | 32 +-
.../ZkSplitLogWorkerCoordination.java | 20 +-
.../coprocessor/AggregateImplementation.java | 10 +-
.../hbase/coprocessor/BaseRegionObserver.java | 6 +-
.../coprocessor/BaseRegionServerObserver.java | 18 +-
.../coprocessor/BaseRowProcessorEndpoint.java | 8 +-
.../hbase/coprocessor/CoprocessorHost.java | 23 +-
.../RegionCoprocessorEnvironment.java | 4 +-
.../hbase/coprocessor/RegionObserver.java | 14 +-
.../hbase/coprocessor/RegionServerObserver.java | 14 +-
.../org/apache/hadoop/hbase/io/FileLink.java | 23 +-
.../hadoop/hbase/io/HalfStoreFileReader.java | 41 +-
.../hbase/io/hfile/AbstractHFileReader.java | 352 -
.../hbase/io/hfile/AbstractHFileWriter.java | 266 -
.../hbase/io/hfile/BlockWithScanInfo.java | 8 +-
.../hadoop/hbase/io/hfile/CacheConfig.java | 82 +-
.../hbase/io/hfile/CombinedBlockCache.java | 47 +-
.../hadoop/hbase/io/hfile/FixedFileTrailer.java | 6 +-
.../org/apache/hadoop/hbase/io/hfile/HFile.java | 83 +-
.../hadoop/hbase/io/hfile/HFileBlock.java | 153 +-
.../hadoop/hbase/io/hfile/HFileBlockIndex.java | 10 +-
.../hbase/io/hfile/HFilePrettyPrinter.java | 2 +-
.../hadoop/hbase/io/hfile/HFileReaderImpl.java | 1648 ++
.../hadoop/hbase/io/hfile/HFileReaderV2.java | 1318 --
.../hadoop/hbase/io/hfile/HFileReaderV3.java | 358 -
.../hadoop/hbase/io/hfile/HFileScanner.java | 91 +-
.../hbase/io/hfile/HFileWriterFactory.java | 40 +
.../hadoop/hbase/io/hfile/HFileWriterImpl.java | 641 +
.../hadoop/hbase/io/hfile/HFileWriterV2.java | 424 -
.../hadoop/hbase/io/hfile/HFileWriterV3.java | 136 -
.../io/hfile/InclusiveCombinedBlockCache.java | 58 +
.../hadoop/hbase/io/hfile/LruBlockCache.java | 37 +-
.../hbase/io/hfile/MemcachedBlockCache.java | 272 +
.../hbase/io/hfile/bucket/BucketCache.java | 216 +-
.../hbase/io/hfile/bucket/CachedEntryQueue.java | 20 +-
.../org/apache/hadoop/hbase/ipc/CallRunner.java | 15 +-
.../hadoop/hbase/ipc/MetricsHBaseServer.java | 4 +
.../apache/hadoop/hbase/ipc/RequestContext.java | 153 -
.../apache/hadoop/hbase/ipc/RpcCallContext.java | 27 +
.../org/apache/hadoop/hbase/ipc/RpcServer.java | 141 +-
.../hadoop/hbase/mapred/TableInputFormat.java | 18 +-
.../hbase/mapred/TableInputFormatBase.java | 189 +-
.../hbase/mapreduce/HFileOutputFormat2.java | 16 +-
.../hbase/mapreduce/LoadIncrementalHFiles.java | 196 +-
.../hbase/mapreduce/TableInputFormat.java | 4 +-
.../hbase/mapreduce/TableInputFormatBase.java | 133 +-
.../hbase/mapreduce/TableRecordReaderImpl.java | 11 +-
.../replication/VerifyReplication.java | 30 +-
.../hadoop/hbase/master/AssignmentManager.java | 13 +-
.../hadoop/hbase/master/CatalogJanitor.java | 27 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 399 +-
.../hadoop/hbase/master/HMasterCommandLine.java | 52 +-
.../hbase/master/MasterCoprocessorHost.java | 12 +-
.../hadoop/hbase/master/MasterFileSystem.java | 10 +-
.../hadoop/hbase/master/MasterRpcServices.java | 95 +-
.../hadoop/hbase/master/MasterServices.java | 15 +-
.../hadoop/hbase/master/RegionStateStore.java | 4 +-
.../hadoop/hbase/master/RegionStates.java | 20 +-
.../hadoop/hbase/master/ServerManager.java | 112 +-
.../hadoop/hbase/master/SplitLogManager.java | 2 +-
.../hadoop/hbase/master/TableLockManager.java | 6 +-
.../hbase/master/TableNamespaceManager.java | 36 +-
.../hadoop/hbase/master/TableStateManager.java | 1 -
.../master/balancer/LoadBalancerFactory.java | 11 +-
.../master/handler/CreateTableHandler.java | 7 +-
.../master/handler/DeleteTableHandler.java | 18 +-
.../master/handler/DisableTableHandler.java | 11 +-
.../master/handler/EnableTableHandler.java | 6 +-
.../master/handler/ModifyTableHandler.java | 9 +-
.../handler/TableDeleteFamilyHandler.java | 6 +-
.../procedure/AddColumnFamilyProcedure.java | 407 +
.../master/procedure/CreateTableProcedure.java | 442 +
.../procedure/DeleteColumnFamilyProcedure.java | 439 +
.../master/procedure/DeleteTableProcedure.java | 450 +
.../master/procedure/DisableTableProcedure.java | 540 +
.../master/procedure/EnableTableProcedure.java | 582 +
.../procedure/MasterDDLOperationHelper.java | 167 +
.../procedure/MasterProcedureConstants.java | 31 +
.../master/procedure/MasterProcedureEnv.java | 123 +
.../master/procedure/MasterProcedureQueue.java | 448 +
.../master/procedure/MasterProcedureUtil.java | 56 +
.../procedure/ModifyColumnFamilyProcedure.java | 382 +
.../master/procedure/ModifyTableProcedure.java | 510 +
.../master/procedure/ProcedurePrepareLatch.java | 105 +
.../master/procedure/ProcedureSyncWait.java | 179 +
.../procedure/TableProcedureInterface.java | 48 +
.../procedure/TruncateTableProcedure.java | 291 +
.../hbase/master/snapshot/SnapshotManager.java | 4 +-
.../hadoop/hbase/mob/DefaultMobCompactor.java | 16 +-
.../hbase/mob/DefaultMobStoreFlusher.java | 18 +-
.../PartitionedMobFileCompactor.java | 31 +-
.../hbase/mob/mapreduce/MemStoreWrapper.java | 12 +-
.../hbase/mob/mapreduce/SweepReducer.java | 13 +-
.../hbase/namespace/NamespaceAuditor.java | 15 -
.../hbase/namespace/NamespaceStateManager.java | 49 +-
.../namespace/NamespaceTableAndRegionInfo.java | 2 +-
.../hbase/procedure/ZKProcedureMemberRpcs.java | 2 +-
.../procedure/flush/FlushTableSubprocedure.java | 15 +-
.../RegionServerFlushTableProcedureManager.java | 6 +-
.../hadoop/hbase/quotas/MasterQuotaManager.java | 15 +-
.../hbase/quotas/RegionServerQuotaManager.java | 15 +-
.../hbase/quotas/RegionStateListener.java | 2 +-
.../hbase/quotas/RegionStateListener.java~HEAD | 54 -
.../quotas/RegionStateListener.java~HEAD_0 | 54 -
.../quotas/RegionStateListener.java~jon_master | 54 -
.../quotas/RegionStateListener.java~master | 54 -
.../hadoop/hbase/quotas/TimeBasedLimiter.java | 9 +-
.../AnnotationReadingPriorityFunction.java | 5 +-
.../hbase/regionserver/CompactSplitThread.java | 60 +-
.../hbase/regionserver/CompactionRequestor.java | 14 +-
.../ConstantSizeRegionSplitPolicy.java | 8 +-
.../hbase/regionserver/DefaultMemStore.java | 16 +-
.../hbase/regionserver/DefaultStoreFlusher.java | 2 +-
.../regionserver/ExplicitColumnTracker.java | 17 +-
.../regionserver/FavoredNodesForRegion.java | 7 +-
.../regionserver/FlushRequestListener.java | 2 +-
.../hbase/regionserver/FlushRequester.java | 8 +-
.../hadoop/hbase/regionserver/HRegion.java | 2359 ++-
.../hbase/regionserver/HRegionFileSystem.java | 10 +-
.../hbase/regionserver/HRegionServer.java | 370 +-
.../hadoop/hbase/regionserver/HStore.java | 144 +-
.../hbase/regionserver/HeapMemoryManager.java | 2 +-
...IncreasingToUpperBoundRegionSplitPolicy.java | 4 +-
.../hbase/regionserver/InternalScanner.java | 9 +-
.../hadoop/hbase/regionserver/KeyValueHeap.java | 49 +-
.../hbase/regionserver/KeyValueScanner.java | 6 +
.../hbase/regionserver/LastSequenceId.java | 10 +-
.../hadoop/hbase/regionserver/LogRoller.java | 2 +-
.../hadoop/hbase/regionserver/MemStore.java | 6 +
.../hbase/regionserver/MemStoreFlusher.java | 170 +-
.../hbase/regionserver/MetricsRegionServer.java | 4 +
.../MetricsRegionServerWrapperImpl.java | 42 +-
.../hbase/regionserver/MobStoreScanner.java | 4 +-
.../regionserver/NoLimitScannerContext.java | 102 +
.../regionserver/NonLazyKeyValueScanner.java | 4 +
.../hbase/regionserver/OnlineRegions.java | 27 +-
.../hbase/regionserver/RSRpcServices.java | 324 +-
.../hadoop/hbase/regionserver/Region.java | 678 +
.../regionserver/RegionCoprocessorHost.java | 29 +-
.../hbase/regionserver/RegionMergeRequest.java | 13 +-
.../regionserver/RegionMergeTransaction.java | 644 +-
.../RegionMergeTransactionFactory.java | 76 +
.../RegionMergeTransactionImpl.java | 702 +
.../hbase/regionserver/RegionScanner.java | 44 +-
.../RegionServerCoprocessorHost.java | 15 +-
.../regionserver/RegionServerServices.java | 13 +-
.../hbase/regionserver/RegionSplitPolicy.java | 6 +-
.../hbase/regionserver/ReplicationService.java | 8 +-
.../regionserver/ReversedMobStoreScanner.java | 4 +-
.../hbase/regionserver/ScanQueryMatcher.java | 44 +-
.../hbase/regionserver/ScannerContext.java | 527 +
.../hadoop/hbase/regionserver/SplitRequest.java | 20 +-
.../hbase/regionserver/SplitTransaction.java | 758 +-
.../regionserver/SplitTransactionFactory.java | 74 +
.../regionserver/SplitTransactionImpl.java | 789 +
.../apache/hadoop/hbase/regionserver/Store.java | 38 +-
.../hadoop/hbase/regionserver/StoreFile.java | 7 +-
.../hbase/regionserver/StoreFileScanner.java | 8 +-
.../hbase/regionserver/StoreFlushContext.java | 16 +
.../hadoop/hbase/regionserver/StoreFlusher.java | 8 +-
.../hadoop/hbase/regionserver/StoreScanner.java | 102 +-
.../regionserver/StorefileRefresherChore.java | 10 +-
.../regionserver/compactions/Compactor.java | 10 +-
.../handler/CloseRegionHandler.java | 4 +-
.../handler/FinishRegionRecoveringHandler.java | 56 +
.../handler/RegionReplicaFlushHandler.java | 187 +
.../snapshot/FlushSnapshotSubprocedure.java | 16 +-
.../snapshot/RegionServerSnapshotManager.java | 12 +-
.../hbase/regionserver/wal/FSWALEntry.java | 30 +-
.../regionserver/wal/ProtobufLogReader.java | 15 +-
.../hbase/regionserver/wal/ReplayHLogKey.java | 53 +
.../wal/SecureProtobufLogReader.java | 8 +-
.../wal/SecureProtobufLogWriter.java | 5 +-
.../hadoop/hbase/regionserver/wal/WALEdit.java | 13 +-
.../hbase/replication/BaseWALEntryFilter.java | 29 +
.../hbase/replication/ReplicationEndpoint.java | 9 +-
.../HBaseInterClusterReplicationEndpoint.java | 8 +-
.../replication/regionserver/MetricsSink.java | 17 +
.../replication/regionserver/MetricsSource.java | 36 +-
.../RegionReplicaReplicationEndpoint.java | 349 +-
.../replication/regionserver/Replication.java | 32 +-
.../regionserver/ReplicationLoad.java | 151 +
.../regionserver/ReplicationSink.java | 8 +
.../regionserver/ReplicationSource.java | 13 +-
.../regionserver/ReplicationSourceManager.java | 9 +-
.../security/access/AccessControlLists.java | 7 +-
.../hbase/security/access/AccessController.java | 462 +-
.../hbase/security/access/AuthResult.java | 81 +-
.../security/access/SecureBulkLoadEndpoint.java | 14 +-
.../hbase/security/access/TableAuthManager.java | 35 +-
.../security/access/ZKPermissionWatcher.java | 17 +
.../hbase/security/token/TokenProvider.java | 5 +-
.../DefaultVisibilityLabelServiceImpl.java | 9 +-
.../visibility/VisibilityController.java | 127 +-
.../security/visibility/VisibilityUtils.java | 10 +-
.../hadoop/hbase/snapshot/SnapshotManifest.java | 2 +-
.../org/apache/hadoop/hbase/tool/Canary.java | 440 +-
.../hadoop/hbase/tool/WriteSinkCoprocessor.java | 2 +-
.../hadoop/hbase/util/CompressionTest.java | 9 +-
.../hadoop/hbase/util/ConnectionCache.java | 34 +-
.../apache/hadoop/hbase/util/FSHDFSUtils.java | 14 +-
.../org/apache/hadoop/hbase/util/FSUtils.java | 55 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 232 +-
.../org/apache/hadoop/hbase/util/HMerge.java | 33 +-
.../org/apache/hadoop/hbase/util/IdLock.java | 16 +
.../org/apache/hadoop/hbase/util/Merge.java | 4 +-
.../hadoop/hbase/util/ModifyRegionUtils.java | 24 +
.../hadoop/hbase/util/MultiHConnection.java | 8 +-
.../hadoop/hbase/util/RegionSizeCalculator.java | 5 +
.../hbase/util/ServerRegionReplicaUtil.java | 62 +-
.../hbase/util/hbck/OfflineMetaRepair.java | 2 +-
.../hbase/util/hbck/TableLockChecker.java | 6 +-
.../hadoop/hbase/wal/WALPrettyPrinter.java | 27 +-
.../apache/hadoop/hbase/wal/WALSplitter.java | 62 +-
.../hbase/zookeeper/MiniZooKeeperCluster.java | 113 +-
.../zookeeper/RecoveringRegionWatcher.java | 11 +-
.../hadoop/hbase/zookeeper/ZKSplitLog.java | 17 +-
.../hbase/zookeeper/ZooKeeperMainServer.java | 30 +-
.../hadoop/hbase/replication/package.html | 140 +-
.../resources/hbase-webapps/master/snapshot.jsp | 1 -
.../resources/hbase-webapps/master/table.jsp | 98 +-
.../org/apache/hadoop/hbase/HBaseTestCase.java | 12 +-
.../hadoop/hbase/HBaseTestingUtility.java | 197 +-
.../hbase/HFilePerformanceEvaluation.java | 4 +-
.../apache/hadoop/hbase/MiniHBaseCluster.java | 30 +-
.../hadoop/hbase/MockRegionServerServices.java | 17 +-
.../hadoop/hbase/PerformanceEvaluation.java | 193 +-
.../hadoop/hbase/ScanPerformanceEvaluation.java | 7 +-
.../ServerResourceCheckerJUnitListener.java | 15 -
.../apache/hadoop/hbase/TestAcidGuarantees.java | 2 -
.../hadoop/hbase/TestGlobalMemStoreSize.java | 16 +-
.../hadoop/hbase/TestHBaseTestingUtility.java | 115 +-
.../org/apache/hadoop/hbase/TestIOFencing.java | 8 +-
.../apache/hadoop/hbase/TestInfoServers.java | 68 +-
.../hadoop/hbase/TestMetaTableAccessor.java | 52 +-
.../hbase/TestMetaTableAccessorNoCluster.java | 4 +-
.../hadoop/hbase/TestMovedRegionsCleaner.java | 96 +
.../hbase/TestPartialResultsFromClientSide.java | 831 +
.../org/apache/hadoop/hbase/TestZooKeeper.java | 115 +-
.../hadoop/hbase/backup/TestHFileArchiving.java | 31 +-
.../TestZooKeeperTableArchiveClient.java | 16 +-
.../hbase/client/HConnectionTestingUtility.java | 70 +-
.../apache/hadoop/hbase/client/TestAdmin2.java | 52 +-
.../hadoop/hbase/client/TestCheckAndMutate.java | 14 +
.../hadoop/hbase/client/TestClientPushback.java | 11 +-
.../hadoop/hbase/client/TestFromClientSide.java | 249 +-
.../org/apache/hadoop/hbase/client/TestHCM.java | 306 +-
.../client/TestHTableMultiplexerFlushCache.java | 35 +-
.../hadoop/hbase/client/TestMetaScanner.java | 243 -
.../hbase/client/TestMetaWithReplicas.java | 33 +-
.../client/TestMobCloneSnapshotFromClient.java | 3 +-
.../TestMobRestoreSnapshotFromClient.java | 6 +-
.../TestMobSnapshotCloneIndependence.java | 17 +-
.../hbase/client/TestMobSnapshotFromClient.java | 4 +-
.../hadoop/hbase/client/TestMultiParallel.java | 2 +-
.../hadoop/hbase/client/TestReplicasClient.java | 90 +-
.../apache/hadoop/hbase/client/TestResult.java | 27 +
.../hbase/client/TestResultSizeEstimation.java | 127 +
.../client/TestScannersFromClientSide.java | 83 +
.../hadoop/hbase/client/TestSizeFailures.java | 176 +
.../TestReplicationAdminWithClusters.java | 163 +
.../coprocessor/ColumnAggregationEndpoint.java | 1 -
.../ColumnAggregationEndpointNullResponse.java | 8 +-
.../ColumnAggregationEndpointWithErrors.java | 6 +-
.../hbase/coprocessor/SimpleRegionObserver.java | 10 +-
.../hbase/coprocessor/TestClassLoading.java | 33 +-
.../TestCoprocessorConfiguration.java | 172 +
.../coprocessor/TestCoprocessorEndpoint.java | 6 +-
.../coprocessor/TestCoprocessorInterface.java | 77 +-
.../TestCoprocessorTableEndpoint.java | 182 +
.../hbase/coprocessor/TestHTableWrapper.java | 34 +-
.../hbase/coprocessor/TestMasterObserver.java | 112 +-
.../TestRegionObserverInterface.java | 37 +-
.../TestRegionObserverScannerOpenHook.java | 17 +-
.../coprocessor/TestRegionServerObserver.java | 30 +-
.../hbase/filter/TestColumnPrefixFilter.java | 19 +-
.../hbase/filter/TestDependentColumnFilter.java | 17 +-
.../apache/hadoop/hbase/filter/TestFilter.java | 29 +-
.../filter/TestInvocationRecordFilter.java | 11 +-
.../filter/TestMultipleColumnPrefixFilter.java | 25 +-
.../hadoop/hbase/fs/TestBlockReorder.java | 11 +-
.../hadoop/hbase/http/ssl/KeyStoreTestUtil.java | 69 +-
.../hbase/io/TestByteBufferOutputStream.java | 49 +
.../apache/hadoop/hbase/io/TestFileLink.java | 36 +
.../hbase/io/encoding/TestEncodedSeekers.java | 13 +-
.../hbase/io/encoding/TestPrefixTree.java | 33 +-
.../hadoop/hbase/io/hfile/CacheTestUtils.java | 6 +-
.../hadoop/hbase/io/hfile/TestCacheConfig.java | 4 +-
.../hadoop/hbase/io/hfile/TestCacheOnWrite.java | 174 +-
.../hbase/io/hfile/TestFixedFileTrailer.java | 18 +-
.../io/hfile/TestForceCacheImportantBlocks.java | 10 +-
.../apache/hadoop/hbase/io/hfile/TestHFile.java | 4 +-
.../hbase/io/hfile/TestHFileBlockIndex.java | 2 +-
.../hbase/io/hfile/TestHFileEncryption.java | 4 +-
.../TestHFileInlineToRootChunkConversion.java | 7 +-
.../hadoop/hbase/io/hfile/TestHFileSeek.java | 2 +-
.../hbase/io/hfile/TestHFileWriterV2.java | 9 +-
.../hbase/io/hfile/TestHFileWriterV3.java | 9 +-
.../hfile/TestLazyDataBlockDecompression.java | 5 +-
.../hadoop/hbase/io/hfile/TestPrefetch.java | 6 +-
.../hadoop/hbase/io/hfile/TestReseekTo.java | 2 +-
.../TestScannerSelectionUsingKeyRange.java | 10 +-
.../io/hfile/TestScannerSelectionUsingTTL.java | 14 +-
.../hadoop/hbase/io/hfile/TestSeekTo.java | 11 +-
.../hbase/io/hfile/bucket/TestBucketCache.java | 87 +-
.../hadoop/hbase/ipc/AbstractTestIPC.java | 269 +
.../apache/hadoop/hbase/ipc/TestAsyncIPC.java | 298 +
.../apache/hadoop/hbase/ipc/TestCallRunner.java | 2 +-
.../hbase/ipc/TestGlobalEventLoopGroup.java | 54 +
.../org/apache/hadoop/hbase/ipc/TestIPC.java | 584 +-
.../apache/hadoop/hbase/ipc/TestRpcMetrics.java | 2 +
.../hbase/mapred/TestTableInputFormat.java | 267 +-
.../hbase/mapreduce/MapreduceTestingShim.java | 27 +-
.../hbase/mapreduce/TestHFileOutputFormat.java | 9 +-
.../hbase/mapreduce/TestHFileOutputFormat2.java | 9 +-
.../hbase/mapreduce/TestImportExport.java | 2 -
.../TestImportTSVWithOperationAttributes.java | 9 +-
.../hbase/mapreduce/TestImportTSVWithTTLs.java | 5 +-
.../mapreduce/TestLoadIncrementalHFiles.java | 85 +-
.../hbase/mapreduce/TestTableInputFormat.java | 487 +
.../hadoop/hbase/master/MockRegionServer.java | 20 +-
.../hbase/master/TestAssignmentListener.java | 8 +-
.../hadoop/hbase/master/TestCatalogJanitor.java | 24 +-
.../hbase/master/TestClockSkewDetection.java | 31 +-
.../master/TestDistributedLogSplitting.java | 7 +-
.../master/TestGetLastFlushedSequenceId.java | 25 +-
.../hbase/master/TestHMasterRPCException.java | 113 +-
.../hadoop/hbase/master/TestMasterFailover.java | 5 +-
.../hbase/master/TestRegionPlacement.java | 27 +-
.../hadoop/hbase/master/TestRestartCluster.java | 9 +-
.../hbase/master/TestTableLockManager.java | 33 +-
.../hbase/master/TestTableStateManager.java | 82 +
.../hadoop/hbase/master/TestWarmupRegion.java | 164 +
.../master/handler/TestEnableTableHandler.java | 160 +-
.../handler/TestTableDeleteFamilyHandler.java | 122 +-
.../TestTableDescriptorModification.java | 124 +-
.../MasterProcedureTestingUtility.java | 408 +
.../procedure/TestAddColumnFamilyProcedure.java | 246 +
.../procedure/TestCreateTableProcedure.java | 257 +
.../TestDeleteColumnFamilyProcedure.java | 302 +
.../procedure/TestDeleteTableProcedure.java | 208 +
.../procedure/TestDisableTableProcedure.java | 182 +
.../procedure/TestEnableTableProcedure.java | 193 +
.../TestMasterFailoverWithProcedures.java | 429 +
.../procedure/TestMasterProcedureQueue.java | 433 +
.../TestModifyColumnFamilyProcedure.java | 238 +
.../procedure/TestModifyTableProcedure.java | 403 +
.../procedure/TestTruncateTableProcedure.java | 246 +
.../apache/hadoop/hbase/mob/MobTestUtil.java | 7 +-
.../hadoop/hbase/mob/TestCachedMobFile.java | 36 +-
.../hbase/mob/TestDefaultMobStoreFlusher.java | 43 +-
.../hbase/mob/TestExpiredMobFileCleaner.java | 20 +-
.../hbase/mob/TestMobDataBlockEncoding.java | 24 +-
.../apache/hadoop/hbase/mob/TestMobFile.java | 22 +-
.../hadoop/hbase/mob/TestMobFileCache.java | 57 +-
.../filecompactions/TestMobFileCompactor.java | 85 +-
.../TestPartitionedMobFileCompactor.java | 13 +-
.../hbase/mob/mapreduce/TestMobSweepJob.java | 3 +-
.../mob/mapreduce/TestMobSweepReducer.java | 19 +-
.../hbase/mob/mapreduce/TestMobSweeper.java | 44 +-
.../hbase/namespace/TestNamespaceAuditor.java | 337 +-
.../hadoop/hbase/quotas/TestQuotaThrottle.java | 23 +-
.../regionserver/DataBlockEncodingTool.java | 7 +-
.../MetricsRegionServerWrapperStub.java | 5 +
.../regionserver/NoOpScanPolicyObserver.java | 2 +-
.../hbase/regionserver/TestAtomicOperation.java | 58 +-
.../hbase/regionserver/TestBlocksRead.java | 36 +-
.../hbase/regionserver/TestBlocksScanned.java | 12 +-
.../hadoop/hbase/regionserver/TestBulkLoad.java | 58 +-
.../regionserver/TestCacheOnWriteInSchema.java | 3 +-
.../hbase/regionserver/TestColumnSeeking.java | 30 +-
.../hbase/regionserver/TestCompaction.java | 6 +-
.../hbase/regionserver/TestCompactionState.java | 10 +-
.../hbase/regionserver/TestDefaultMemStore.java | 13 +-
.../hbase/regionserver/TestDeleteMobTable.java | 31 +-
.../regionserver/TestEncryptionKeyRotation.java | 19 +-
.../TestEncryptionRandomKeying.java | 9 +-
.../TestEndToEndSplitTransaction.java | 228 +-
.../regionserver/TestExplicitColumnTracker.java | 64 +-
.../regionserver/TestGetClosestAtOrBefore.java | 45 +-
.../hbase/regionserver/TestHMobStore.java | 2 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 340 +-
.../regionserver/TestHRegionReplayEvents.java | 1576 ++
.../regionserver/TestHeapMemoryManager.java | 4 +-
.../hbase/regionserver/TestKeepDeletes.java | 121 +-
.../hbase/regionserver/TestMajorCompaction.java | 71 +-
.../regionserver/TestMetricsRegionServer.java | 1 +
.../hbase/regionserver/TestMinVersions.java | 36 +-
.../hbase/regionserver/TestMinorCompaction.java | 12 +-
.../hbase/regionserver/TestMobCompaction.java | 25 +-
.../hbase/regionserver/TestMobStoreScanner.java | 59 +-
.../regionserver/TestMultiColumnScanner.java | 8 +-
.../regionserver/TestPerColumnFamilyFlush.java | 228 +-
.../hbase/regionserver/TestQueryMatcher.java | 21 -
.../regionserver/TestRegionFavoredNodes.java | 10 +-
.../TestRegionMergeTransaction.java | 68 +-
.../TestRegionMergeTransactionOnCluster.java | 2 +-
.../regionserver/TestRegionReplicaFailover.java | 373 +
.../hbase/regionserver/TestRegionReplicas.java | 30 +-
.../regionserver/TestRegionServerHostname.java | 108 +
.../regionserver/TestRegionServerMetrics.java | 76 +-
.../regionserver/TestRegionServerNoMaster.java | 4 +-
.../TestRegionServerOnlineConfigChange.java | 2 +-
.../TestRegionServerReportForDuty.java | 183 +
.../regionserver/TestRegionSplitPolicy.java | 39 +-
.../regionserver/TestResettingCounters.java | 10 +-
.../regionserver/TestReversibleScanners.java | 10 +-
.../hbase/regionserver/TestRowTooBig.java | 10 +-
.../regionserver/TestScanWithBloomError.java | 8 +-
.../hadoop/hbase/regionserver/TestScanner.java | 18 +-
.../regionserver/TestSeekOptimizations.java | 8 +-
.../regionserver/TestSplitTransaction.java | 62 +-
.../TestSplitTransactionOnCluster.java | 71 +-
.../hbase/regionserver/TestStoreFileInfo.java | 28 +-
.../TestStoreFileRefresherChore.java | 23 +-
.../hbase/regionserver/TestStoreScanner.java | 2 +-
.../hbase/regionserver/TestStripeCompactor.java | 26 +-
.../hbase/regionserver/TestWideScanner.java | 4 +-
.../TestCompactionWithThroughputController.java | 7 +-
.../compactions/TestStripeCompactionPolicy.java | 9 +-
.../hbase/regionserver/wal/TestDurability.java | 23 +
.../hbase/regionserver/wal/TestFSHLog.java | 3 +-
.../hbase/regionserver/wal/TestLogRolling.java | 18 +-
.../hbase/regionserver/wal/TestWALReplay.java | 58 +-
.../replication/TestMasterReplication.java | 4 +-
.../replication/TestMultiSlaveReplication.java | 2 +-
.../replication/TestReplicationSmallTests.java | 45 +
.../TestRegionReplicaReplicationEndpoint.java | 134 +-
...egionReplicaReplicationEndpointNoMaster.java | 67 +-
.../hbase/security/HBaseKerberosUtils.java | 36 +-
.../hadoop/hbase/security/TestSecureRPC.java | 103 +-
.../TestUsersOperationsWithSecureHadoop.java | 62 +-
.../hbase/security/access/SecureTestUtil.java | 294 +-
.../access/TestAccessControlFilter.java | 2 +-
.../security/access/TestAccessController.java | 542 +-
.../security/access/TestAccessController2.java | 270 +-
.../access/TestCellACLWithMultipleVersions.java | 35 +-
.../hbase/security/access/TestCellACLs.java | 115 +-
.../security/access/TestNamespaceCommands.java | 161 +-
.../access/TestScanEarlyTermination.java | 3 +-
.../access/TestWithDisabledAuthorization.java | 1079 ++
.../token/TestGenerateDelegationToken.java | 173 +
.../security/token/TestTokenAuthentication.java | 5 +-
.../ExpAsStringVisibilityLabelServiceImpl.java | 10 +-
.../TestDefaultScanLabelGeneratorStack.java | 27 +-
.../TestEnforcingScanLabelGenerator.java | 22 +-
...sibilityLabelReplicationWithExpAsString.java | 30 +-
.../visibility/TestVisibilityLabels.java | 164 +-
.../TestVisibilityLabelsReplication.java | 36 +-
.../visibility/TestVisibilityLabelsWithACL.java | 41 +-
...ibilityLabelsWithDefaultVisLabelService.java | 11 +-
.../TestVisibilityLabelsWithDeletes.java | 306 +-
.../TestVisibilityLabelsWithSLGStack.java | 8 +-
.../TestVisibilityLablesWithGroups.java | 36 +-
.../TestVisibilityWithCheckAuths.java | 49 +-
.../TestWithDisabledAuthorization.java | 237 +
.../hbase/snapshot/MobSnapshotTestingUtils.java | 16 +-
.../hbase/snapshot/SnapshotTestingUtils.java | 6 +-
.../TestMobFlushSnapshotFromClient.java | 13 +-
.../TestMobRestoreFlushSnapshotFromClient.java | 5 +-
.../snapshot/TestMobRestoreSnapshotHelper.java | 4 -
.../snapshot/TestSnapshotClientRetries.java | 125 +
.../hadoop/hbase/util/ConstantDelayQueue.java | 196 +
.../util/LoadTestDataGeneratorWithMOB.java | 2 +-
.../apache/hadoop/hbase/util/LoadTestTool.java | 87 +-
.../hadoop/hbase/util/MultiThreadedAction.java | 4 +-
.../hbase/util/MultiThreadedWriterBase.java | 7 +-
.../hadoop/hbase/util/RestartMetaTest.java | 6 +-
.../hadoop/hbase/util/TestEncryptionTest.java | 6 +-
.../apache/hadoop/hbase/util/TestFSUtils.java | 19 +-
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 194 +-
.../hbase/util/TestHBaseFsckEncryption.java | 14 +-
.../hadoop/hbase/util/TestMergeTable.java | 5 +-
.../apache/hadoop/hbase/util/TestMergeTool.java | 4 +-
.../apache/hadoop/hbase/util/TestTableName.java | 15 +-
.../hadoop/hbase/util/hbck/HbckTestingUtil.java | 37 +-
.../util/hbck/TestOfflineMetaRebuildBase.java | 1 -
.../hadoop/hbase/wal/TestWALFiltering.java | 6 +-
.../hadoop/hbase/zookeeper/TestHQuorumPeer.java | 9 +
.../zookeeper/TestRecoverableZooKeeper.java | 3 +-
.../zookeeper/TestZooKeeperMainServer.java | 10 +
.../src/test/resources/log4j.properties | 2 +-
hbase-shell/pom.xml | 62 +-
hbase-shell/src/main/asciidoc/.gitignore | 0
hbase-shell/src/main/ruby/hbase.rb | 5 +
hbase-shell/src/main/ruby/hbase/admin.rb | 67 +-
hbase-shell/src/main/ruby/hbase/hbase.rb | 6 +-
hbase-shell/src/main/ruby/hbase/quotas.rb | 10 +-
.../src/main/ruby/hbase/replication_admin.rb | 82 +-
hbase-shell/src/main/ruby/hbase/security.rb | 26 +-
hbase-shell/src/main/ruby/hbase/table.rb | 44 +-
.../src/main/ruby/hbase/visibility_labels.rb | 13 +-
hbase-shell/src/main/ruby/shell.rb | 4 +
hbase-shell/src/main/ruby/shell/commands.rb | 18 +-
.../src/main/ruby/shell/commands/add_peer.rb | 38 +-
.../ruby/shell/commands/balancer_enabled.rb | 41 +
.../src/main/ruby/shell/commands/clear_auths.rb | 5 +-
.../main/ruby/shell/commands/clone_snapshot.rb | 7 +
.../shell/commands/disable_table_replication.rb | 42 +
.../shell/commands/enable_table_replication.rb | 42 +
.../src/main/ruby/shell/commands/get_auths.rb | 5 +-
.../src/main/ruby/shell/commands/get_counter.rb | 10 +-
.../src/main/ruby/shell/commands/get_splits.rb | 46 +
.../src/main/ruby/shell/commands/incr.rb | 6 +-
.../src/main/ruby/shell/commands/set_auths.rb | 5 +-
.../src/main/ruby/shell/commands/status.rb | 9 +-
.../hadoop/hbase/client/AbstractTestShell.java | 69 +
.../hbase/client/TestReplicationShell.java | 38 +
.../apache/hadoop/hbase/client/TestShell.java | 52 +-
hbase-shell/src/test/ruby/hbase/admin_test.rb | 12 +
.../test/ruby/hbase/replication_admin_test.rb | 191 +
hbase-shell/src/test/ruby/hbase/table_test.rb | 17 +
.../ruby/hbase/visibility_labels_admin_test.rb | 2 +
hbase-shell/src/test/ruby/test_helper.rb | 20 +
hbase-shell/src/test/ruby/tests_runner.rb | 18 +
hbase-testing-util/src/main/asciidoc/.gitignore | 0
hbase-thrift/pom.xml | 77 +-
hbase-thrift/src/main/asciidoc/.gitignore | 0
.../hadoop/hbase/thrift/ThriftHttpServlet.java | 50 +-
.../hadoop/hbase/thrift/ThriftServerRunner.java | 228 +-
.../apache/hadoop/hbase/thrift2/HTablePool.java | 696 -
.../thrift2/ThriftHBaseServiceHandler.java | 68 +-
.../apache/hadoop/hbase/thrift2/hbase.thrift | 32 +-
.../hadoop/hbase/thrift2/TestHTablePool.java | 366 -
pom.xml | 294 +-
.../asciidoc/_chapters/appendix_acl_matrix.adoc | 134 +-
src/main/asciidoc/_chapters/architecture.adoc | 50 +-
src/main/asciidoc/_chapters/configuration.adoc | 7 +-
src/main/asciidoc/_chapters/developer.adoc | 2 +-
src/main/asciidoc/_chapters/hbase-default.adoc | 6 +-
src/main/asciidoc/_chapters/hbase_apis.adoc | 7 +
src/main/asciidoc/_chapters/images | 1 +
src/main/asciidoc/_chapters/ops_mgt.adoc | 95 +-
src/main/asciidoc/_chapters/performance.adoc | 11 +
src/main/asciidoc/_chapters/preface.adoc | 5 +
src/main/asciidoc/_chapters/rpc.adoc | 2 +-
src/main/asciidoc/_chapters/security.adoc | 103 +-
src/main/asciidoc/asciidoctor.css | 399 +
src/main/asciidoc/book.adoc | 18 +-
src/main/asciidoc/images | 1 +
.../resources/images/hbase_logo_with_orca.png | Bin 0 -> 11618 bytes
.../resources/images/hbase_logo_with_orca.xcf | Bin 0 -> 84265 bytes
.../images/jumping-orca_transparent_rotated.xcf | Bin 0 -> 135399 bytes
.../resources/images/region_split_process.png | Bin 0 -> 338255 bytes
src/main/site/site.xml | 37 +-
src/main/site/xdoc/index.xml | 12 +-
832 files changed, 95288 insertions(+), 31304 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --cc hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index e507245,efbc7d2..fe5a5f1
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@@ -3821,101 -4088,234 +4088,336 @@@ public class HBaseAdmin implements Admi
}
/**
+ * {@inheritDoc}
+ */
+ @Override
+ public void compactMob(final TableName tableName, final byte[] columnFamily)
+ throws IOException, InterruptedException {
+ checkTableNameNotNull(tableName);
+ checkFamilyNameNotNull(columnFamily);
+ validateMobColumnFamily(tableName, columnFamily);
+ compactMob(tableName, columnFamily, false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void compactMob(final TableName tableName) throws IOException, InterruptedException {
+ checkTableNameNotNull(tableName);
+ compactMob(tableName, null, false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void majorCompactMob(final TableName tableName, final byte[] columnFamily)
+ throws IOException, InterruptedException {
+ checkTableNameNotNull(tableName);
+ checkFamilyNameNotNull(columnFamily);
+ validateMobColumnFamily(tableName, columnFamily);
+ compactMob(tableName, columnFamily, true);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void majorCompactMob(final TableName tableName) throws IOException, InterruptedException {
+ checkTableNameNotNull(tableName);
+ compactMob(tableName, null, true);
+ }
+
++ /**
++ * {@inheritDoc}
++ */
+ @Override
+ public CompactionState getMobCompactionState(TableName tableName) throws IOException {
+ checkTableNameNotNull(tableName);
+ try {
+ ServerName master = getClusterStatus().getMaster();
+ HRegionInfo info = new HRegionInfo(tableName, Bytes.toBytes(".mob"),
+ HConstants.EMPTY_END_ROW, false, 0);
+ GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest(
+ info.getRegionName(), true);
+ GetRegionInfoResponse response = this.connection.getAdmin(master)
+ .getRegionInfo(null, request);
+ return response.getCompactionState();
+ } catch (ServiceException se) {
+ throw ProtobufUtil.getRemoteException(se);
+ }
+ }
+
+ /**
+ * Compacts the mob files in a mob-enabled column family. Asynchronous operation.
+ * @param tableName The table to compact.
+ * @param columnFamily The column family to compact. If it is null, all the mob-enabled
+ * column families in this table will be compacted.
+ * @param major Whether to select all the mob files in the compaction.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ private void compactMob(final TableName tableName, final byte[] columnFamily, boolean major)
+ throws IOException, InterruptedException {
+ // get the mob region info, this is a dummy region.
+ HRegionInfo info = new HRegionInfo(tableName, Bytes.toBytes(".mob"), HConstants.EMPTY_END_ROW,
+ false, 0);
+ ServerName master = getClusterStatus().getMaster();
+ compact(master, info, major, columnFamily);
+ }
+
+ private void checkTableNameNotNull(TableName tableName) {
+ if (tableName == null) {
+ throw new IllegalArgumentException("TableName cannot be null");
+ }
+ }
+
+ private void checkFamilyNameNotNull(byte[] columnFamily) {
+ if (columnFamily == null) {
+ throw new IllegalArgumentException("The column family name cannot be null");
+ }
+ }
+
+ private void validateMobColumnFamily(TableName tableName, byte[] columnFamily)
+ throws IOException {
+ HTableDescriptor htd = getTableDescriptor(tableName);
+ HColumnDescriptor family = htd.getFamily(columnFamily);
+ if (family == null || !family.isMobEnabled()) {
+ throw new IllegalArgumentException("Column family " + columnFamily
- + " is not a mob column family");
++ + " is not a mob column family");
++ }
++ }
++
++ /**
+ * Future that waits on a procedure result.
+ * Returned by the async version of the Admin calls,
+ * and used internally by the sync calls to wait on the result of the procedure.
+ */
+ @InterfaceAudience.Private
+ @InterfaceStability.Evolving
+ protected static class ProcedureFuture<V> implements Future<V> {
+ private ExecutionException exception = null;
+ private boolean procResultFound = false;
+ private boolean done = false;
+ private V result = null;
+
+ private final HBaseAdmin admin;
+ private final Long procId;
+
+ public ProcedureFuture(final HBaseAdmin admin, final Long procId) {
+ this.admin = admin;
+ this.procId = procId;
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isCancelled() {
+ // TODO: Abort not implemented yet
+ return false;
+ }
+
+ @Override
+ public V get() throws InterruptedException, ExecutionException {
+ // TODO: should we ever spin forever?
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public V get(long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ if (!done) {
+ long deadlineTs = EnvironmentEdgeManager.currentTime() + unit.toMillis(timeout);
+ try {
+ try {
+ // if the master support procedures, try to wait the result
+ if (procId != null) {
+ result = waitProcedureResult(procId, deadlineTs);
+ }
+ // if we don't have a proc result, try the compatibility wait
+ if (!procResultFound) {
+ result = waitOperationResult(deadlineTs);
+ }
+ result = postOperationResult(result, deadlineTs);
+ done = true;
+ } catch (IOException e) {
+ result = postOpeartionFailure(e, deadlineTs);
+ done = true;
+ }
+ } catch (IOException e) {
+ exception = new ExecutionException(e);
+ done = true;
+ }
+ }
+ if (exception != null) {
+ throw exception;
+ }
+ return result;
+ }
+
+ @Override
+ public boolean isDone() {
+ return done;
+ }
+
+ protected HBaseAdmin getAdmin() {
+ return admin;
+ }
+
+ private V waitProcedureResult(long procId, long deadlineTs)
+ throws IOException, TimeoutException, InterruptedException {
+ GetProcedureResultRequest request = GetProcedureResultRequest.newBuilder()
+ .setProcId(procId)
+ .build();
+
+ int tries = 0;
+ IOException serviceEx = null;
+ while (EnvironmentEdgeManager.currentTime() < deadlineTs) {
+ GetProcedureResultResponse response = null;
+ try {
+ // Try to fetch the result
+ response = getProcedureResult(request);
+ } catch (IOException e) {
+ serviceEx = unwrapException(e);
+
+ // the master may be down
+ LOG.warn("failed to get the procedure result procId=" + procId, serviceEx);
+
+ // Not much to do, if we have a DoNotRetryIOException
+ if (serviceEx instanceof DoNotRetryIOException) {
+ // TODO: looks like there is no way to unwrap this exception and get the proper
+ // UnsupportedOperationException aside from looking at the message.
+ // anyway, if we fail here we just failover to the compatibility side
+ // and that is always a valid solution.
+ LOG.warn("Proc-v2 is unsupported on this master: " + serviceEx.getMessage(), serviceEx);
+ procResultFound = false;
+ return null;
+ }
+ }
+
+ // If the procedure is no longer running, we should have a result
+ if (response != null && response.getState() != GetProcedureResultResponse.State.RUNNING) {
+ procResultFound = response.getState() != GetProcedureResultResponse.State.NOT_FOUND;
+ return convertResult(response);
+ }
+
+ try {
+ Thread.sleep(getAdmin().getPauseTime(tries++));
+ } catch (InterruptedException e) {
+ throw new InterruptedException(
+ "Interrupted while waiting for the result of proc " + procId);
+ }
+ }
+ if (serviceEx != null) {
+ throw serviceEx;
+ } else {
+ throw new TimeoutException("The procedure " + procId + " is still running");
+ }
+ }
+
+ private static IOException unwrapException(IOException e) {
+ if (e instanceof RemoteException) {
+ return ((RemoteException)e).unwrapRemoteException();
+ }
+ return e;
+ }
+
+ protected GetProcedureResultResponse getProcedureResult(final GetProcedureResultRequest request)
+ throws IOException {
+ return admin.executeCallable(new MasterCallable<GetProcedureResultResponse>(
+ admin.getConnection()) {
+ @Override
+ public GetProcedureResultResponse call(int callTimeout) throws ServiceException {
+ return master.getProcedureResult(null, request);
+ }
+ });
+ }
+
+ /**
+ * Convert the procedure result response to a specified type.
+ * @param response the procedure result object to parse
+ * @return the result data of the procedure.
+ */
+ protected V convertResult(final GetProcedureResultResponse response) throws IOException {
+ if (response.hasException()) {
+ throw ForeignExceptionUtil.toIOException(response.getException());
+ }
+ return null;
+ }
+
+ /**
+ * Fallback implementation in case the procedure is not supported by the server.
+ * It should try to wait until the operation is completed.
+ * @param deadlineTs the timestamp after which this method should throw a TimeoutException
+ * @return the result data of the operation
+ */
+ protected V waitOperationResult(final long deadlineTs)
+ throws IOException, TimeoutException {
+ return null;
+ }
+
+ /**
+ * Called after the operation is completed and the result fetched.
+ * this allows to perform extra steps after the procedure is completed.
+ * it allows to apply transformations to the result that will be returned by get().
+ * @param result the result of the procedure
+ * @param deadlineTs the timestamp after which this method should throw a TimeoutException
+ * @return the result of the procedure, which may be the same as the passed one
+ */
+ protected V postOperationResult(final V result, final long deadlineTs)
+ throws IOException, TimeoutException {
+ return result;
+ }
+
+ /**
+ * Called after the operation is terminated with a failure.
+ * this allows to perform extra steps after the procedure is terminated.
+ * it allows to apply transformations to the result that will be returned by get().
+ * The default implementation will rethrow the exception
+ * @param exception the exception got from fetching the result
+ * @param deadlineTs the timestamp after which this method should throw a TimeoutException
+ * @return the result of the procedure, which may be the same as the passed one
+ */
+ protected V postOpeartionFailure(final IOException exception, final long deadlineTs)
+ throws IOException, TimeoutException {
+ throw exception;
+ }
+
+ protected interface WaitForStateCallable {
+ boolean checkState(int tries) throws IOException;
+ void throwInterruptedException() throws InterruptedIOException;
+ void throwTimeoutException(long elapsed) throws TimeoutException;
+ }
+
+ protected void waitForState(final long deadlineTs, final WaitForStateCallable callable)
+ throws IOException, TimeoutException {
+ int tries = 0;
+ IOException serverEx = null;
+ long startTime = EnvironmentEdgeManager.currentTime();
+ while (EnvironmentEdgeManager.currentTime() < deadlineTs) {
+ serverEx = null;
+ try {
+ if (callable.checkState(tries)) {
+ return;
+ }
+ } catch (IOException e) {
+ serverEx = e;
+ }
+ try {
+ Thread.sleep(getAdmin().getPauseTime(tries++));
+ } catch (InterruptedException e) {
+ callable.throwInterruptedException();
+ }
+ }
+ if (serverEx != null) {
+ throw unwrapException(serverEx);
+ } else {
+ callable.throwTimeoutException(EnvironmentEdgeManager.currentTime() - startTime);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --cc hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 7bb9de1,8b5b2d7..3b71435
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@@ -39,8 -37,6 +39,7 @@@ import java.util.Map.Entry
import java.util.NavigableSet;
import java.util.concurrent.TimeUnit;
+import com.google.protobuf.*;
- import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
@@@ -2990,85 -3024,39 +3019,120 @@@ public final class ProtobufUtil
return desc.build();
}
+
-
+ /**
+ * This version of protobuf's mergeDelimitedFrom avoid the hard-coded 64MB limit for decoding
+ * buffers
+ * @param builder current message builder
+ * @param in Inputsream with delimited protobuf data
+ * @throws IOException
+ */
+ public static void mergeDelimitedFrom(Message.Builder builder, InputStream in) throws IOException {
+ // This used to be builder.mergeDelimitedFrom(in);
+ // but is replaced to allow us to bump the protobuf size limit.
+ final int firstByte = in.read();
+ if (firstByte == -1) {
+ // bail out. (was return false;)
+ } else {
+ final int size = CodedInputStream.readRawVarint32(firstByte, in);
+ final InputStream limitedInput = new LimitedInputStream(in, size);
+ final CodedInputStream codedInput = CodedInputStream.newInstance(limitedInput);
+ codedInput.setSizeLimit(size);
+ builder.mergeFrom(codedInput);
+ codedInput.checkLastTagWas(0);
+ }
+ }
+
+ /**
+ * This is cut and paste from protobuf's package private AbstractMessageLite.
+ *
+ * An InputStream implementations which reads from some other InputStream
+ * but is limited to a particular number of bytes. Used by
+ * mergeDelimitedFrom(). This is intentionally package-private so that
+ * UnknownFieldSet can share it.
+ */
+ static final class LimitedInputStream extends FilterInputStream {
+ private int limit;
+
+ LimitedInputStream(InputStream in, int limit) {
+ super(in);
+ this.limit = limit;
+ }
+
+ @Override
+ public int available() throws IOException {
+ return Math.min(super.available(), limit);
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (limit <= 0) {
+ return -1;
+ }
+ final int result = super.read();
+ if (result >= 0) {
+ --limit;
+ }
+ return result;
+ }
+
+ @Override
+ public int read(final byte[] b, final int off, int len)
+ throws IOException {
+ if (limit <= 0) {
+ return -1;
+ }
+ len = Math.min(len, limit);
+ final int result = super.read(b, off, len);
+ if (result >= 0) {
+ limit -= result;
+ }
+ return result;
+ }
+
+ @Override
+ public long skip(final long n) throws IOException {
+ final long result = super.skip(Math.min(n, limit));
+ if (result >= 0) {
+ limit -= result;
+ }
+ return result;
+ }
+ }
++
+ public static ReplicationLoadSink toReplicationLoadSink(
+ ClusterStatusProtos.ReplicationLoadSink cls) {
+ return new ReplicationLoadSink(cls.getAgeOfLastAppliedOp(), cls.getTimeStampsOfLastAppliedOp());
+ }
+
+ public static ReplicationLoadSource toReplicationLoadSource(
+ ClusterStatusProtos.ReplicationLoadSource cls) {
+ return new ReplicationLoadSource(cls.getPeerID(), cls.getAgeOfLastShippedOp(),
+ cls.getSizeOfLogQueue(), cls.getTimeStampOfLastShippedOp(), cls.getReplicationLag());
+ }
+
+ public static List<ReplicationLoadSource> toReplicationLoadSourceList(
+ List<ClusterStatusProtos.ReplicationLoadSource> clsList) {
+ ArrayList<ReplicationLoadSource> rlsList = new ArrayList<ReplicationLoadSource>();
+ for (ClusterStatusProtos.ReplicationLoadSource cls : clsList) {
+ rlsList.add(toReplicationLoadSource(cls));
+ }
+ return rlsList;
+ }
+
+ /**
+ * Get a protocol buffer VersionInfo
+ *
+ * @return the converted protocol buffer VersionInfo
+ */
+ public static RPCProtos.VersionInfo getVersionInfo() {
+ RPCProtos.VersionInfo.Builder builder = RPCProtos.VersionInfo.newBuilder();
+ builder.setVersion(VersionInfo.getVersion());
+ builder.setUrl(VersionInfo.getUrl());
+ builder.setRevision(VersionInfo.getRevision());
+ builder.setUser(VersionInfo.getUser());
+ builder.setDate(VersionInfo.getDate());
+ builder.setSrcChecksum(VersionInfo.getSrcChecksum());
+ return builder.build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
----------------------------------------------------------------------
diff --cc hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
index 7f3838f,7728112..678cc7b
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
@@@ -36,16 -34,8 +36,16 @@@ public class PrettyPrinter
StringBuilder human = new StringBuilder();
switch (unit) {
case TIME_INTERVAL:
- human.append(humanReadableTTL(Long.valueOf(value)));
+ human.append(humanReadableTTL(Long.parseLong(value)));
break;
+ case LONG:
+ byte[] longBytes = Bytes.toBytesBinary(value);
+ human.append(String.valueOf(Bytes.toLong(longBytes)));
+ break;
+ case BOOLEAN:
+ byte[] booleanBytes = Bytes.toBytesBinary(value);
+ human.append(String.valueOf(Bytes.toBoolean(booleanBytes)));
+ break;
default:
human.append(value);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
----------------------------------------------------------------------
diff --cc hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index 2aad115,9a66da0..b609b4a
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@@ -253,77 -258,6 +258,76 @@@ public interface MetricsRegionServerWra
long getMajorCompactedCellsSize();
/**
- <<<<<<< HEAD
+ * Gets the number of cells move to mob during compaction.
+ */
+ long getMobCompactedIntoMobCellsCount();
+
+ /**
+ * Gets the number of cells move from mob during compaction.
+ */
+ long getMobCompactedFromMobCellsCount();
+
+ /**
+ * Gets the total amount of cells move to mob during compaction, in bytes.
+ */
+ long getMobCompactedIntoMobCellsSize();
+
+ /**
+ * Gets the total amount of cells move from mob during compaction, in bytes.
+ */
+ long getMobCompactedFromMobCellsSize();
+
+ /**
+ * Gets the number of the flushes in mob-enabled stores.
+ */
+ long getMobFlushCount();
+
+ /**
+ * Gets the number of mob cells flushed to disk.
+ */
+ long getMobFlushedCellsCount();
+
+ /**
+ * Gets the total amount of mob cells flushed to disk, in bytes.
+ */
+ long getMobFlushedCellsSize();
+
+ /**
+ * Gets the number of scanned mob cells.
+ */
+ long getMobScanCellsCount();
+
+ /**
+ * Gets the total amount of scanned mob cells, in bytes.
+ */
+ long getMobScanCellsSize();
+
+ /**
+ * Gets the count of accesses to the mob file cache.
+ */
+ long getMobFileCacheAccessCount();
+
+ /**
+ * Gets the count of misses to the mob file cache.
+ */
+ long getMobFileCacheMissCount();
+
+ /**
+ * Gets the number of items evicted from the mob file cache.
+ */
+ long getMobFileCacheEvictedCount();
+
+ /**
+ * Gets the count of cached mob files.
+ */
+ long getMobFileCacheCount();
+
+ /**
+ * Gets the hit percent to the mob file cache.
+ */
+ int getMobFileCacheHitPercent();
+
+ /**
* @return Count of hedged read operations
*/
public long getHedgedReadOps();
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithMOB.java
----------------------------------------------------------------------
diff --cc hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithMOB.java
index 85c01cc,0000000..82a599c
mode 100644,000000..100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithMOB.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithMOB.java
@@@ -1,155 -1,0 +1,156 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.hadoop.conf.Configuration;
- import org.apache.hadoop.hbase.client.HBaseAdmin;
++import org.apache.hadoop.hbase.client.Admin;
++import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.testclassification.IntegrationTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.LoadTestDataGeneratorWithMOB;
+import org.apache.hadoop.hbase.util.LoadTestTool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Integration Test for MOB ingest.
+ */
+@Category(IntegrationTests.class)
+public class IntegrationTestIngestWithMOB extends IntegrationTestIngest {
+ private static final char COLON = ':';
+
- private byte[] mobColumnFamily = LoadTestTool.COLUMN_FAMILY;
++ private byte[] mobColumnFamily = LoadTestTool.DEFAULT_COLUMN_FAMILY;
+ public static final String THRESHOLD = "threshold";
+ public static final String MIN_MOB_DATA_SIZE = "minMobDataSize";
+ public static final String MAX_MOB_DATA_SIZE = "maxMobDataSize";
+ private int threshold = 1024; // 1KB
+ private int minMobDataSize = 512; // 512B
+ private int maxMobDataSize = threshold * 5; // 5KB
+ private static final long JUNIT_RUN_TIME = 2 * 60 * 1000; // 2 minutes
+
+ //similar to LOAD_TEST_TOOL_INIT_ARGS except OPT_IN_MEMORY is removed
+ protected String[] LOAD_TEST_TOOL_MOB_INIT_ARGS = {
+ LoadTestTool.OPT_COMPRESSION,
+ LoadTestTool.OPT_DATA_BLOCK_ENCODING,
+ LoadTestTool.OPT_ENCRYPTION,
+ LoadTestTool.OPT_NUM_REGIONS_PER_SERVER,
+ LoadTestTool.OPT_REGION_REPLICATION,
+ };
+
+ @Override
+ protected String[] getArgsForLoadTestToolInitTable() {
+ List<String> args = new ArrayList<String>();
+ args.add("-tn");
+ args.add(getTablename().getNameAsString());
+ // pass all remaining args from conf with keys <test class name>.<load test tool arg>
+ String clazz = this.getClass().getSimpleName();
+ for (String arg : LOAD_TEST_TOOL_MOB_INIT_ARGS) {
+ String val = conf.get(String.format("%s.%s", clazz, arg));
+ if (val != null) {
+ args.add("-" + arg);
+ args.add(val);
+ }
+ }
+ args.add("-init_only");
+ return args.toArray(new String[args.size()]);
+ }
+
+ @Override
+ protected void addOptions() {
+ super.addOptions();
+ super.addOptWithArg(THRESHOLD, "The threshold to classify cells to mob data");
+ super.addOptWithArg(MIN_MOB_DATA_SIZE, "Minimum value size for mob data");
+ super.addOptWithArg(MAX_MOB_DATA_SIZE, "Maximum value size for mob data");
+ }
+
+ @Override
+ protected void processOptions(CommandLine cmd) {
+ super.processOptions(cmd);
+ if (cmd.hasOption(THRESHOLD)) {
+ threshold = Integer.parseInt(cmd.getOptionValue(THRESHOLD));
+ }
+ if (cmd.hasOption(MIN_MOB_DATA_SIZE)) {
+ minMobDataSize = Integer.parseInt(cmd.getOptionValue(MIN_MOB_DATA_SIZE));
+ }
+ if (cmd.hasOption(MAX_MOB_DATA_SIZE)) {
+ maxMobDataSize = Integer.parseInt(cmd.getOptionValue(MAX_MOB_DATA_SIZE));
+ }
+ if (minMobDataSize > maxMobDataSize) {
+ throw new IllegalArgumentException(
+ "The minMobDataSize should not be larger than minMobDataSize");
+ }
+ }
+
+ @Test
+ public void testIngest() throws Exception {
+ runIngestTest(JUNIT_RUN_TIME, 100, 10, 1024, 10, 20);
+ };
+
+ @Override
+ protected void initTable() throws IOException {
+ super.initTable();
+
- byte[] tableName = getTablename().getName();
- HBaseAdmin admin = new HBaseAdmin(conf);
++ TableName tableName = getTablename();
++ Admin admin = ConnectionFactory.createConnection().getAdmin();
+ HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
+ LOG.info("Disabling table " + getTablename());
+ admin.disableTable(tableName);
+ for (HColumnDescriptor columnDescriptor : tableDesc.getFamilies()) {
+ if(Arrays.equals(columnDescriptor.getName(), mobColumnFamily)) {
+ columnDescriptor.setMobEnabled(true);
+ columnDescriptor.setMobThreshold((long) threshold);
+ admin.modifyColumn(tableName, columnDescriptor);
+ }
+ }
+ LOG.info("Enabling table " + getTablename());
+ admin.enableTable(tableName);
+ admin.close();
+ }
+
+ @Override
+ protected String[] getArgsForLoadTestTool(String mode, String modeSpecificArg, long startKey,
+ long numKeys) {
+ String[] args = super.getArgsForLoadTestTool(mode, modeSpecificArg, startKey, numKeys);
+ List<String> tmp = new ArrayList<String>(Arrays.asList(args));
+ // LoadTestDataGeneratorMOB:mobColumnFamily:minMobDataSize:maxMobDataSize
+ tmp.add(HIPHEN + LoadTestTool.OPT_GENERATOR);
+ StringBuilder sb = new StringBuilder(LoadTestDataGeneratorWithMOB.class.getName());
+ sb.append(COLON);
+ sb.append(Bytes.toString(mobColumnFamily));
+ sb.append(COLON);
+ sb.append(minMobDataSize);
+ sb.append(COLON);
+ sb.append(maxMobDataSize);
+ tmp.add(sb.toString());
+ return tmp.toArray(new String[tmp.size()]);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+ IntegrationTestingUtility.setUseDistributedCluster(conf);
+ int ret = ToolRunner.run(conf, new IntegrationTestIngestWithMOB(), args);
+ System.exit(ret);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e20bbf6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --cc hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index fe6f06e,9bd1dbb..5a1e188
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@@ -110,8 -107,8 +108,10 @@@ import org.apache.hadoop.hbase.monitori
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager;
+ import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+ import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
@@@ -127,10 -124,9 +127,10 @@@ import org.apache.hadoop.hbase.util.Byt
import org.apache.hadoop.hbase.util.CompressionTest;
import org.apache.hadoop.hbase.util.EncryptionTest;
import org.apache.hadoop.hbase.util.FSUtils;
- import org.apache.hadoop.hbase.util.HBaseFsckRepair;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.hbase.util.HasThread;
+import org.apache.hadoop.hbase.util.IdLock;
+ import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.VersionInfo;
@@@ -1078,13 -1077,35 +1096,41 @@@ public class HMaster extends HRegionSer
if (this.mpmHost != null) this.mpmHost.stop("server shutting down.");
}
+ private void startProcedureExecutor() throws IOException {
+ final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
+ final Path logDir = new Path(fileSystemManager.getRootDir(),
+ MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR);
+
+ procedureStore = new WALProcedureStore(conf, fileSystemManager.getFileSystem(), logDir,
+ new MasterProcedureEnv.WALStoreLeaseRecovery(this));
+ procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
+ procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore,
+ procEnv.getProcedureQueue());
+
+ final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
+ Math.max(Runtime.getRuntime().availableProcessors(),
+ MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
+ procedureStore.start(numThreads);
+ procedureExecutor.start(numThreads);
+ }
+
+ private void stopProcedureExecutor() {
+ if (procedureExecutor != null) {
+ procedureExecutor.stop();
+ }
+
+ if (procedureStore != null) {
+ procedureStore.stop(isAborted());
+ }
+ }
+
private void stopChores() {
+ if (this.expiredMobFileCleanerChore != null) {
+ this.expiredMobFileCleanerChore.cancel(true);
+ }
+ if (this.mobFileCompactChore != null) {
+ this.mobFileCompactChore.cancel(true);
+ }
if (this.balancerChore != null) {
this.balancerChore.cancel(true);
}
@@@ -2333,56 -2408,23 +2436,76 @@@
}
/**
+ * Gets the mob file compaction state for a specific table.
+ * Whether all the mob files are selected is known during the compaction execution, but
+ * the statistic is done just before compaction starts, it is hard to know the compaction
+ * type at that time, so the rough statistics are chosen for the mob file compaction. Only two
+ * compaction states are available, CompactionState.MAJOR_AND_MINOR and CompactionState.NONE.
+ * @param tableName The current table name.
+ * @return If a given table is in mob file compaction now.
+ */
+ public CompactionState getMobCompactionState(TableName tableName) {
+ AtomicInteger compactionsCount = mobFileCompactionStates.get(tableName);
+ if (compactionsCount != null && compactionsCount.get() != 0) {
+ return CompactionState.MAJOR_AND_MINOR;
+ }
+ return CompactionState.NONE;
+ }
+
+ public void reportMobFileCompactionStart(TableName tableName) throws IOException {
+ IdLock.Entry lockEntry = null;
+ try {
+ lockEntry = mobFileCompactionLock.getLockEntry(tableName.hashCode());
+ AtomicInteger compactionsCount = mobFileCompactionStates.get(tableName);
+ if (compactionsCount == null) {
+ compactionsCount = new AtomicInteger(0);
+ mobFileCompactionStates.put(tableName, compactionsCount);
+ }
+ compactionsCount.incrementAndGet();
+ } finally {
+ if (lockEntry != null) {
+ mobFileCompactionLock.releaseLockEntry(lockEntry);
+ }
+ }
+ }
+
+ public void reportMobFileCompactionEnd(TableName tableName) throws IOException {
+ IdLock.Entry lockEntry = null;
+ try {
+ lockEntry = mobFileCompactionLock.getLockEntry(tableName.hashCode());
+ AtomicInteger compactionsCount = mobFileCompactionStates.get(tableName);
+ if (compactionsCount != null) {
+ int count = compactionsCount.decrementAndGet();
+ // remove the entry if the count is 0.
+ if (count == 0) {
+ mobFileCompactionStates.remove(tableName);
+ }
+ }
+ } finally {
+ if (lockEntry != null) {
+ mobFileCompactionLock.releaseLockEntry(lockEntry);
+ }
+ }
++ }
++
++ /**
+ * Queries the state of the {@link LoadBalancerTracker}. If the balancer is not initialized,
+ * false is returned.
+ *
+ * @return The state of the load balancer, or false if the load balancer isn't defined.
+ */
+ public boolean isBalancerOn() {
+ if (null == loadBalancerTracker) return false;
+ return loadBalancerTracker.isBalancerOn();
+ }
+ /**
+ * Fetch the configured {@link LoadBalancer} class name. If none is set, a default is returned.
+ *
+ * @return The name of the {@link LoadBalancer} in use.
+ */
+ public String getLoadBalancerClassName() {
+ return conf.get(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, LoadBalancerFactory
+ .getDefaultLoadBalancerClass().getName());
}
}