You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2014/05/23 13:05:39 UTC
[1/2] git commit: HBASE-10801-Ensure DBE interfaces can work with
Cell (Ram)
Repository: hbase
Updated Branches:
refs/heads/master c61cb7fb5 -> 8e13fea26
HBASE-10801-Ensure DBE interfaces can work with Cell (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f36b42b9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f36b42b9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f36b42b9
Branch: refs/heads/master
Commit: f36b42b9e9778e8a267e89c58b78eab7622af319
Parents: 264725d
Author: Ramkrishna <ra...@intel.com>
Authored: Fri May 23 16:33:11 2014 +0530
Committer: Ramkrishna <ra...@intel.com>
Committed: Fri May 23 16:33:11 2014 +0530
----------------------------------------------------------------------
.../io/encoding/BufferedDataBlockEncoder.java | 350 ++++++++++++++++++-
1 file changed, 344 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f36b42b9/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
index af78c1c..fe019d1 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.io.TagCompressionContext;
import org.apache.hadoop.hbase.io.hfile.BlockType;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
@@ -74,7 +75,9 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
return internalDecodeKeyValues(source, 0, 0, decodingCtx);
}
- protected static class SeekerState {
+ protected static class SeekerState implements Cell {
+ protected ByteBuffer currentBuffer;
+ protected TagCompressionContext tagCompressionContext;
protected int valueOffset = -1;
protected int keyLength;
protected int valueLength;
@@ -90,6 +93,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected long memstoreTS;
protected int nextKvOffset;
+ protected KeyValue.KeyOnlyKeyValue currentKey = new KeyValue.KeyOnlyKeyValue();
protected boolean isValid() {
return valueOffset != -1;
@@ -98,7 +102,9 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected void invalidate() {
valueOffset = -1;
tagsCompressedLength = 0;
+ currentKey = new KeyValue.KeyOnlyKeyValue();
uncompressTags = true;
+ currentBuffer = null;
}
protected void ensureSpaceForKey() {
@@ -127,6 +133,11 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
}
}
+ protected void createKeyOnlyKeyValue(byte[] keyBuffer, long memTS) {
+ currentKey.setKey(keyBuffer, 0, keyLength);
+ memstoreTS = memTS;
+ }
+
/**
* Copy the state from the next one into this instance (the previous state
* placeholder). Used to save the previous state when we are advancing the
@@ -146,6 +157,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
keyBuffer, nextState.lastCommonPrefix, nextState.keyLength
- nextState.lastCommonPrefix);
}
+ currentKey = nextState.currentKey;
valueOffset = nextState.valueOffset;
keyLength = nextState.keyLength;
@@ -153,8 +165,330 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
lastCommonPrefix = nextState.lastCommonPrefix;
nextKvOffset = nextState.nextKvOffset;
memstoreTS = nextState.memstoreTS;
+ currentBuffer = nextState.currentBuffer;
+ if (nextState.tagCompressionContext != null) {
+ tagCompressionContext = nextState.tagCompressionContext;
+ }
+ }
+
+ @Override
+ public byte[] getRowArray() {
+ return currentKey.getRowArray();
+ }
+
+ @Override
+ public int getRowOffset() {
+ return Bytes.SIZEOF_SHORT;
+ }
+
+ @Override
+ public short getRowLength() {
+ return currentKey.getRowLength();
+ }
+
+ @Override
+ public byte[] getFamilyArray() {
+ return currentKey.getFamilyArray();
+ }
+
+ @Override
+ public int getFamilyOffset() {
+ return currentKey.getFamilyOffset();
+ }
+
+ @Override
+ public byte getFamilyLength() {
+ return currentKey.getFamilyLength();
+ }
+
+ @Override
+ public byte[] getQualifierArray() {
+ return currentKey.getQualifierArray();
+ }
+
+ @Override
+ public int getQualifierOffset() {
+ return currentKey.getQualifierOffset();
+ }
+
+ @Override
+ public int getQualifierLength() {
+ return currentKey.getQualifierLength();
+ }
+
+ @Override
+ public long getTimestamp() {
+ return currentKey.getTimestamp();
+ }
+
+ @Override
+ public byte getTypeByte() {
+ return currentKey.getTypeByte();
+ }
+
+ @Override
+ public long getMvccVersion() {
+ return memstoreTS;
+ }
+
+ @Override
+ public byte[] getValueArray() {
+ return currentBuffer.array();
+ }
+
+ @Override
+ public int getValueOffset() {
+ return currentBuffer.arrayOffset() + valueOffset;
+ }
+
+ @Override
+ public int getValueLength() {
+ return valueLength;
+ }
+
+ @Override
+ public byte[] getTagsArray() {
+ if (tagCompressionContext != null) {
+ return tagsBuffer;
+ }
+ return currentBuffer.array();
+ }
+
+ @Override
+ public int getTagsOffset() {
+ if (tagCompressionContext != null) {
+ return 0;
+ }
+ return currentBuffer.arrayOffset() + tagsOffset;
+ }
+
+ @Override
+ public short getTagsLength() {
+ return (short) tagsLength;
+ }
+
+ @Override
+ @Deprecated
+ public byte[] getValue() {
+ throw new UnsupportedOperationException("getValue() not supported");
+ }
+
+ @Override
+ @Deprecated
+ public byte[] getFamily() {
+ throw new UnsupportedOperationException("getFamily() not supported");
+ }
+
+ @Override
+ @Deprecated
+ public byte[] getQualifier() {
+ throw new UnsupportedOperationException("getQualifier() not supported");
+ }
+
+ @Override
+ @Deprecated
+ public byte[] getRow() {
+ throw new UnsupportedOperationException("getRow() not supported");
+ }
+
+ @Override
+ public String toString() {
+ KeyValue kv = KeyValueUtil.copyToNewKeyValue(this);
+ if (kv == null) {
+ return "null";
+ }
+ return kv.toString();
+ }
+
+ public Cell shallowCopy() {
+ return new ClonedSeekerState(currentBuffer, keyBuffer, currentKey.getRowLength(),
+ currentKey.getFamilyOffset(), currentKey.getFamilyLength(), keyLength,
+ currentKey.getQualifierOffset(), currentKey.getQualifierLength(),
+ currentKey.getTimestamp(), currentKey.getTypeByte(), valueLength, valueOffset,
+ memstoreTS, tagsOffset, tagsLength, tagCompressionContext, tagsBuffer);
+ }
+
+ }
+
+ /**
+ * Copies only the key part of the keybuffer by doing a deep copy and passes the
+ * seeker state members for taking a clone.
+ * Note that the value byte[] part is still pointing to the currentBuffer and the
+ * represented by the valueOffset and valueLength
+ */
+ protected static class ClonedSeekerState implements Cell {
+ private byte[] keyOnlyBuffer;
+ private ByteBuffer currentBuffer;
+ private short rowLength;
+ private int familyOffset;
+ private byte familyLength;
+ private int qualifierOffset;
+ private int qualifierLength;
+ private long timestamp;
+ private byte typeByte;
+ private int valueOffset;
+ private int valueLength;
+ private int tagsLength;
+ private int tagsOffset;
+ private byte[] cloneTagsBuffer;
+ private long memstoreTS;
+ private TagCompressionContext tagCompressionContext;
+
+ protected ClonedSeekerState(ByteBuffer currentBuffer, byte[] keyBuffer, short rowLength,
+ int familyOffset, byte familyLength, int keyLength, int qualOffset, int qualLength,
+ long timeStamp, byte typeByte, int valueLen, int valueOffset, long memStoreTS,
+ int tagsOffset, int tagsLength, TagCompressionContext tagCompressionContext,
+ byte[] tagsBuffer) {
+ this.currentBuffer = currentBuffer;
+ keyOnlyBuffer = new byte[keyLength];
+ this.tagCompressionContext = tagCompressionContext;
+ this.rowLength = rowLength;
+ this.familyOffset = familyOffset;
+ this.familyLength = familyLength;
+ this.qualifierOffset = qualOffset;
+ this.qualifierLength = qualLength;
+ this.timestamp = timeStamp;
+ this.typeByte = typeByte;
+ this.valueLength = valueLen;
+ this.valueOffset = valueOffset;
+ this.memstoreTS = memStoreTS;
+ this.tagsOffset = tagsOffset;
+ this.tagsLength = tagsLength;
+ System.arraycopy(keyBuffer, 0, keyOnlyBuffer, 0, keyLength);
+ if (tagCompressionContext != null) {
+ this.cloneTagsBuffer = new byte[tagsLength];
+ System.arraycopy(tagsBuffer, 0, this.cloneTagsBuffer, 0, tagsLength);
+ }
}
+ @Override
+ public byte[] getRowArray() {
+ return keyOnlyBuffer;
+ }
+
+ @Override
+ public byte[] getFamilyArray() {
+ return keyOnlyBuffer;
+ }
+
+ @Override
+ public byte[] getQualifierArray() {
+ return keyOnlyBuffer;
+ }
+
+ @Override
+ public int getRowOffset() {
+ return Bytes.SIZEOF_SHORT;
+ }
+
+ @Override
+ public short getRowLength() {
+ return rowLength;
+ }
+
+ @Override
+ public int getFamilyOffset() {
+ return familyOffset;
+ }
+
+ @Override
+ public byte getFamilyLength() {
+ return familyLength;
+ }
+
+ @Override
+ public int getQualifierOffset() {
+ return qualifierOffset;
+ }
+
+ @Override
+ public int getQualifierLength() {
+ return qualifierLength;
+ }
+
+ @Override
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public byte getTypeByte() {
+ return typeByte;
+ }
+
+ @Override
+ public long getMvccVersion() {
+ return memstoreTS;
+ }
+
+ @Override
+ public byte[] getValueArray() {
+ return currentBuffer.array();
+ }
+
+ @Override
+ public int getValueOffset() {
+ return currentBuffer.arrayOffset() + valueOffset;
+ }
+
+ @Override
+ public int getValueLength() {
+ return valueLength;
+ }
+
+ @Override
+ public byte[] getTagsArray() {
+ if (tagCompressionContext != null) {
+ return cloneTagsBuffer;
+ }
+ return currentBuffer.array();
+ }
+
+ @Override
+ public int getTagsOffset() {
+ if (tagCompressionContext != null) {
+ return 0;
+ }
+ return currentBuffer.arrayOffset() + tagsOffset;
+ }
+
+ @Override
+ public short getTagsLength() {
+ return (short) tagsLength;
+ }
+
+ @Override
+ @Deprecated
+ public byte[] getValue() {
+ throw new UnsupportedOperationException("getValue() not supported");
+ }
+
+ @Override
+ @Deprecated
+ public byte[] getFamily() {
+ throw new UnsupportedOperationException("getFamily() not supported");
+ }
+
+ @Override
+ @Deprecated
+ public byte[] getQualifier() {
+ throw new UnsupportedOperationException("getQualifier() not supported");
+ }
+
+ @Override
+ @Deprecated
+ public byte[] getRow() {
+ throw new UnsupportedOperationException("getRow() not supported");
+ }
+
+ @Override
+ public String toString() {
+ KeyValue kv = KeyValueUtil.copyToNewKeyValue(this);
+ if (kv == null) {
+ return "null";
+ }
+ return kv.toString();
+ }
}
protected abstract static class
@@ -208,7 +542,12 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
this.tagCompressionContext.clear();
}
currentBuffer = buffer;
+ current.currentBuffer = currentBuffer;
+ if(tagCompressionContext != null) {
+ current.tagCompressionContext = tagCompressionContext;
+ }
decodeFirst();
+ current.createKeyOnlyKeyValue(current.keyBuffer, current.memstoreTS);
previous.invalidate();
}
@@ -260,11 +599,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
@Override
public Cell getKeyValue() {
- ByteBuffer kvBuf = getKeyValueBuffer();
- KeyValue kv = new KeyValue(kvBuf.array(), kvBuf.arrayOffset(), kvBuf.array().length
- - kvBuf.arrayOffset());
- kv.setMvccVersion(current.memstoreTS);
- return kv;
+ return current.shallowCopy();
}
@Override
@@ -274,6 +609,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
tagCompressionContext.clear();
}
decodeFirst();
+ current.createKeyOnlyKeyValue(current.keyBuffer, current.memstoreTS);
previous.invalidate();
}
@@ -283,6 +619,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
return false;
}
decodeNext();
+ current.createKeyOnlyKeyValue(current.keyBuffer, current.memstoreTS);
previous.invalidate();
return true;
}
@@ -416,6 +753,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
if (currentBuffer.hasRemaining()) {
previous.copyFromNext(current);
decodeNext();
+ current.createKeyOnlyKeyValue(current.keyBuffer, current.memstoreTS);
} else {
break;
}
[2/2] git commit: Merge remote-tracking branch 'origin/master'
Posted by ra...@apache.org.
Merge remote-tracking branch 'origin/master'
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8e13fea2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8e13fea2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8e13fea2
Branch: refs/heads/master
Commit: 8e13fea268be3a8709f1b2213af8d5d49638d13a
Parents: f36b42b c61cb7f
Author: Ramkrishna <ra...@intel.com>
Authored: Fri May 23 16:35:11 2014 +0530
Committer: Ramkrishna <ra...@intel.com>
Committed: Fri May 23 16:35:11 2014 +0530
----------------------------------------------------------------------
.../apache/hadoop/hbase/ConsensusProvider.java | 58 ---
.../hadoop/hbase/CoordinatedStateException.java | 46 +++
.../hadoop/hbase/CoordinatedStateManager.java | 66 +++
.../java/org/apache/hadoop/hbase/Server.java | 4 +-
.../apache/hadoop/hbase/TableStateManager.java | 115 ++++++
.../hadoop/hbase/client/AsyncProcess.java | 8 +-
.../hadoop/hbase/client/ClientScanner.java | 46 ++-
.../client/ClientSmallReversedScanner.java | 2 +-
.../hadoop/hbase/client/ClientSmallScanner.java | 28 +-
.../hadoop/hbase/client/ConnectionManager.java | 5 +-
.../org/apache/hadoop/hbase/client/HTable.java | 41 +-
.../hbase/client/MultiServerCallable.java | 9 +-
.../hbase/client/ReversedClientScanner.java | 5 +-
.../hbase/client/ReversedScannerCallable.java | 20 +-
.../hadoop/hbase/client/ScannerCallable.java | 16 +-
.../hadoop/hbase/client/ZooKeeperRegistry.java | 6 +-
.../DelegatingPayloadCarryingRpcController.java | 58 +++
.../org/apache/hadoop/hbase/ipc/RpcClient.java | 5 +-
.../hadoop/hbase/ipc/RpcControllerFactory.java | 59 +++
.../hbase/security/HBaseSaslRpcClient.java | 27 +-
.../apache/hadoop/hbase/security/SaslUtil.java | 32 ++
.../apache/hadoop/hbase/zookeeper/ZKTable.java | 406 -------------------
.../hadoop/hbase/zookeeper/ZKTableReadOnly.java | 167 --------
.../zookeeper/ZKTableStateClientSideReader.java | 168 ++++++++
.../hbase/zookeeper/ZKTableStateManager.java | 330 +++++++++++++++
.../hadoop/hbase/client/TestAsyncProcess.java | 5 +-
.../org/apache/hadoop/hbase/HConstants.java | 3 +-
.../hbase/io/encoding/FastDiffDeltaEncoder.java | 3 +-
.../io/encoding/PrefixKeyDeltaEncoder.java | 3 +-
.../src/main/resources/hbase-default.xml | 6 +-
.../regionserver/MetricsRegionServerSource.java | 2 +-
.../MetricsRegionServerWrapper.java | 2 +-
.../hbase/IntegrationTestIngestWithACL.java | 32 +-
.../hadoop/hbase/ConsensusProviderFactory.java | 43 --
.../hbase/CoordinatedStateManagerFactory.java | 43 ++
.../apache/hadoop/hbase/LocalHBaseCluster.java | 13 +-
.../hbase/consensus/BaseConsensusProvider.java | 49 ---
.../consensus/BaseCoordinatedStateManager.java | 55 +++
.../hbase/consensus/ZkConsensusProvider.java | 42 --
.../consensus/ZkCoordinatedStateManager.java | 59 +++
.../hbase/mapreduce/KeyValueSortReducer.java | 2 +-
.../hadoop/hbase/master/AssignmentManager.java | 109 +++--
.../org/apache/hadoop/hbase/master/HMaster.java | 26 +-
.../hadoop/hbase/master/HMasterCommandLine.java | 13 +-
.../hadoop/hbase/master/SplitLogManager.java | 35 +-
.../master/handler/ClosedRegionHandler.java | 5 +-
.../master/handler/CreateTableHandler.java | 22 +-
.../master/handler/DeleteTableHandler.java | 10 +-
.../master/handler/DisableTableHandler.java | 22 +-
.../master/handler/EnableTableHandler.java | 29 +-
.../master/handler/OpenedRegionHandler.java | 4 +-
.../master/handler/ServerShutdownHandler.java | 12 +-
.../hbase/master/handler/TableEventHandler.java | 11 +-
.../master/handler/TruncateTableHandler.java | 17 +-
.../hbase/master/snapshot/SnapshotManager.java | 11 +-
.../hbase/regionserver/HRegionServer.java | 26 +-
.../regionserver/HRegionServerCommandLine.java | 6 +-
.../MetricsRegionServerWrapperImpl.java | 2 +-
.../hbase/regionserver/SplitLogWorker.java | 5 +-
.../hbase/regionserver/wal/HLogSplitter.java | 28 +-
.../regionserver/wal/WALEditsReplaySink.java | 5 +-
.../regionserver/ReplicationSyncUp.java | 6 +-
.../hbase/security/HBaseSaslRpcServer.java | 33 +-
.../security/access/SecureBulkLoadEndpoint.java | 6 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 4 +-
.../hadoop/hbase/util/JVMClusterUtil.java | 10 +-
.../apache/hadoop/hbase/io/hfile/package.html | 25 --
.../hadoop/hbase/HBaseTestingUtility.java | 8 +-
.../apache/hadoop/hbase/MiniHBaseCluster.java | 2 +-
.../hadoop/hbase/MockRegionServerServices.java | 2 +-
.../apache/hadoop/hbase/TestDrainingServer.java | 15 +-
.../hadoop/hbase/TestLocalHBaseCluster.java | 4 +-
.../hbase/client/HConnectionTestingUtility.java | 6 +-
.../apache/hadoop/hbase/client/TestAdmin.java | 19 +-
.../client/TestClientScannerRPCTimeout.java | 4 +-
.../hbase/client/TestRpcControllerFactory.java | 203 ++++++++++
.../hadoop/hbase/io/hfile/TestCacheConfig.java | 176 ++++++++
.../hadoop/hbase/master/MockRegionServer.java | 4 +-
.../hbase/master/TestActiveMasterManager.java | 4 +-
.../hbase/master/TestAssignmentManager.java | 115 ++++--
.../master/TestAssignmentManagerOnCluster.java | 9 +-
.../hadoop/hbase/master/TestCatalogJanitor.java | 6 +-
.../hbase/master/TestClockSkewDetection.java | 4 +-
.../hbase/master/TestHMasterRPCException.java | 6 +-
.../apache/hadoop/hbase/master/TestMaster.java | 4 +-
.../hadoop/hbase/master/TestMasterFailover.java | 18 +-
.../hadoop/hbase/master/TestMasterMetrics.java | 4 +-
.../hbase/master/TestMasterNoCluster.java | 13 +-
.../TestMasterRestartAfterDisablingTable.java | 10 +-
.../hbase/master/TestOpenedRegionHandler.java | 4 +-
.../hbase/master/TestSplitLogManager.java | 45 --
.../hbase/master/cleaner/TestHFileCleaner.java | 4 +-
.../master/cleaner/TestHFileLinkCleaner.java | 4 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 4 +-
.../MetricsRegionServerWrapperStub.java | 2 +-
.../hbase/regionserver/OOMERegionServer.java | 4 +-
.../hbase/regionserver/TestClusterId.java | 6 +-
.../regionserver/TestHeapMemoryManager.java | 4 +-
.../regionserver/TestMetricsRegionServer.java | 2 +-
.../hbase/regionserver/TestPriorityRpc.java | 6 +-
.../TestRSKilledWhenInitializing.java | 4 +-
.../TestRegionMergeTransaction.java | 10 +-
.../regionserver/TestReversibleScanners.java | 47 ++-
.../TestSplitTransactionOnCluster.java | 4 +-
.../hbase/regionserver/wal/TestHLogMethods.java | 2 +-
.../hbase/regionserver/wal/TestHLogSplit.java | 8 +-
.../hbase/regionserver/wal/TestWALReplay.java | 4 +-
.../replication/TestReplicationStateZKImpl.java | 4 +-
.../TestReplicationTrackerZKImpl.java | 4 +-
.../TestReplicationSourceManager.java | 4 +-
.../hbase/security/TestHBaseSaslRpcClient.java | 18 +
.../security/token/TestTokenAuthentication.java | 4 +-
.../apache/hadoop/hbase/util/LoadTestTool.java | 93 ++++-
.../apache/hadoop/hbase/util/MockServer.java | 4 +-
.../hbase/util/MultiThreadedReaderWithACL.java | 7 +-
.../hbase/util/MultiThreadedUpdaterWithACL.java | 19 +-
.../hadoop/hbase/zookeeper/TestZKTable.java | 110 -----
.../zookeeper/TestZKTableStateManager.java | 114 ++++++
src/main/docbkx/developer.xml | 108 +++--
119 files changed, 2391 insertions(+), 1469 deletions(-)
----------------------------------------------------------------------