You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by an...@apache.org on 2015/05/28 07:42:58 UTC
hbase git commit: HBASE-13754 Allow non KeyValue Cell types also to
oswrite.
Repository: hbase
Updated Branches:
refs/heads/master 353b046d6 -> 7de4881e3
HBASE-13754 Allow non KeyValue Cell types also to oswrite.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7de4881e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7de4881e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7de4881e
Branch: refs/heads/master
Commit: 7de4881e3a6a18b00a9d14e24c6aae7f1b64919d
Parents: 353b046
Author: anoopsjohn <an...@gmail.com>
Authored: Thu May 28 11:12:37 2015 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Thu May 28 11:12:37 2015 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/KeyValue.java | 44 ++++++++----------
.../org/apache/hadoop/hbase/KeyValueUtil.java | 6 +--
.../org/apache/hadoop/hbase/NoTagsKeyValue.java | 15 +++++++
.../org/apache/hadoop/hbase/Streamable.java | 47 ++++++++++++++++++++
.../io/encoding/BufferedDataBlockEncoder.java | 40 ++++++++++++++++-
5 files changed, 122 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7de4881e/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index 8835957..1b7dd6b 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -79,7 +79,8 @@ import com.google.common.annotations.VisibleForTesting;
* and actual tag bytes length.
*/
@InterfaceAudience.Private
-public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, SettableTimestamp {
+public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId,
+ SettableTimestamp, Streamable {
private static final ArrayList<Tag> EMPTY_ARRAY_LIST = new ArrayList<Tag>();
private static final Log LOG = LogFactory.getLog(KeyValue.class);
@@ -2501,46 +2502,37 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId,
* Named <code>oswrite</code> so does not clash with {@link #write(KeyValue, DataOutput)}
* @param kv
* @param out
- * @return Length written on stream
- * @throws IOException
- * @see #create(DataInput) for the inverse function
- * @see #write(KeyValue, DataOutput)
- * @deprecated use {@link #oswrite(KeyValue, OutputStream, boolean)} instead
- */
- @Deprecated
- public static long oswrite(final KeyValue kv, final OutputStream out)
- throws IOException {
- int length = kv.getLength();
- // This does same as DataOuput#writeInt (big-endian, etc.)
- out.write(Bytes.toBytes(length));
- out.write(kv.getBuffer(), kv.getOffset(), length);
- return length + Bytes.SIZEOF_INT;
- }
-
- /**
- * Write out a KeyValue in the manner in which we used to when KeyValue was a Writable but do
- * not require a {@link DataOutput}, just take plain {@link OutputStream}
- * Named <code>oswrite</code> so does not clash with {@link #write(KeyValue, DataOutput)}
- * @param kv
- * @param out
* @param withTags
* @return Length written on stream
* @throws IOException
* @see #create(DataInput) for the inverse function
* @see #write(KeyValue, DataOutput)
* @see KeyValueUtil#oswrite(Cell, OutputStream, boolean)
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0.
+ * Instead use {@link #write(OutputStream, boolean)}
*/
+ @Deprecated
public static long oswrite(final KeyValue kv, final OutputStream out, final boolean withTags)
throws IOException {
+ return kv.write(out, withTags);
+ }
+
+ @Override
+ public int write(OutputStream out) throws IOException {
+ return write(out, true);
+ }
+
+ @Override
+ public int write(OutputStream out, boolean withTags) throws IOException {
// In KeyValueUtil#oswrite we do a Cell serialization as KeyValue. Any changes doing here, pls
// check KeyValueUtil#oswrite also and do necessary changes.
- int length = kv.getLength();
+ int length = this.length;
if (!withTags) {
- length = kv.getKeyLength() + kv.getValueLength() + KEYVALUE_INFRASTRUCTURE_SIZE;
+ length = this.getKeyLength() + this.getValueLength() + KEYVALUE_INFRASTRUCTURE_SIZE;
}
// This does same as DataOuput#writeInt (big-endian, etc.)
StreamUtils.writeInt(out, length);
- out.write(kv.getBuffer(), kv.getOffset(), length);
+ out.write(this.bytes, this.offset, length);
return length + Bytes.SIZEOF_INT;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7de4881e/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
index c94807b..3b0c05c 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
@@ -58,7 +58,7 @@ public class KeyValueUtil {
cell.getValueLength(), cell.getTagsLength(), true);
}
- private static int length(short rlen, byte flen, int qlen, int vlen, int tlen, boolean withTags) {
+ public static int length(short rlen, byte flen, int qlen, int vlen, int tlen, boolean withTags) {
if (withTags) {
return (int) (KeyValue.getKeyValueDataStructureSize(rlen, flen, qlen, vlen, tlen));
}
@@ -669,8 +669,8 @@ public class KeyValueUtil {
public static void oswrite(final Cell cell, final OutputStream out, final boolean withTags)
throws IOException {
- if (cell instanceof KeyValue) {
- KeyValue.oswrite((KeyValue) cell, out, withTags);
+ if (cell instanceof Streamable) {
+ ((Streamable)cell).write(out, withTags);
} else {
short rlen = cell.getRowLength();
byte flen = cell.getFamilyLength();
http://git-wip-us.apache.org/repos/asf/hbase/blob/7de4881e/hbase-common/src/main/java/org/apache/hadoop/hbase/NoTagsKeyValue.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/NoTagsKeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/NoTagsKeyValue.java
index c4c8351..6de6653 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/NoTagsKeyValue.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/NoTagsKeyValue.java
@@ -19,7 +19,12 @@
*/
package org.apache.hadoop.hbase;
+import java.io.IOException;
+import java.io.OutputStream;
+
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.io.util.StreamUtils;
+import org.apache.hadoop.hbase.util.Bytes;
/**
* An extension of the KeyValue where the tags length is always 0
@@ -34,4 +39,14 @@ public class NoTagsKeyValue extends KeyValue {
public int getTagsLength() {
return 0;
}
+
+ @Override
+ public int write(OutputStream out, boolean withTags) throws IOException {
+ // In KeyValueUtil#oswrite we do a Cell serialization as KeyValue. Any changes doing here, pls
+ // check KeyValueUtil#oswrite also and do necessary changes.
+ // This does same as DataOuput#writeInt (big-endian, etc.)
+ StreamUtils.writeInt(out, this.length);
+ out.write(this.bytes, this.offset, this.length);
+ return this.length + Bytes.SIZEOF_INT;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/7de4881e/hbase-common/src/main/java/org/apache/hadoop/hbase/Streamable.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/Streamable.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/Streamable.java
new file mode 100644
index 0000000..be91a56
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/Streamable.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * This marks a Cell as streamable to a given OutputStream.
+ */
+@InterfaceAudience.Private
+public interface Streamable {
+
+ /**
+ * Write this cell to an OutputStream.
+ * @param out Stream to which cell has to be written
+ * @return how many bytes are written.
+ * @throws IOException
+ */
+ int write(OutputStream out) throws IOException;
+
+ /**
+ * Write this cell to an OutputStream.
+ * @param out Stream to which cell has to be written
+ * @param withTags Whether to write tags.
+ * @return how many bytes are written.
+ * @throws IOException
+ */
+ int write(OutputStream out, boolean withTags) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/7de4881e/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
index 55fd811..62e81ab 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.io.encoding;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.OutputStream;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.Cell;
@@ -28,6 +29,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.Streamable;
import org.apache.hadoop.hbase.SettableSequenceId;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.io.HeapSize;
@@ -35,6 +37,7 @@ import org.apache.hadoop.hbase.io.TagCompressionContext;
import org.apache.hadoop.hbase.io.hfile.BlockType;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.util.LRUDictionary;
+import org.apache.hadoop.hbase.io.util.StreamUtils;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
@@ -345,7 +348,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
// there. So this has to be an instance of SettableSequenceId. SeekerState need not be
// SettableSequenceId as we never return that to top layers. When we have to, we make
// ClonedSeekerState from it.
- protected static class ClonedSeekerState implements Cell, HeapSize, SettableSequenceId {
+ protected static class ClonedSeekerState implements Cell, HeapSize, SettableSequenceId,
+ Streamable {
private static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT
+ (4 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG) + (7 * Bytes.SIZEOF_INT)
+ (Bytes.SIZEOF_SHORT) + (2 * Bytes.SIZEOF_BYTE) + (2 * ClassSize.ARRAY));
@@ -534,6 +538,40 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
public long heapSize() {
return FIXED_OVERHEAD + rowLength + familyLength + qualifierLength + valueLength + tagsLength;
}
+
+ @Override
+ public int write(OutputStream out) throws IOException {
+ return write(out, true);
+ }
+
+ @Override
+ public int write(OutputStream out, boolean withTags) throws IOException {
+ int lenToWrite = KeyValueUtil.length(rowLength, familyLength, qualifierLength, valueLength,
+ tagsLength, withTags);
+ StreamUtils.writeInt(out, lenToWrite);
+ StreamUtils.writeInt(out, keyOnlyBuffer.length);
+ StreamUtils.writeInt(out, valueLength);
+ // Write key
+ out.write(keyOnlyBuffer);
+ // Write value
+ assert this.currentBuffer.hasArray();
+ out.write(this.currentBuffer.array(), this.currentBuffer.arrayOffset() + this.valueOffset,
+ this.valueLength);
+ if (withTags) {
+ // 2 bytes tags length followed by tags bytes
+ // tags length is serialized with 2 bytes only(short way) even if the type is int.
+ // As this is non -ve numbers, we save the sign bit. See HBASE-11437
+ out.write((byte) (0xff & (this.tagsLength >> 8)));
+ out.write((byte) (0xff & this.tagsLength));
+ if (this.tagCompressionContext != null) {
+ out.write(cloneTagsBuffer);
+ } else {
+ out.write(this.currentBuffer.array(), this.currentBuffer.arrayOffset() + this.tagsOffset,
+ this.tagsLength);
+ }
+ }
+ return lenToWrite + Bytes.SIZEOF_INT;
+ }
}
protected abstract static class