You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2013/11/26 05:22:35 UTC
svn commit: r1545536 [3/4] - in /hbase/trunk:
hbase-client/src/main/java/org/apache/hadoop/hbase/
hbase-client/src/main/java/org/apache/hadoop/hbase/security/
hbase-client/src/test/java/org/apache/hadoop/hbase/security/
hbase-common/src/main/java/org/a...
Modified: hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java (original)
+++ hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java Tue Nov 26 04:22:33 2013
@@ -102,6 +102,16 @@ public final class WALProtos {
* <code>optional bool has_compression = 1;</code>
*/
boolean getHasCompression();
+
+ // optional bytes encryption_key = 2;
+ /**
+ * <code>optional bytes encryption_key = 2;</code>
+ */
+ boolean hasEncryptionKey();
+ /**
+ * <code>optional bytes encryption_key = 2;</code>
+ */
+ com.google.protobuf.ByteString getEncryptionKey();
}
/**
* Protobuf type {@code WALHeader}
@@ -159,6 +169,11 @@ public final class WALProtos {
hasCompression_ = input.readBool();
break;
}
+ case 18: {
+ bitField0_ |= 0x00000002;
+ encryptionKey_ = input.readBytes();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -215,8 +230,25 @@ public final class WALProtos {
return hasCompression_;
}
+ // optional bytes encryption_key = 2;
+ public static final int ENCRYPTION_KEY_FIELD_NUMBER = 2;
+ private com.google.protobuf.ByteString encryptionKey_;
+ /**
+ * <code>optional bytes encryption_key = 2;</code>
+ */
+ public boolean hasEncryptionKey() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional bytes encryption_key = 2;</code>
+ */
+ public com.google.protobuf.ByteString getEncryptionKey() {
+ return encryptionKey_;
+ }
+
private void initFields() {
hasCompression_ = false;
+ encryptionKey_ = com.google.protobuf.ByteString.EMPTY;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -233,6 +265,9 @@ public final class WALProtos {
if (((bitField0_ & 0x00000001) == 0x00000001)) {
output.writeBool(1, hasCompression_);
}
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeBytes(2, encryptionKey_);
+ }
getUnknownFields().writeTo(output);
}
@@ -246,6 +281,10 @@ public final class WALProtos {
size += com.google.protobuf.CodedOutputStream
.computeBoolSize(1, hasCompression_);
}
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(2, encryptionKey_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -274,6 +313,11 @@ public final class WALProtos {
result = result && (getHasCompression()
== other.getHasCompression());
}
+ result = result && (hasEncryptionKey() == other.hasEncryptionKey());
+ if (hasEncryptionKey()) {
+ result = result && getEncryptionKey()
+ .equals(other.getEncryptionKey());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -291,6 +335,10 @@ public final class WALProtos {
hash = (37 * hash) + HAS_COMPRESSION_FIELD_NUMBER;
hash = (53 * hash) + hashBoolean(getHasCompression());
}
+ if (hasEncryptionKey()) {
+ hash = (37 * hash) + ENCRYPTION_KEY_FIELD_NUMBER;
+ hash = (53 * hash) + getEncryptionKey().hashCode();
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -402,6 +450,8 @@ public final class WALProtos {
super.clear();
hasCompression_ = false;
bitField0_ = (bitField0_ & ~0x00000001);
+ encryptionKey_ = com.google.protobuf.ByteString.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000002);
return this;
}
@@ -434,6 +484,10 @@ public final class WALProtos {
to_bitField0_ |= 0x00000001;
}
result.hasCompression_ = hasCompression_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.encryptionKey_ = encryptionKey_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -453,6 +507,9 @@ public final class WALProtos {
if (other.hasHasCompression()) {
setHasCompression(other.getHasCompression());
}
+ if (other.hasEncryptionKey()) {
+ setEncryptionKey(other.getEncryptionKey());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -513,6 +570,42 @@ public final class WALProtos {
return this;
}
+ // optional bytes encryption_key = 2;
+ private com.google.protobuf.ByteString encryptionKey_ = com.google.protobuf.ByteString.EMPTY;
+ /**
+ * <code>optional bytes encryption_key = 2;</code>
+ */
+ public boolean hasEncryptionKey() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional bytes encryption_key = 2;</code>
+ */
+ public com.google.protobuf.ByteString getEncryptionKey() {
+ return encryptionKey_;
+ }
+ /**
+ * <code>optional bytes encryption_key = 2;</code>
+ */
+ public Builder setEncryptionKey(com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000002;
+ encryptionKey_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bytes encryption_key = 2;</code>
+ */
+ public Builder clearEncryptionKey() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ encryptionKey_ = getDefaultInstance().getEncryptionKey();
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:WALHeader)
}
@@ -4991,24 +5084,25 @@ public final class WALProtos {
descriptor;
static {
java.lang.String[] descriptorData = {
- "\n\tWAL.proto\032\013HBase.proto\"$\n\tWALHeader\022\027\n" +
- "\017has_compression\030\001 \001(\010\"\202\002\n\006WALKey\022\033\n\023enc" +
- "oded_region_name\030\001 \002(\014\022\022\n\ntable_name\030\002 \002" +
- "(\014\022\033\n\023log_sequence_number\030\003 \002(\004\022\022\n\nwrite" +
- "_time\030\004 \002(\004\022\035\n\ncluster_id\030\005 \001(\0132\005.UUIDB\002" +
- "\030\001\022\034\n\006scopes\030\006 \003(\0132\014.FamilyScope\022\032\n\022foll" +
- "owing_kv_count\030\007 \001(\r\022\032\n\013cluster_ids\030\010 \003(" +
- "\0132\005.UUID\022\022\n\nnonceGroup\030\t \001(\004\022\r\n\005nonce\030\n " +
- "\001(\004\"=\n\013FamilyScope\022\016\n\006family\030\001 \002(\014\022\036\n\nsc" +
- "ope_type\030\002 \002(\0162\n.ScopeType\"\251\001\n\024Compactio",
- "nDescriptor\022\022\n\ntable_name\030\001 \002(\014\022\033\n\023encod" +
- "ed_region_name\030\002 \002(\014\022\023\n\013family_name\030\003 \002(" +
- "\014\022\030\n\020compaction_input\030\004 \003(\t\022\031\n\021compactio" +
- "n_output\030\005 \003(\t\022\026\n\016store_home_dir\030\006 \002(\t\"\014" +
- "\n\nWALTrailer*F\n\tScopeType\022\033\n\027REPLICATION" +
- "_SCOPE_LOCAL\020\000\022\034\n\030REPLICATION_SCOPE_GLOB" +
- "AL\020\001B?\n*org.apache.hadoop.hbase.protobuf" +
- ".generatedB\tWALProtosH\001\210\001\000\240\001\001"
+ "\n\tWAL.proto\032\013HBase.proto\"<\n\tWALHeader\022\027\n" +
+ "\017has_compression\030\001 \001(\010\022\026\n\016encryption_key" +
+ "\030\002 \001(\014\"\202\002\n\006WALKey\022\033\n\023encoded_region_name" +
+ "\030\001 \002(\014\022\022\n\ntable_name\030\002 \002(\014\022\033\n\023log_sequen" +
+ "ce_number\030\003 \002(\004\022\022\n\nwrite_time\030\004 \002(\004\022\035\n\nc" +
+ "luster_id\030\005 \001(\0132\005.UUIDB\002\030\001\022\034\n\006scopes\030\006 \003" +
+ "(\0132\014.FamilyScope\022\032\n\022following_kv_count\030\007" +
+ " \001(\r\022\032\n\013cluster_ids\030\010 \003(\0132\005.UUID\022\022\n\nnonc" +
+ "eGroup\030\t \001(\004\022\r\n\005nonce\030\n \001(\004\"=\n\013FamilySco" +
+ "pe\022\016\n\006family\030\001 \002(\014\022\036\n\nscope_type\030\002 \002(\0162\n",
+ ".ScopeType\"\251\001\n\024CompactionDescriptor\022\022\n\nt" +
+ "able_name\030\001 \002(\014\022\033\n\023encoded_region_name\030\002" +
+ " \002(\014\022\023\n\013family_name\030\003 \002(\014\022\030\n\020compaction_" +
+ "input\030\004 \003(\t\022\031\n\021compaction_output\030\005 \003(\t\022\026" +
+ "\n\016store_home_dir\030\006 \002(\t\"\014\n\nWALTrailer*F\n\t" +
+ "ScopeType\022\033\n\027REPLICATION_SCOPE_LOCAL\020\000\022\034" +
+ "\n\030REPLICATION_SCOPE_GLOBAL\020\001B?\n*org.apac" +
+ "he.hadoop.hbase.protobuf.generatedB\tWALP" +
+ "rotosH\001\210\001\000\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -5020,7 +5114,7 @@ public final class WALProtos {
internal_static_WALHeader_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_WALHeader_descriptor,
- new java.lang.String[] { "HasCompression", });
+ new java.lang.String[] { "HasCompression", "EncryptionKey", });
internal_static_WALKey_descriptor =
getDescriptor().getMessageTypes().get(1);
internal_static_WALKey_fieldAccessorTable = new
Added: hbase/trunk/hbase-protocol/src/main/protobuf/Encryption.proto
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-protocol/src/main/protobuf/Encryption.proto?rev=1545536&view=auto
==============================================================================
--- hbase/trunk/hbase-protocol/src/main/protobuf/Encryption.proto (added)
+++ hbase/trunk/hbase-protocol/src/main/protobuf/Encryption.proto Tue Nov 26 04:22:33 2013
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers used for encryption
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "EncryptionProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+message WrappedKey {
+ required string algorithm = 1;
+ required uint32 length = 2;
+ required bytes data = 3;
+ optional bytes iv = 4;
+ optional fixed32 crc = 5;
+}
Modified: hbase/trunk/hbase-protocol/src/main/protobuf/HFile.proto
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-protocol/src/main/protobuf/HFile.proto?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-protocol/src/main/protobuf/HFile.proto (original)
+++ hbase/trunk/hbase-protocol/src/main/protobuf/HFile.proto Tue Nov 26 04:22:33 2013
@@ -42,4 +42,5 @@ message FileTrailerProto {
optional uint64 last_data_block_offset = 10;
optional string comparator_class_name = 11;
optional uint32 compression_codec = 12;
+ optional bytes encryption_key = 13;
}
Modified: hbase/trunk/hbase-protocol/src/main/protobuf/WAL.proto
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-protocol/src/main/protobuf/WAL.proto?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-protocol/src/main/protobuf/WAL.proto (original)
+++ hbase/trunk/hbase-protocol/src/main/protobuf/WAL.proto Tue Nov 26 04:22:33 2013
@@ -25,6 +25,7 @@ import "HBase.proto";
message WALHeader {
optional bool has_compression = 1;
+ optional bytes encryption_key = 2;
}
// Protocol buffer version of HLogKey; see HLogKey comment, not really a key but WALEdit header for some KVs
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java Tue Nov 26 04:22:33 2013
@@ -24,6 +24,7 @@ import java.nio.ByteBuffer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
@@ -65,11 +66,13 @@ public class HalfStoreFileReader extends
* @param p path to hfile
* @param cacheConf
* @param r original reference file (contains top or bottom)
+ * @param conf Configuration
* @throws IOException
*/
public HalfStoreFileReader(final FileSystem fs, final Path p,
- final CacheConfig cacheConf, final Reference r) throws IOException {
- super(fs, p, cacheConf);
+ final CacheConfig cacheConf, final Reference r, final Configuration conf)
+ throws IOException {
+ super(fs, p, cacheConf, conf);
// This is not actual midkey for this half-file; its just border
// around which we split top and bottom. Have to look in files to find
// actual last and first keys for bottom and top halves. Half-files don't
@@ -88,11 +91,13 @@ public class HalfStoreFileReader extends
* @param size Full size of the hfile file
* @param cacheConf
* @param r original reference file (contains top or bottom)
+ * @param conf Configuration
* @throws IOException
*/
public HalfStoreFileReader(final FileSystem fs, final Path p, final FSDataInputStreamWrapper in,
- long size, final CacheConfig cacheConf, final Reference r) throws IOException {
- super(fs, p, in, size, cacheConf);
+ long size, final CacheConfig cacheConf, final Reference r, final Configuration conf)
+ throws IOException {
+ super(fs, p, in, size, cacheConf, conf);
// This is not actual midkey for this half-file; its just border
// around which we split top and bottom. Have to look in files to find
// actual last and first keys for bottom and top halves. Half-files don't
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java Tue Nov 26 04:22:33 2013
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue;
@@ -35,7 +37,8 @@ import org.apache.hadoop.hbase.io.hfile.
* Common functionality needed by all versions of {@link HFile} readers.
*/
@InterfaceAudience.Private
-public abstract class AbstractHFileReader implements HFile.Reader {
+public abstract class AbstractHFileReader
+ implements HFile.Reader, Configurable {
/** Stream to read from. Does checksum verifications in file system */
protected FSDataInputStream istream;
@@ -71,7 +74,7 @@ public abstract class AbstractHFileReade
protected int avgValueLen = -1;
/** Key comparator */
- protected KVComparator comparator;
+ protected KVComparator comparator = new KVComparator();
/** Size of this file. */
protected final long fileSize;
@@ -90,8 +93,11 @@ public abstract class AbstractHFileReade
/** The filesystem used for accesing data */
protected HFileSystem hfs;
+ protected Configuration conf;
+
protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
- final long fileSize, final CacheConfig cacheConf, final HFileSystem hfs) {
+ final long fileSize, final CacheConfig cacheConf, final HFileSystem hfs,
+ final Configuration conf) {
this.trailer = trailer;
this.compressAlgo = trailer.getCompressionCodec();
this.cacheConf = cacheConf;
@@ -99,6 +105,7 @@ public abstract class AbstractHFileReade
this.path = path;
this.name = path.getName();
this.hfs = hfs;
+ this.conf = conf;
}
@SuppressWarnings("serial")
@@ -330,4 +337,14 @@ public abstract class AbstractHFileReade
}
public abstract int getMajorVersion();
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java Tue Nov 26 04:22:33 2013
@@ -26,8 +26,6 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.hbase.KeyValue;
@@ -36,6 +34,8 @@ import org.apache.hadoop.hbase.io.compre
import org.apache.hadoop.hbase.protobuf.generated.HFileProtos;
import org.apache.hadoop.hbase.util.Bytes;
+import com.google.protobuf.ZeroCopyLiteralByteString;
+
/**
* The {@link HFile} has a fixed trailer which contains offsets to other
* variable parts of the file. Also includes basic metadata on this file. The
@@ -110,6 +110,9 @@ public class FixedFileTrailer {
/** Raw key comparator class name in version 3 */
private String comparatorClassName = KeyValue.COMPARATOR.getLegacyKeyComparatorName();
+ /** The encryption key */
+ private byte[] encryptionKey;
+
/** The {@link HFile} format major version. */
private final int majorVersion;
@@ -127,10 +130,10 @@ public class FixedFileTrailer {
// We support only 2 major versions now. ie. V2, V3
versionToSize[2] = 212;
for (int version = 3; version <= HFile.MAX_FORMAT_VERSION; version++) {
- // Max FFT size for V3 and above is taken as 1KB for future enhancements
+ // Max FFT size for V3 and above is taken as 4KB for future enhancements
// if any.
- // Unless the trailer size exceeds 1024 this can continue
- versionToSize[version] = 1024;
+ // Unless the trailer size exceeds 4K this can continue
+ versionToSize[version] = 1024 * 4;
}
return versionToSize;
}
@@ -187,7 +190,7 @@ public class FixedFileTrailer {
*/
void serializeAsPB(DataOutputStream output) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
- HFileProtos.FileTrailerProto.newBuilder()
+ HFileProtos.FileTrailerProto.Builder builder = HFileProtos.FileTrailerProto.newBuilder()
.setFileInfoOffset(fileInfoOffset)
.setLoadOnOpenDataOffset(loadOnOpenDataOffset)
.setUncompressedDataIndexSize(uncompressedDataIndexSize)
@@ -201,8 +204,11 @@ public class FixedFileTrailer {
// TODO this is a classname encoded into an HFile's trailer. We are going to need to have
// some compat code here.
.setComparatorClassName(comparatorClassName)
- .setCompressionCodec(compressionCodec.ordinal())
- .build().writeDelimitedTo(baos);
+ .setCompressionCodec(compressionCodec.ordinal());
+ if (encryptionKey != null) {
+ builder.setEncryptionKey(ZeroCopyLiteralByteString.wrap(encryptionKey));
+ }
+ builder.build().writeDelimitedTo(baos);
output.write(baos.toByteArray());
// Pad to make up the difference between variable PB encoding length and the
// length when encoded as writable under earlier V2 formats. Failure to pad
@@ -251,52 +257,55 @@ public class FixedFileTrailer {
void deserializeFromPB(DataInputStream inputStream) throws IOException {
// read PB and skip padding
int start = inputStream.available();
- HFileProtos.FileTrailerProto.Builder builder = HFileProtos.FileTrailerProto.newBuilder();
- builder.mergeDelimitedFrom(inputStream);
+ HFileProtos.FileTrailerProto trailerProto =
+ HFileProtos.FileTrailerProto.PARSER.parseDelimitedFrom(inputStream);
int size = start - inputStream.available();
inputStream.skip(getTrailerSize() - NOT_PB_SIZE - size);
// process the PB
- if (builder.hasFileInfoOffset()) {
- fileInfoOffset = builder.getFileInfoOffset();
+ if (trailerProto.hasFileInfoOffset()) {
+ fileInfoOffset = trailerProto.getFileInfoOffset();
}
- if (builder.hasLoadOnOpenDataOffset()) {
- loadOnOpenDataOffset = builder.getLoadOnOpenDataOffset();
+ if (trailerProto.hasLoadOnOpenDataOffset()) {
+ loadOnOpenDataOffset = trailerProto.getLoadOnOpenDataOffset();
}
- if (builder.hasUncompressedDataIndexSize()) {
- uncompressedDataIndexSize = builder.getUncompressedDataIndexSize();
+ if (trailerProto.hasUncompressedDataIndexSize()) {
+ uncompressedDataIndexSize = trailerProto.getUncompressedDataIndexSize();
}
- if (builder.hasTotalUncompressedBytes()) {
- totalUncompressedBytes = builder.getTotalUncompressedBytes();
+ if (trailerProto.hasTotalUncompressedBytes()) {
+ totalUncompressedBytes = trailerProto.getTotalUncompressedBytes();
}
- if (builder.hasDataIndexCount()) {
- dataIndexCount = builder.getDataIndexCount();
+ if (trailerProto.hasDataIndexCount()) {
+ dataIndexCount = trailerProto.getDataIndexCount();
}
- if (builder.hasMetaIndexCount()) {
- metaIndexCount = builder.getMetaIndexCount();
+ if (trailerProto.hasMetaIndexCount()) {
+ metaIndexCount = trailerProto.getMetaIndexCount();
}
- if (builder.hasEntryCount()) {
- entryCount = builder.getEntryCount();
+ if (trailerProto.hasEntryCount()) {
+ entryCount = trailerProto.getEntryCount();
}
- if (builder.hasNumDataIndexLevels()) {
- numDataIndexLevels = builder.getNumDataIndexLevels();
+ if (trailerProto.hasNumDataIndexLevels()) {
+ numDataIndexLevels = trailerProto.getNumDataIndexLevels();
}
- if (builder.hasFirstDataBlockOffset()) {
- firstDataBlockOffset = builder.getFirstDataBlockOffset();
+ if (trailerProto.hasFirstDataBlockOffset()) {
+ firstDataBlockOffset = trailerProto.getFirstDataBlockOffset();
}
- if (builder.hasLastDataBlockOffset()) {
- lastDataBlockOffset = builder.getLastDataBlockOffset();
+ if (trailerProto.hasLastDataBlockOffset()) {
+ lastDataBlockOffset = trailerProto.getLastDataBlockOffset();
}
- if (builder.hasComparatorClassName()) {
+ if (trailerProto.hasComparatorClassName()) {
// TODO this is a classname encoded into an HFile's trailer. We are going to need to have
// some compat code here.
- setComparatorClass(getComparatorClass(builder.getComparatorClassName()));
+ setComparatorClass(getComparatorClass(trailerProto.getComparatorClassName()));
}
- if (builder.hasCompressionCodec()) {
- compressionCodec = Compression.Algorithm.values()[builder.getCompressionCodec()];
+ if (trailerProto.hasCompressionCodec()) {
+ compressionCodec = Compression.Algorithm.values()[trailerProto.getCompressionCodec()];
} else {
compressionCodec = Compression.Algorithm.NONE;
}
+ if (trailerProto.hasEncryptionKey()) {
+ encryptionKey = trailerProto.getEncryptionKey().toByteArray();
+ }
}
/**
@@ -344,6 +353,9 @@ public class FixedFileTrailer {
append(sb, "firstDataBlockOffset=" + firstDataBlockOffset);
append(sb, "lastDataBlockOffset=" + lastDataBlockOffset);
append(sb, "comparatorClassName=" + comparatorClassName);
+ if (majorVersion >= 3) {
+ append(sb, "encryptionKey=" + (encryptionKey != null ? "PRESENT" : "NONE"));
+ }
append(sb, "majorVersion=" + majorVersion);
append(sb, "minorVersion=" + minorVersion);
@@ -597,6 +609,15 @@ public class FixedFileTrailer {
this.uncompressedDataIndexSize = uncompressedDataIndexSize;
}
+ public byte[] getEncryptionKey() {
+ expectAtLeastMajorVersion(3);
+ return encryptionKey;
+ }
+
+ public void setEncryptionKey(byte[] keyBytes) {
+ this.encryptionKey = keyBytes;
+ }
+
/**
* Extracts the major version for a 4-byte serialized version data.
* The major version is the 3 least significant bytes
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Tue Nov 26 04:22:33 2013
@@ -320,6 +320,11 @@ public class HFile {
* HFile V2.
*/
void addDeleteFamilyBloomFilter(BloomFilterWriter bfw) throws IOException;
+
+ /**
+ * Return the file context for the HFile this writer belongs to
+ */
+ HFileContext getFileContext();
}
/**
@@ -499,6 +504,11 @@ public class HFile {
DataBlockEncoding getDataBlockEncoding();
boolean hasMVCCInfo();
+
+ /**
+ * Return the file context of the HFile this reader belongs to
+ */
+ HFileContext getFileContext();
}
/**
@@ -514,8 +524,7 @@ public class HFile {
* @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
*/
private static Reader pickReaderVersion(Path path, FSDataInputStreamWrapper fsdis,
- long size, CacheConfig cacheConf,
- HFileSystem hfs) throws IOException {
+ long size, CacheConfig cacheConf, HFileSystem hfs, Configuration conf) throws IOException {
FixedFileTrailer trailer = null;
try {
boolean isHBaseChecksum = fsdis.shouldUseHBaseChecksum();
@@ -523,11 +532,9 @@ public class HFile {
trailer = FixedFileTrailer.readFromStream(fsdis.getStream(isHBaseChecksum), size);
switch (trailer.getMajorVersion()) {
case 2:
- return new HFileReaderV2(
- path, trailer, fsdis, size, cacheConf, hfs);
+ return new HFileReaderV2(path, trailer, fsdis, size, cacheConf, hfs, conf);
case 3 :
- return new HFileReaderV3(
- path, trailer, fsdis, size, cacheConf, hfs);
+ return new HFileReaderV3(path, trailer, fsdis, size, cacheConf, hfs, conf);
default:
throw new IllegalArgumentException("Invalid HFile version " + trailer.getMajorVersion());
}
@@ -547,11 +554,13 @@ public class HFile {
* @param fsdis a stream of path's file
* @param size max size of the trailer.
* @param cacheConf Cache configuration for hfile's contents
+ * @param conf Configuration
* @return A version specific Hfile Reader
* @throws IOException If file is invalid, will throw CorruptHFileException flavored IOException
*/
public static Reader createReader(FileSystem fs, Path path,
- FSDataInputStreamWrapper fsdis, long size, CacheConfig cacheConf) throws IOException {
+ FSDataInputStreamWrapper fsdis, long size, CacheConfig cacheConf, Configuration conf)
+ throws IOException {
HFileSystem hfs = null;
// If the fs is not an instance of HFileSystem, then create an
@@ -563,7 +572,7 @@ public class HFile {
} else {
hfs = (HFileSystem)fs;
}
- return pickReaderVersion(path, fsdis, size, cacheConf, hfs);
+ return pickReaderVersion(path, fsdis, size, cacheConf, hfs, conf);
}
/**
@@ -575,21 +584,21 @@ public class HFile {
* @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile is corrupt/invalid.
*/
public static Reader createReader(
- FileSystem fs, Path path, CacheConfig cacheConf) throws IOException {
+ FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf) throws IOException {
Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf");
FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path);
return pickReaderVersion(path, stream, fs.getFileStatus(path).getLen(),
- cacheConf, stream.getHfs());
+ cacheConf, stream.getHfs(), conf);
}
/**
* This factory method is used only by unit tests
*/
static Reader createReaderFromStream(Path path,
- FSDataInputStream fsdis, long size, CacheConfig cacheConf)
+ FSDataInputStream fsdis, long size, CacheConfig cacheConf, Configuration conf)
throws IOException {
FSDataInputStreamWrapper wrapper = new FSDataInputStreamWrapper(fsdis);
- return pickReaderVersion(path, wrapper, size, cacheConf, null);
+ return pickReaderVersion(path, wrapper, size, cacheConf, null, conf);
}
/**
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java Tue Nov 26 04:22:33 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.fs.HFileS
import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;
@@ -1491,11 +1492,17 @@ public class HFileBlock implements Cache
- hdrSize, true, offset + hdrSize, pread);
onDiskSizeWithHeader = b.onDiskSizeWithoutHeader + hdrSize;
}
+
Algorithm compressAlgo = fileContext.getCompression();
boolean isCompressed =
compressAlgo != null
&& compressAlgo != Compression.Algorithm.NONE;
- if (!isCompressed) {
+
+ Encryption.Context cryptoContext = fileContext.getEncryptionContext();
+ boolean isEncrypted = cryptoContext != null
+ && cryptoContext != Encryption.Context.NONE;
+
+ if (!isCompressed && !isEncrypted) {
b.assumeUncompressed();
}
@@ -1504,7 +1511,7 @@ public class HFileBlock implements Cache
return null; // checksum mismatch
}
- if (isCompressed) {
+ if (isCompressed || isEncrypted) {
// This will allocate a new buffer but keep header bytes.
b.allocateBuffer(nextBlockOnDiskSize > 0);
if (b.blockType == BlockType.ENCODED_DATA) {
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java Tue Nov 26 04:22:33 2013
@@ -214,7 +214,7 @@ public class HFilePrettyPrinter {
System.err.println("ERROR, file doesnt exist: " + file);
}
- HFile.Reader reader = HFile.createReader(fs, file, new CacheConfig(conf));
+ HFile.Reader reader = HFile.createReader(fs, file, new CacheConfig(conf), conf);
Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java Tue Nov 26 04:22:33 2013
@@ -26,6 +26,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
@@ -98,6 +99,7 @@ public class HFileReaderV2 extends Abstr
/** Minor versions starting with this number have faked index key */
static final int MINOR_VERSION_WITH_FAKED_KEY = 3;
+
protected HFileContext hfileContext;
/**
@@ -110,15 +112,16 @@ public class HFileReaderV2 extends Abstr
* @param size Length of the stream.
* @param cacheConf Cache configuration.
* @param hfs
+ * @param conf
*/
public HFileReaderV2(Path path, FixedFileTrailer trailer,
final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf,
- final HFileSystem hfs)
- throws IOException {
- super(path, trailer, size, cacheConf, hfs);
+ final HFileSystem hfs, final Configuration conf) throws IOException {
+ super(path, trailer, size, cacheConf, hfs, conf);
+ this.conf = conf;
trailer.expectMajorVersion(getMajorVersion());
validateMinorVersion(path, trailer.getMinorVersion());
- this.hfileContext = createHFileContext(trailer);
+ this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer);
HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis, fileSize, hfs, path,
hfileContext);
this.fsBlockReader = fsBlockReaderV2; // upcast
@@ -174,13 +177,13 @@ public class HFileReaderV2 extends Abstr
}
}
- protected HFileContext createHFileContext(FixedFileTrailer trailer) {
- HFileContext hFileContext = new HFileContextBuilder()
- .withIncludesMvcc(this.includesMemstoreTS)
- .withCompression(this.compressAlgo)
- .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
- .build();
- return hFileContext;
+ protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
+ HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
+ return new HFileContextBuilder()
+ .withIncludesMvcc(this.includesMemstoreTS)
+ .withCompression(this.compressAlgo)
+ .withHBaseCheckSum(trailer.getMinorVersion() >= MINOR_VERSION_WITH_CHECKSUM)
+ .build();
}
/**
@@ -1185,4 +1188,9 @@ public class HFileReaderV2 extends Abstr
public int getMajorVersion() {
return 2;
}
+
+ @Override
+ public HFileContext getFileContext() {
+ return hfileContext;
+ }
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java Tue Nov 26 04:22:33 2013
@@ -18,25 +18,36 @@
package org.apache.hadoop.hbase.io.hfile;
import java.io.IOException;
+import java.security.Key;
+import java.security.KeyException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
+import org.apache.hadoop.hbase.io.crypto.Cipher;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
+import org.apache.hadoop.hbase.security.EncryptionUtil;
+import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.WritableUtils;
/**
* {@link HFile} reader for version 3.
- * This Reader is aware of Tags.
*/
@InterfaceAudience.Private
public class HFileReaderV3 extends HFileReaderV2 {
+ private static final Log LOG = LogFactory.getLog(HFileReaderV3.class);
+
public static final int MAX_MINOR_VERSION = 0;
+
/**
* Opens a HFile. You must load the index before you can use it by calling
* {@link #loadFileInfo()}.
@@ -50,11 +61,15 @@ public class HFileReaderV3 extends HFile
* Length of the stream.
* @param cacheConf
* Cache configuration.
+ * @param hfs
+ * The file system.
+ * @param conf
+ * Configuration
*/
public HFileReaderV3(Path path, FixedFileTrailer trailer, final FSDataInputStreamWrapper fsdis,
- final long size, final CacheConfig cacheConf,
- final HFileSystem hfs) throws IOException {
- super(path, trailer, fsdis, size, cacheConf, hfs);
+ final long size, final CacheConfig cacheConf, final HFileSystem hfs,
+ final Configuration conf) throws IOException {
+ super(path, trailer, fsdis, size, cacheConf, hfs, conf);
byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
// max tag length is not present in the HFile means tags were not at all written to file.
if (tmp != null) {
@@ -67,13 +82,62 @@ public class HFileReaderV3 extends HFile
}
@Override
- protected HFileContext createHFileContext(FixedFileTrailer trailer) {
- HFileContext hfileContext = new HFileContextBuilder()
- .withIncludesMvcc(this.includesMemstoreTS)
- .withHBaseCheckSum(true)
- .withCompression(this.compressAlgo)
- .build();
- return hfileContext;
+ protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
+ HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
+ trailer.expectMajorVersion(3);
+ HFileContextBuilder builder = new HFileContextBuilder()
+ .withIncludesMvcc(this.includesMemstoreTS)
+ .withHBaseCheckSum(true)
+ .withCompression(this.compressAlgo);
+
+ // Check for any key material available
+ byte[] keyBytes = trailer.getEncryptionKey();
+ if (keyBytes != null) {
+ Encryption.Context cryptoContext = Encryption.newContext(conf);
+ Key key;
+ String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
+ User.getCurrent().getShortName());
+ try {
+ // First try the master key
+ key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
+ } catch (KeyException e) {
+ // If the current master key fails to unwrap, try the alternate, if
+ // one is configured
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
+ }
+ String alternateKeyName =
+ conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
+ if (alternateKeyName != null) {
+ try {
+ key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
+ } catch (KeyException ex) {
+ throw new IOException(ex);
+ }
+ } else {
+ throw new IOException(e);
+ }
+ }
+ // Use the algorithm the key wants
+ Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
+ if (cipher == null) {
+ throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available");
+ }
+ cryptoContext.setCipher(cipher);
+ cryptoContext.setKey(key);
+ builder.withEncryptionContext(cryptoContext);
+ }
+
+ HFileContext context = builder.build();
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Reader" + (path != null ? " for " + path : "" ) +
+ " initialized with cacheConf: " + cacheConf +
+ " comparator: " + comparator.getClass().getSimpleName() +
+ " fileContext: " + context);
+ }
+
+ return context;
}
/**
@@ -140,19 +204,24 @@ public class HFileReaderV3 extends HFile
blockBuffer.mark();
currKeyLen = blockBuffer.getInt();
currValueLen = blockBuffer.getInt();
+ if (currKeyLen < 0 || currValueLen < 0 || currKeyLen > blockBuffer.limit()
+ || currValueLen > blockBuffer.limit()) {
+ throw new IllegalStateException("Invalid currKeyLen " + currKeyLen + " or currValueLen "
+ + currValueLen + ". Block offset: "
+ + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
+ + blockBuffer.position() + " (without header).");
+ }
ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
if (reader.hfileContext.isIncludesTags()) {
currTagsLen = blockBuffer.getShort();
+ if (currTagsLen < 0 || currTagsLen > blockBuffer.limit()) {
+ throw new IllegalStateException("Invalid currTagsLen " + currTagsLen + ". Block offset: "
+ + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
+ + blockBuffer.position() + " (without header).");
+ }
ByteBufferUtils.skip(blockBuffer, currTagsLen);
}
readMvccVersion();
- if (currKeyLen < 0 || currValueLen < 0 || currTagsLen < 0 || currKeyLen > blockBuffer.limit()
- || currValueLen > blockBuffer.limit() || currTagsLen > blockBuffer.limit()) {
- throw new IllegalStateException("Invalid currKeyLen " + currKeyLen + " or currValueLen "
- + currValueLen + " or currTagLen " + currTagsLen + ". Block offset: "
- + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
- + blockBuffer.position() + " (without header).");
- }
blockBuffer.reset();
}
@@ -184,9 +253,21 @@ public class HFileReaderV3 extends HFile
blockBuffer.mark();
klen = blockBuffer.getInt();
vlen = blockBuffer.getInt();
+ if (klen < 0 || vlen < 0 || klen > blockBuffer.limit()
+ || vlen > blockBuffer.limit()) {
+ throw new IllegalStateException("Invalid klen " + klen + " or vlen "
+ + vlen + ". Block offset: "
+ + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
+ + blockBuffer.position() + " (without header).");
+ }
ByteBufferUtils.skip(blockBuffer, klen + vlen);
if (reader.hfileContext.isIncludesTags()) {
tlen = blockBuffer.getShort();
+ if (tlen < 0 || tlen > blockBuffer.limit()) {
+ throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: "
+ + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
+ + blockBuffer.position() + " (without header).");
+ }
ByteBufferUtils.skip(blockBuffer, tlen);
}
if (this.reader.shouldIncludeMemstoreTS()) {
@@ -262,8 +343,8 @@ public class HFileReaderV3 extends HFile
*/
protected static class EncodedScannerV3 extends EncodedScannerV2 {
public EncodedScannerV3(HFileReaderV3 reader, boolean cacheBlocks, boolean pread,
- boolean isCompaction, HFileContext meta) {
- super(reader, cacheBlocks, pread, isCompaction, meta);
+ boolean isCompaction, HFileContext context) {
+ super(reader, cacheBlocks, pread, isCompaction, context);
}
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java Tue Nov 26 04:22:33 2013
@@ -459,4 +459,9 @@ public class HFileWriterV2 extends Abstr
protected int getMinorVersion() {
return HFileReaderV2.MAX_MINOR_VERSION;
}
+
+ @Override
+ public HFileContext getFileContext() {
+ return hFileContext;
+ }
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV3.java Tue Nov 26 04:22:33 2013
@@ -17,9 +17,12 @@
*/
package org.apache.hadoop.hbase.io.hfile;
+import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -28,18 +31,23 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
+import org.apache.hadoop.hbase.security.EncryptionUtil;
+import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.WritableUtils;
/**
- * This is an extension of HFileWriterV2 that is tags aware.
+ * {@link HFile} writer for version 3.
*/
@InterfaceAudience.Private
public class HFileWriterV3 extends HFileWriterV2 {
+ private static final Log LOG = LogFactory.getLog(HFileWriterV3.class);
+
private int maxTagsLength = 0;
static class WriterFactoryV3 extends HFile.WriterFactory {
@@ -60,6 +68,12 @@ public class HFileWriterV3 extends HFile
FSDataOutputStream ostream, final KVComparator comparator,
final HFileContext fileContext) throws IOException {
super(conf, cacheConf, fs, path, ostream, comparator, fileContext);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Writer" + (path != null ? " for " + path : "") +
+ " initialized with cacheConf: " + cacheConf +
+ " comparator: " + comparator.getClass().getSimpleName() +
+ " fileContext: " + fileContext);
+ }
}
/**
@@ -194,4 +208,25 @@ public class HFileWriterV3 extends HFile
protected int getMinorVersion() {
return HFileReaderV3.MAX_MINOR_VERSION;
}
+
+ @Override
+ protected void finishClose(FixedFileTrailer trailer) throws IOException {
+ // Write out encryption metadata before finalizing if we have a valid crypto context
+ Encryption.Context cryptoContext = hFileContext.getEncryptionContext();
+ if (cryptoContext != Encryption.Context.NONE) {
+ // Wrap the context's key and write it as the encryption metadata, the wrapper includes
+ // all information needed for decryption
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ os.write(EncryptionUtil.wrapKey(cryptoContext.getConf(),
+ cryptoContext.getConf().get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
+ User.getCurrent().getShortName()),
+ cryptoContext.getKey()));
+ os.close();
+ trailer.setEncryptionKey(os.toByteArray());
+ }
+
+ // Now we can finish the close
+ super.finishClose(trailer);
+ }
+
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java Tue Nov 26 04:22:33 2013
@@ -474,7 +474,7 @@ public class LoadIncrementalHFiles exten
final Path hfilePath = item.hfilePath;
final FileSystem fs = hfilePath.getFileSystem(getConf());
HFile.Reader hfr = HFile.createReader(fs, hfilePath,
- new CacheConfig(getConf()));
+ new CacheConfig(getConf()), getConf());
final byte[] first, last;
try {
hfr.loadFileInfo();
@@ -651,7 +651,7 @@ public class LoadIncrementalHFiles exten
HalfStoreFileReader halfReader = null;
StoreFile.Writer halfWriter = null;
try {
- halfReader = new HalfStoreFileReader(fs, inFile, cacheConf, reference);
+ halfReader = new HalfStoreFileReader(fs, inFile, cacheConf, reference, conf);
Map<byte[], byte[]> fileInfo = halfReader.loadFileInfo();
int blocksize = familyDescriptor.getBlocksize();
@@ -769,11 +769,11 @@ public class LoadIncrementalHFiles exten
for (Path hfile : hfiles) {
if (hfile.getName().startsWith("_")) continue;
HFile.Reader reader = HFile.createReader(fs, hfile,
- new CacheConfig(getConf()));
+ new CacheConfig(getConf()), getConf());
final byte[] first, last;
try {
- if (hcd.getCompressionType() != reader.getCompressionAlgorithm()) {
- hcd.setCompressionType(reader.getCompressionAlgorithm());
+ if (hcd.getCompressionType() != reader.getFileContext().getCompression()) {
+ hcd.setCompressionType(reader.getFileContext().getCompression());
LOG.info("Setting compression " + hcd.getCompressionType().name() +
" for family " + hcd.toString());
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java Tue Nov 26 04:22:33 2013
@@ -21,6 +21,8 @@ package org.apache.hadoop.hbase.regionse
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.InetSocketAddress;
+import java.security.Key;
+import java.security.KeyException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -56,6 +58,8 @@ import org.apache.hadoop.hbase.RemoteExc
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.compress.Compression;
+import org.apache.hadoop.hbase.io.crypto.Cipher;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
@@ -73,6 +77,8 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
+import org.apache.hadoop.hbase.security.EncryptionUtil;
+import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.ClassSize;
@@ -174,6 +180,8 @@ public class HStore implements Store {
private long blockingFileCount;
private int compactionCheckMultiplier;
+ private Encryption.Context cryptoContext = Encryption.Context.NONE;
+
/**
* Constructor
* @param region
@@ -254,6 +262,63 @@ public class HStore implements Store {
"hbase.hstore.flush.retries.number must be > 0, not "
+ flushRetriesNumber);
}
+
+ // Crypto context for new store files
+ String cipherName = family.getEncryptionType();
+ if (cipherName != null) {
+ Cipher cipher;
+ Key key;
+ byte[] keyBytes = family.getEncryptionKey();
+ if (keyBytes != null) {
+ // Family provides specific key material
+ String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
+ User.getCurrent().getShortName());
+ try {
+ // First try the master key
+ key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
+ } catch (KeyException e) {
+ // If the current master key fails to unwrap, try the alternate, if
+ // one is configured
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
+ }
+ String alternateKeyName =
+ conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
+ if (alternateKeyName != null) {
+ try {
+ key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
+ } catch (KeyException ex) {
+ throw new IOException(ex);
+ }
+ } else {
+ throw new IOException(e);
+ }
+ }
+ // Use the algorithm the key wants
+ cipher = Encryption.getCipher(conf, key.getAlgorithm());
+ if (cipher == null) {
+ throw new RuntimeException("Cipher '" + cipher + "' is not available");
+ }
+ // Fail if misconfigured
+ // We use the encryption type specified in the column schema as a sanity check on
+ // what the wrapped key is telling us
+ if (!cipher.getName().equalsIgnoreCase(cipherName)) {
+ throw new RuntimeException("Encryption for family '" + family.getNameAsString() +
+ "' configured with type '" + cipherName +
+ "' but key specifies algorithm '" + cipher.getName() + "'");
+ }
+ } else {
+ // Family does not provide key material, create a random key
+ cipher = Encryption.getCipher(conf, cipherName);
+ if (cipher == null) {
+ throw new RuntimeException("Cipher '" + cipher + "' is not available");
+ }
+ key = cipher.getRandomKey();
+ }
+ cryptoContext = Encryption.newContext(conf);
+ cryptoContext.setCipher(cipher);
+ cryptoContext.setKey(key);
+ }
}
/**
@@ -534,7 +599,7 @@ public class HStore implements Store {
LOG.info("Validating hfile at " + srcPath + " for inclusion in "
+ "store " + this + " region " + this.getRegionInfo().getRegionNameAsString());
reader = HFile.createReader(srcPath.getFileSystem(conf),
- srcPath, cacheConf);
+ srcPath, cacheConf, conf);
reader.loadFileInfo();
byte[] firstKey = reader.getFirstRowKey();
@@ -805,7 +870,8 @@ public class HStore implements Store {
favoredNodes = region.getRegionServerServices().getFavoredNodesForRegion(
region.getRegionInfo().getEncodedName());
}
- HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag);
+ HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag,
+ cryptoContext);
StoreFile.Writer w = new StoreFile.WriterBuilder(conf, writerCacheConf,
this.getFileSystem())
.withFilePath(fs.createTempName())
@@ -819,7 +885,7 @@ public class HStore implements Store {
}
private HFileContext createFileContext(Compression.Algorithm compression,
- boolean includeMVCCReadpoint, boolean includesTag) {
+ boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) {
if (compression == null) {
compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;
}
@@ -833,6 +899,7 @@ public class HStore implements Store {
.withBlockSize(blocksize)
.withHBaseCheckSum(true)
.withDataBlockEncoding(family.getDataBlockEncoding())
+ .withEncryptionContext(cryptoContext)
.build();
return hFileContext;
}
@@ -1916,7 +1983,7 @@ public class HStore implements Store {
}
public static final long FIXED_OVERHEAD =
- ClassSize.align(ClassSize.OBJECT + (15 * ClassSize.REFERENCE) + (4 * Bytes.SIZEOF_LONG)
+ ClassSize.align(ClassSize.OBJECT + (16 * ClassSize.REFERENCE) + (4 * Bytes.SIZEOF_LONG)
+ (5 * Bytes.SIZEOF_INT) + (2 * Bytes.SIZEOF_BOOLEAN));
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Tue Nov 26 04:22:33 2013
@@ -1012,14 +1012,15 @@ public class StoreFile {
private byte[] lastBloomKey;
private long deleteFamilyCnt = -1;
- public Reader(FileSystem fs, Path path, CacheConfig cacheConf) throws IOException {
- reader = HFile.createReader(fs, path, cacheConf);
+ public Reader(FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf)
+ throws IOException {
+ reader = HFile.createReader(fs, path, cacheConf, conf);
bloomFilterType = BloomType.NONE;
}
public Reader(FileSystem fs, Path path, FSDataInputStreamWrapper in, long size,
- CacheConfig cacheConf) throws IOException {
- reader = HFile.createReader(fs, path, in, size, cacheConf);
+ CacheConfig cacheConf, Configuration conf) throws IOException {
+ reader = HFile.createReader(fs, path, in, size, cacheConf, conf);
bloomFilterType = BloomType.NONE;
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java Tue Nov 26 04:22:33 2013
@@ -67,6 +67,9 @@ public class StoreFileInfo {
Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
+ // Configuration
+ private Configuration conf;
+
// HDFS blocks distribution information
private HDFSBlocksDistribution hdfsBlocksDistribution = null;
@@ -100,6 +103,7 @@ public class StoreFileInfo {
*/
public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
throws IOException {
+ this.conf = conf;
this.fileStatus = fileStatus;
Path p = fileStatus.getPath();
if (HFileLink.isHFileLink(p)) {
@@ -201,9 +205,10 @@ public class StoreFileInfo {
}
if (reader == null) {
if (this.reference != null) {
- reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference);
+ reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference,
+ conf);
} else {
- reader = new StoreFile.Reader(fs, this.getPath(), in, length, cacheConf);
+ reader = new StoreFile.Reader(fs, this.getPath(), in, length, cacheConf, conf);
}
}
if (this.coprocessorHost != null) {
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java Tue Nov 26 04:22:33 2013
@@ -28,10 +28,12 @@ import java.util.Arrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.hbase.codec.Codec;
import org.apache.hadoop.hbase.io.LimitInputStream;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALHeader.Builder;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALKey;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALTrailer;
import org.apache.hadoop.hbase.util.Bytes;
@@ -55,10 +57,10 @@ public class ProtobufLogReader extends R
private static final Log LOG = LogFactory.getLog(ProtobufLogReader.class);
static final byte[] PB_WAL_MAGIC = Bytes.toBytes("PWAL");
static final byte[] PB_WAL_COMPLETE_MAGIC = Bytes.toBytes("LAWP");
- private FSDataInputStream inputStream;
- private Codec.Decoder cellDecoder;
- private WALCellCodec.ByteStringUncompressor byteStringUncompressor;
- private boolean hasCompression = false;
+ protected FSDataInputStream inputStream;
+ protected Codec.Decoder cellDecoder;
+ protected WALCellCodec.ByteStringUncompressor byteStringUncompressor;
+ protected boolean hasCompression = false;
// walEditsStopOffset is the position of the last byte to read. After reading the last WALEdit entry
// in the hlog, the inputstream's position is equal to walEditsStopOffset.
private long walEditsStopOffset;
@@ -92,6 +94,10 @@ public class ProtobufLogReader extends R
initInternal(stream, true);
}
+ protected boolean readHeader(Builder builder, FSDataInputStream stream) throws IOException {
+ return builder.mergeDelimitedFrom(stream);
+ }
+
private void initInternal(FSDataInputStream stream, boolean isFirst) throws IOException {
close();
long expectedPos = PB_WAL_MAGIC.length;
@@ -104,7 +110,7 @@ public class ProtobufLogReader extends R
}
// Initialize metadata or, when we reset, just skip the header.
WALProtos.WALHeader.Builder builder = WALProtos.WALHeader.newBuilder();
- boolean hasHeader = builder.mergeDelimitedFrom(stream);
+ boolean hasHeader = readHeader(builder, stream);
if (!hasHeader) {
throw new EOFException("Couldn't read WAL PB header");
}
@@ -176,9 +182,14 @@ public class ProtobufLogReader extends R
return false;
}
+ protected WALCellCodec getCodec(Configuration conf, CompressionContext compressionContext)
+ throws IOException {
+ return WALCellCodec.create(conf, compressionContext);
+ }
+
@Override
protected void initAfterCompression() throws IOException {
- WALCellCodec codec = WALCellCodec.create(this.conf, this.compressionContext);
+ WALCellCodec codec = getCodec(this.conf, this.compressionContext);
this.cellDecoder = codec.getDecoder(this.inputStream);
if (this.hasCompression) {
this.byteStringUncompressor = codec.getByteStringUncompressor();
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java?rev=1545536&r1=1545535&r2=1545536&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java Tue Nov 26 04:22:33 2013
@@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.util.FSUt
@InterfaceAudience.Private
public class ProtobufLogWriter extends WriterBase {
private final Log LOG = LogFactory.getLog(this.getClass());
- private FSDataOutputStream output;
- private Codec.Encoder cellEncoder;
- private WALCellCodec.ByteStringCompressor compressor;
+ protected FSDataOutputStream output;
+ protected Codec.Encoder cellEncoder;
+ protected WALCellCodec.ByteStringCompressor compressor;
private boolean trailerWritten;
private WALTrailer trailer;
// maximum size of the wal Trailer in bytes. If a user writes/reads a trailer with size larger
@@ -53,9 +53,19 @@ public class ProtobufLogWriter extends W
super();
}
+ protected WALCellCodec getCodec(Configuration conf, CompressionContext compressionContext)
+ throws IOException {
+ return WALCellCodec.create(conf, compressionContext);
+ }
+
+ protected WALHeader buildWALHeader(WALHeader.Builder builder) throws IOException {
+ return builder.build();
+ }
+
@Override
@SuppressWarnings("deprecation")
public void init(FileSystem fs, Path path, Configuration conf, boolean overwritable) throws IOException {
+ super.init(fs, path, conf, overwritable);
assert this.output == null;
boolean doCompress = initializeCompressionContext(conf, path);
this.trailerWarnSize = conf.getInt(HLog.WAL_TRAILER_WARN_SIZE,
@@ -67,13 +77,10 @@ public class ProtobufLogWriter extends W
FSUtils.getDefaultBlockSize(fs, path));
output = fs.createNonRecursive(path, overwritable, bufferSize, replication, blockSize, null);
output.write(ProtobufLogReader.PB_WAL_MAGIC);
- WALHeader.newBuilder().setHasCompression(doCompress).build().writeDelimitedTo(output);
+ buildWALHeader(WALHeader.newBuilder().setHasCompression(doCompress)).writeDelimitedTo(output);
+
+ initAfterHeader(doCompress);
- WALCellCodec codec = WALCellCodec.create(conf, this.compressionContext);
- this.cellEncoder = codec.getEncoder(this.output);
- if (doCompress) {
- this.compressor = codec.getByteStringCompressor();
- }
// instantiate trailer to default value.
trailer = WALTrailer.newBuilder().build();
if (LOG.isTraceEnabled()) {
@@ -81,6 +88,14 @@ public class ProtobufLogWriter extends W
}
}
+ protected void initAfterHeader(boolean doCompress) throws IOException {
+ WALCellCodec codec = getCodec(conf, this.compressionContext);
+ this.cellEncoder = codec.getEncoder(this.output);
+ if (doCompress) {
+ this.compressor = codec.getByteStringCompressor();
+ }
+ }
+
@Override
public void append(HLog.Entry entry) throws IOException {
entry.setCompressionContext(compressionContext);
@@ -106,13 +121,17 @@ public class ProtobufLogWriter extends W
}
}
+ protected WALTrailer buildWALTrailer(WALTrailer.Builder builder) {
+ return builder.build();
+ }
+
private void writeWALTrailer() {
try {
int trailerSize = 0;
if (this.trailer == null) {
// use default trailer.
LOG.warn("WALTrailer is null. Continuing with default.");
- this.trailer = WALTrailer.newBuilder().build();
+ this.trailer = buildWALTrailer(WALTrailer.newBuilder());
trailerSize = this.trailer.getSerializedSize();
} else if ((trailerSize = this.trailer.getSerializedSize()) > this.trailerWarnSize) {
// continue writing after warning the user.
@@ -120,8 +139,8 @@ public class ProtobufLogWriter extends W
trailerSize + " > " + this.trailerWarnSize);
}
this.trailer.writeTo(output);
- this.output.writeInt(trailerSize);
- this.output.write(ProtobufLogReader.PB_WAL_COMPLETE_MAGIC);
+ output.writeInt(trailerSize);
+ output.write(ProtobufLogReader.PB_WAL_COMPLETE_MAGIC);
this.trailerWritten = true;
} catch (IOException ioe) {
LOG.error("Got IOException while writing trailer", ioe);
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java?rev=1545536&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java Tue Nov 26 04:22:33 2013
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.IOException;
+import java.security.Key;
+import java.security.KeyException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.crypto.Cipher;
+import org.apache.hadoop.hbase.io.crypto.Decryptor;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALHeader;
+import org.apache.hadoop.hbase.security.EncryptionUtil;
+import org.apache.hadoop.hbase.security.User;
+
+public class SecureProtobufLogReader extends ProtobufLogReader {
+
+ private static final Log LOG = LogFactory.getLog(SecureProtobufLogReader.class);
+
+ private Decryptor decryptor = null;
+
+ @Override
+ protected boolean readHeader(WALHeader.Builder builder, FSDataInputStream stream)
+ throws IOException {
+ boolean result = super.readHeader(builder, stream);
+ // We need to unconditionally handle the case where the WAL has a key in
+ // the header, meaning it is encrypted, even if ENABLE_WAL_ENCRYPTION is
+ // no longer set in the site configuration.
+ if (result && builder.hasEncryptionKey()) {
+ // Serialized header data has been merged into the builder from the
+ // stream.
+
+ // Retrieve a usable key
+
+ byte[] keyBytes = builder.getEncryptionKey().toByteArray();
+ Key key = null;
+ String walKeyName = conf.get(HConstants.CRYPTO_WAL_KEY_NAME_CONF_KEY);
+ // First try the WAL key, if one is configured
+ if (walKeyName != null) {
+ try {
+ key = EncryptionUtil.unwrapKey(conf, walKeyName, keyBytes);
+ } catch (KeyException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unable to unwrap key with WAL key '" + walKeyName + "'");
+ }
+ key = null;
+ }
+ }
+ if (key == null) {
+ String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
+ User.getCurrent().getShortName());
+ try {
+ // Then, try the cluster master key
+ key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
+ } catch (KeyException e) {
+ // If the current master key fails to unwrap, try the alternate, if
+ // one is configured
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
+ }
+ String alternateKeyName =
+ conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
+ if (alternateKeyName != null) {
+ try {
+ key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
+ } catch (KeyException ex) {
+ throw new IOException(ex);
+ }
+ } else {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ // Use the algorithm the key wants
+
+ Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
+ if (cipher == null) {
+ throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available");
+ }
+
+ // Set up the decryptor for this WAL
+
+ decryptor = cipher.getDecryptor();
+ decryptor.setKey(key);
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Initialized secure protobuf WAL: cipher=" + cipher.getName());
+ }
+ }
+
+ return result;
+ }
+
+ @Override
+ protected void initAfterCompression() throws IOException {
+ WALCellCodec codec = SecureWALCellCodec.getCodec(this.conf, decryptor);
+ this.cellDecoder = codec.getDecoder(this.inputStream);
+ // We do not support compression
+ this.compressionContext = null;
+ this.hasCompression = false;
+ }
+
+}
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java?rev=1545536&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java Tue Nov 26 04:22:33 2013
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.IOException;
+import java.security.Key;
+import java.security.SecureRandom;
+
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.crypto.Cipher;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
+import org.apache.hadoop.hbase.io.crypto.Encryptor;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALHeader;
+import org.apache.hadoop.hbase.security.EncryptionUtil;
+import org.apache.hadoop.hbase.security.User;
+
+import com.google.protobuf.ZeroCopyLiteralByteString;
+
+public class SecureProtobufLogWriter extends ProtobufLogWriter {
+
+ private static final Log LOG = LogFactory.getLog(SecureProtobufLogWriter.class);
+ private static final String DEFAULT_CIPHER = "AES";
+
+ private Encryptor encryptor = null;
+
+ @Override
+ protected WALHeader buildWALHeader(WALHeader.Builder builder) throws IOException {
+ if (conf.getBoolean(HConstants.ENABLE_WAL_ENCRYPTION, false)) {
+ // Get an instance of our cipher
+ Cipher cipher = Encryption.getCipher(conf,
+ conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, DEFAULT_CIPHER));
+ if (cipher == null) {
+ throw new RuntimeException("Cipher '" + cipher + "' is not available");
+ }
+
+ // Generate an encryption key for this WAL
+ SecureRandom rng = new SecureRandom();
+ byte[] keyBytes = new byte[cipher.getKeyLength()];
+ rng.nextBytes(keyBytes);
+ Key key = new SecretKeySpec(keyBytes, cipher.getName());
+ builder.setEncryptionKey(ZeroCopyLiteralByteString.wrap(EncryptionUtil.wrapKey(conf,
+ conf.get(HConstants.CRYPTO_WAL_KEY_NAME_CONF_KEY,
+ conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
+ User.getCurrent().getShortName())),
+ key)));
+
+ // Set up the encryptor
+ encryptor = cipher.getEncryptor();
+ encryptor.setKey(key);
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Initialized secure protobuf WAL: cipher=" + cipher.getName());
+ }
+ }
+
+ return super.buildWALHeader(builder);
+ }
+
+ @Override
+ protected void initAfterHeader(boolean doCompress) throws IOException {
+ if (conf.getBoolean(HConstants.ENABLE_WAL_ENCRYPTION, false) && encryptor != null) {
+ WALCellCodec codec = SecureWALCellCodec.getCodec(this.conf, encryptor);
+ this.cellEncoder = codec.getEncoder(this.output);
+ // We do not support compression
+ this.compressionContext = null;
+ } else {
+ super.initAfterHeader(doCompress);
+ }
+ }
+
+}
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java?rev=1545536&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java Tue Nov 26 04:22:33 2013
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.SecureRandom;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.codec.KeyValueCodec;
+import org.apache.hadoop.hbase.io.crypto.Decryptor;
+import org.apache.hadoop.hbase.io.crypto.Encryptor;
+import org.apache.hadoop.hbase.io.util.StreamUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * A WALCellCodec that encrypts the WALedits.
+ */
+public class SecureWALCellCodec extends WALCellCodec {
+
+ private static final SecureRandom RNG = new SecureRandom();
+
+ private Encryptor encryptor;
+ private Decryptor decryptor;
+
+ public SecureWALCellCodec(Configuration conf, Encryptor encryptor) {
+ super(conf, null);
+ this.encryptor = encryptor;
+ }
+
+ public SecureWALCellCodec(Configuration conf, Decryptor decryptor) {
+ super(conf, null);
+ this.decryptor = decryptor;
+ }
+
+ static class EncryptedKvDecoder extends KeyValueCodec.KeyValueDecoder {
+
+ private Decryptor decryptor;
+ private byte[] iv;
+
+ public EncryptedKvDecoder(InputStream in) {
+ super(in);
+ }
+
+ public EncryptedKvDecoder(InputStream in, Decryptor decryptor) {
+ super(in);
+ this.decryptor = decryptor;
+ this.iv = new byte[decryptor.getIvLength()];
+ }
+
+ @Override
+ protected Cell parseCell() throws IOException {
+ int ivLength = 0;
+ try {
+ ivLength = StreamUtils.readRawVarint32(in);
+ } catch (EOFException e) {
+ // EOF at start is OK
+ return null;
+ }
+
+ // TODO: An IV length of 0 could signify an unwrapped cell, when the
+ // encoder supports that just read the remainder in directly
+
+ if (ivLength != this.iv.length) {
+ throw new IOException("Incorrect IV length: expected=" + iv.length + " have=" +
+ ivLength);
+ }
+ IOUtils.readFully(in, this.iv);
+
+ int codedLength = StreamUtils.readRawVarint32(in);
+ byte[] codedBytes = new byte[codedLength];
+ IOUtils.readFully(in, codedBytes);
+
+ decryptor.setIv(iv);
+ decryptor.reset();
+
+ InputStream cin = decryptor.createDecryptionStream(new ByteArrayInputStream(codedBytes));
+
+ // TODO: Add support for WAL compression
+
+ int keylength = StreamUtils.readRawVarint32(cin);
+ int vlength = StreamUtils.readRawVarint32(cin);
+ int tagsLength = StreamUtils.readRawVarint32(cin);
+ int length = 0;
+ if (tagsLength == 0) {
+ length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength;
+ } else {
+ length = KeyValue.KEYVALUE_WITH_TAGS_INFRASTRUCTURE_SIZE + keylength + vlength + tagsLength;
+ }
+
+ byte[] backingArray = new byte[length];
+ int pos = 0;
+ pos = Bytes.putInt(backingArray, pos, keylength);
+ pos = Bytes.putInt(backingArray, pos, vlength);
+
+ // Row
+ int elemLen = StreamUtils.readRawVarint32(cin);
+ pos = Bytes.putShort(backingArray, pos, (short)elemLen);
+ IOUtils.readFully(cin, backingArray, pos, elemLen);
+ pos += elemLen;
+ // Family
+ elemLen = StreamUtils.readRawVarint32(cin);
+ pos = Bytes.putByte(backingArray, pos, (byte)elemLen);
+ IOUtils.readFully(cin, backingArray, pos, elemLen);
+ pos += elemLen;
+ // Qualifier
+ elemLen = StreamUtils.readRawVarint32(cin);
+ IOUtils.readFully(cin, backingArray, pos, elemLen);
+ pos += elemLen;
+ // Remainder
+ IOUtils.readFully(cin, backingArray, pos, length - pos);
+ return new KeyValue(backingArray, 0, length);
+ }
+
+ }
+
+ static class EncryptedKvEncoder extends KeyValueCodec.KeyValueEncoder {
+
+ private Encryptor encryptor;
+ private byte[] iv;
+
+ public EncryptedKvEncoder(OutputStream os) {
+ super(os);
+ }
+
+ public EncryptedKvEncoder(OutputStream os, Encryptor encryptor) {
+ super(os);
+ this.encryptor = encryptor;
+ iv = new byte[encryptor.getIvLength()];
+ }
+
+ @Override
+ public void write(Cell cell) throws IOException {
+ if (!(cell instanceof KeyValue)) throw new IOException("Cannot write non-KV cells to WAL");
+
+ KeyValue kv = (KeyValue)cell;
+ byte[] kvBuffer = kv.getBuffer();
+ int offset = kv.getOffset();
+
+ RNG.nextBytes(iv);
+ encryptor.setIv(iv);
+ encryptor.reset();
+
+ // TODO: Check if this is a cell for an encrypted CF. If not, we can
+ // write a 0 here to signal an unwrapped cell and just dump the KV bytes
+ // afterward
+
+ StreamUtils.writeRawVInt32(out, iv.length);
+ out.write(iv);
+
+ // TODO: Add support for WAL compression
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ OutputStream cout = encryptor.createEncryptionStream(baos);
+
+ // Write the KeyValue infrastructure as VInts.
+ StreamUtils.writeRawVInt32(cout, kv.getKeyLength());
+ StreamUtils.writeRawVInt32(cout, kv.getValueLength());
+ // To support tags
+ StreamUtils.writeRawVInt32(cout, kv.getTagsLength());
+
+ // Write row, qualifier, and family
+ StreamUtils.writeRawVInt32(cout, kv.getRowLength());
+ cout.write(kvBuffer, kv.getRowOffset(), kv.getRowLength());
+ StreamUtils.writeRawVInt32(cout, kv.getFamilyLength());
+ cout.write(kvBuffer, kv.getFamilyOffset(), kv.getFamilyLength());
+ StreamUtils.writeRawVInt32(cout, kv.getQualifierLength());
+ cout.write(kvBuffer, kv.getQualifierOffset(), kv.getQualifierLength());
+ // Write the rest
+ int pos = kv.getTimestampOffset();
+ int remainingLength = kv.getLength() + offset - pos;
+ cout.write(kvBuffer, pos, remainingLength);
+ cout.close();
+
+ byte[] codedBytes = baos.toByteArray();
+ StreamUtils.writeRawVInt32(out, codedBytes.length);
+ out.write(codedBytes);
+ }
+
+ }
+
+ @Override
+ public Decoder getDecoder(InputStream is) {
+ return new EncryptedKvDecoder(is, decryptor);
+ }
+
+ @Override
+ public Encoder getEncoder(OutputStream os) {
+ return new EncryptedKvEncoder(os, encryptor);
+ }
+
+ public static WALCellCodec getCodec(Configuration conf, Encryptor encryptor) {
+ return new SecureWALCellCodec(conf, encryptor);
+ }
+
+ public static WALCellCodec getCodec(Configuration conf, Decryptor decryptor) {
+ return new SecureWALCellCodec(conf, decryptor);
+ }
+
+}