You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2014/07/04 19:51:51 UTC
[1/6] git commit: HBASE-11446 Reduce the frequency of RNG calls in
SecureWALCellCodec#EncryptedKvEncoder
Repository: hbase
Updated Branches:
refs/heads/0.98 5e9a85714 -> f348aa719
refs/heads/branch-1 aa414085e -> f36bf219e
refs/heads/master 360df59c0 -> 94d3dbea8
HBASE-11446 Reduce the frequency of RNG calls in SecureWALCellCodec#EncryptedKvEncoder
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2416d798
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2416d798
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2416d798
Branch: refs/heads/master
Commit: 2416d798facc8e5721940b8ae65597cf48e3cd1f
Parents: 360df59
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Jul 4 10:45:19 2014 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Jul 4 10:45:19 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/io/crypto/Decryptor.java | 6 +++++
.../hadoop/hbase/io/crypto/Encryption.java | 22 ++++++++++-----
.../hadoop/hbase/io/crypto/Encryptor.java | 7 ++++-
.../apache/hadoop/hbase/io/crypto/aes/AES.java | 1 +
.../hbase/io/crypto/aes/AESDecryptor.java | 5 ++++
.../hbase/io/crypto/aes/AESEncryptor.java | 5 ++++
.../regionserver/wal/SecureWALCellCodec.java | 28 ++++++++++++++++----
7 files changed, 61 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2416d798/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java
index ff0fc8c..adb8121 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java
@@ -43,6 +43,12 @@ public interface Decryptor {
public int getIvLength();
/**
+ * Get the cipher's internal block size
+ * @return the cipher's internal block size
+ */
+ public int getBlockSize();
+
+ /**
* Set the initialization vector
* @param iv
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/2416d798/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java
index aabf65d..87b1c60 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java
@@ -537,16 +537,24 @@ public final class Encryption {
}
public static void incrementIv(byte[] iv) {
+ incrementIv(iv, 1);
+ }
+
+ public static void incrementIv(byte[] iv, int v) {
int length = iv.length;
boolean carry = true;
- for (int i = 0; i < length; i++) {
- if (carry) {
- iv[i] = (byte) ((iv[i] + 1) & 0xFF);
- carry = 0 == iv[i];
- } else {
- break;
+ // TODO: Optimize for v > 1, e.g. 16, 32
+ do {
+ for (int i = 0; i < length; i++) {
+ if (carry) {
+ iv[i] = (byte) ((iv[i] + 1) & 0xFF);
+ carry = 0 == iv[i];
+ } else {
+ break;
+ }
}
- }
+ v--;
+ } while (v > 0);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2416d798/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java
index 902a536..1163c3f 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java
@@ -43,6 +43,12 @@ public interface Encryptor {
public int getIvLength();
/**
+ * Get the cipher's internal block size
+ * @return the cipher's internal block size
+ */
+ public int getBlockSize();
+
+ /**
* Get the initialization vector
*/
public byte[] getIv();
@@ -63,5 +69,4 @@ public interface Encryptor {
* Reset state, reinitialize with the key and iv
*/
void reset();
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2416d798/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
index 86e5e78..fa20a8a 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
@@ -53,6 +53,7 @@ public class AES extends Cipher {
public static final int KEY_LENGTH = 16;
public static final int KEY_LENGTH_BITS = KEY_LENGTH * 8;
+ public static final int BLOCK_SIZE = 16;
public static final int IV_LENGTH = 16;
public static final String CIPHER_MODE_KEY = "hbase.crypto.algorithm.aes.mode";
http://git-wip-us.apache.org/repos/asf/hbase/blob/2416d798/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java
index 8dc6b9e..2cb0618 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java
@@ -59,6 +59,11 @@ public class AESDecryptor implements Decryptor {
}
@Override
+ public int getBlockSize() {
+ return AES.BLOCK_SIZE;
+ }
+
+ @Override
public void setIv(byte[] iv) {
Preconditions.checkNotNull(iv, "IV cannot be null");
Preconditions.checkArgument(iv.length == AES.IV_LENGTH, "Invalid IV length");
http://git-wip-us.apache.org/repos/asf/hbase/blob/2416d798/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java
index 38e6b8d..1c93fbd 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java
@@ -61,6 +61,11 @@ public class AESEncryptor implements Encryptor {
}
@Override
+ public int getBlockSize() {
+ return AES.BLOCK_SIZE;
+ }
+
+ @Override
public byte[] getIv() {
return iv;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2416d798/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
index 35b7193..e36d741 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.codec.KeyValueCodec;
import org.apache.hadoop.hbase.io.crypto.Decryptor;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.crypto.Encryptor;
import org.apache.hadoop.hbase.io.util.StreamUtils;
import org.apache.hadoop.hbase.util.Bytes;
@@ -40,8 +41,6 @@ import org.apache.hadoop.hbase.util.Bytes;
*/
public class SecureWALCellCodec extends WALCellCodec {
- private static final SecureRandom RNG = new SecureRandom();
-
private Encryptor encryptor;
private Decryptor decryptor;
@@ -139,7 +138,24 @@ public class SecureWALCellCodec extends WALCellCodec {
static class EncryptedKvEncoder extends KeyValueCodec.KeyValueEncoder {
private Encryptor encryptor;
- private byte[] iv;
+ private final ThreadLocal<byte[]> iv = new ThreadLocal<byte[]>() {
+ @Override
+ protected byte[] initialValue() {
+ byte[] iv = new byte[encryptor.getIvLength()];
+ new SecureRandom().nextBytes(iv);
+ return iv;
+ }
+ };
+
+ protected byte[] nextIv() {
+ byte[] b = iv.get(), ret = new byte[b.length];
+ System.arraycopy(b, 0, ret, 0, b.length);
+ return ret;
+ }
+
+ protected void incrementIv(int v) {
+ Encryption.incrementIv(iv.get(), 1 + (v / encryptor.getBlockSize()));
+ }
public EncryptedKvEncoder(OutputStream os) {
super(os);
@@ -148,7 +164,6 @@ public class SecureWALCellCodec extends WALCellCodec {
public EncryptedKvEncoder(OutputStream os, Encryptor encryptor) {
super(os);
this.encryptor = encryptor;
- iv = new byte[encryptor.getIvLength()];
}
@Override
@@ -159,7 +174,7 @@ public class SecureWALCellCodec extends WALCellCodec {
byte[] kvBuffer = kv.getBuffer();
int offset = kv.getOffset();
- RNG.nextBytes(iv);
+ byte[] iv = nextIv();
encryptor.setIv(iv);
encryptor.reset();
@@ -196,6 +211,9 @@ public class SecureWALCellCodec extends WALCellCodec {
StreamUtils.writeRawVInt32(out, baos.size());
baos.writeTo(out);
+
+ // Increment IV given the final payload length
+ incrementIv(baos.size());
}
}
[6/6] git commit: HBASE-11457 Increment HFile block encoding IVs
accounting for cipher's internal use
Posted by ap...@apache.org.
HBASE-11457 Increment HFile block encoding IVs accounting for cipher's internal use
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f348aa71
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f348aa71
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f348aa71
Branch: refs/heads/0.98
Commit: f348aa719975e64ae5a6d349ca304fe77aa41b5c
Parents: 624a66e
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Jul 4 10:45:32 2014 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Jul 4 10:50:25 2014 -0700
----------------------------------------------------------------------
.../hbase/io/encoding/HFileBlockDefaultEncodingContext.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f348aa71/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
index 8386377..98581cc 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
@@ -206,7 +206,6 @@ public class HFileBlockDefaultEncodingContext implements
Preconditions.checkState(ivLength <= Byte.MAX_VALUE, "IV length out of range");
cryptoByteStream.write(ivLength);
if (ivLength > 0) {
- Encryption.incrementIv(iv);
encryptor.setIv(iv);
cryptoByteStream.write(iv);
}
@@ -216,6 +215,9 @@ public class HFileBlockDefaultEncodingContext implements
onDiskBytesWithHeader = cryptoByteStream.toByteArray();
+ // Increment the IV given the final block size
+ Encryption.incrementIv(iv, 1 + (onDiskBytesWithHeader.length / encryptor.getBlockSize()));
+
} else {
cryptoByteStream.write(0);
[3/6] git commit: HBASE-11446 Reduce the frequency of RNG calls in
SecureWALCellCodec#EncryptedKvEncoder
Posted by ap...@apache.org.
HBASE-11446 Reduce the frequency of RNG calls in SecureWALCellCodec#EncryptedKvEncoder
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0b692dfd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0b692dfd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0b692dfd
Branch: refs/heads/branch-1
Commit: 0b692dfd5ab1cdbcd187551d43d30915665ea57b
Parents: aa41408
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Jul 4 10:45:19 2014 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Jul 4 10:48:57 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/io/crypto/Decryptor.java | 6 +++++
.../hadoop/hbase/io/crypto/Encryption.java | 22 ++++++++++-----
.../hadoop/hbase/io/crypto/Encryptor.java | 7 ++++-
.../apache/hadoop/hbase/io/crypto/aes/AES.java | 1 +
.../hbase/io/crypto/aes/AESDecryptor.java | 5 ++++
.../hbase/io/crypto/aes/AESEncryptor.java | 5 ++++
.../regionserver/wal/SecureWALCellCodec.java | 28 ++++++++++++++++----
7 files changed, 61 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b692dfd/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java
index ff0fc8c..adb8121 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java
@@ -43,6 +43,12 @@ public interface Decryptor {
public int getIvLength();
/**
+ * Get the cipher's internal block size
+ * @return the cipher's internal block size
+ */
+ public int getBlockSize();
+
+ /**
* Set the initialization vector
* @param iv
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b692dfd/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java
index aabf65d..87b1c60 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java
@@ -537,16 +537,24 @@ public final class Encryption {
}
public static void incrementIv(byte[] iv) {
+ incrementIv(iv, 1);
+ }
+
+ public static void incrementIv(byte[] iv, int v) {
int length = iv.length;
boolean carry = true;
- for (int i = 0; i < length; i++) {
- if (carry) {
- iv[i] = (byte) ((iv[i] + 1) & 0xFF);
- carry = 0 == iv[i];
- } else {
- break;
+ // TODO: Optimize for v > 1, e.g. 16, 32
+ do {
+ for (int i = 0; i < length; i++) {
+ if (carry) {
+ iv[i] = (byte) ((iv[i] + 1) & 0xFF);
+ carry = 0 == iv[i];
+ } else {
+ break;
+ }
}
- }
+ v--;
+ } while (v > 0);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b692dfd/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java
index 902a536..1163c3f 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java
@@ -43,6 +43,12 @@ public interface Encryptor {
public int getIvLength();
/**
+ * Get the cipher's internal block size
+ * @return the cipher's internal block size
+ */
+ public int getBlockSize();
+
+ /**
* Get the initialization vector
*/
public byte[] getIv();
@@ -63,5 +69,4 @@ public interface Encryptor {
* Reset state, reinitialize with the key and iv
*/
void reset();
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b692dfd/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
index 86e5e78..fa20a8a 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
@@ -53,6 +53,7 @@ public class AES extends Cipher {
public static final int KEY_LENGTH = 16;
public static final int KEY_LENGTH_BITS = KEY_LENGTH * 8;
+ public static final int BLOCK_SIZE = 16;
public static final int IV_LENGTH = 16;
public static final String CIPHER_MODE_KEY = "hbase.crypto.algorithm.aes.mode";
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b692dfd/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java
index 8dc6b9e..2cb0618 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java
@@ -59,6 +59,11 @@ public class AESDecryptor implements Decryptor {
}
@Override
+ public int getBlockSize() {
+ return AES.BLOCK_SIZE;
+ }
+
+ @Override
public void setIv(byte[] iv) {
Preconditions.checkNotNull(iv, "IV cannot be null");
Preconditions.checkArgument(iv.length == AES.IV_LENGTH, "Invalid IV length");
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b692dfd/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java
index 38e6b8d..1c93fbd 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java
@@ -61,6 +61,11 @@ public class AESEncryptor implements Encryptor {
}
@Override
+ public int getBlockSize() {
+ return AES.BLOCK_SIZE;
+ }
+
+ @Override
public byte[] getIv() {
return iv;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b692dfd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
index 35b7193..e36d741 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.codec.KeyValueCodec;
import org.apache.hadoop.hbase.io.crypto.Decryptor;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.crypto.Encryptor;
import org.apache.hadoop.hbase.io.util.StreamUtils;
import org.apache.hadoop.hbase.util.Bytes;
@@ -40,8 +41,6 @@ import org.apache.hadoop.hbase.util.Bytes;
*/
public class SecureWALCellCodec extends WALCellCodec {
- private static final SecureRandom RNG = new SecureRandom();
-
private Encryptor encryptor;
private Decryptor decryptor;
@@ -139,7 +138,24 @@ public class SecureWALCellCodec extends WALCellCodec {
static class EncryptedKvEncoder extends KeyValueCodec.KeyValueEncoder {
private Encryptor encryptor;
- private byte[] iv;
+ private final ThreadLocal<byte[]> iv = new ThreadLocal<byte[]>() {
+ @Override
+ protected byte[] initialValue() {
+ byte[] iv = new byte[encryptor.getIvLength()];
+ new SecureRandom().nextBytes(iv);
+ return iv;
+ }
+ };
+
+ protected byte[] nextIv() {
+ byte[] b = iv.get(), ret = new byte[b.length];
+ System.arraycopy(b, 0, ret, 0, b.length);
+ return ret;
+ }
+
+ protected void incrementIv(int v) {
+ Encryption.incrementIv(iv.get(), 1 + (v / encryptor.getBlockSize()));
+ }
public EncryptedKvEncoder(OutputStream os) {
super(os);
@@ -148,7 +164,6 @@ public class SecureWALCellCodec extends WALCellCodec {
public EncryptedKvEncoder(OutputStream os, Encryptor encryptor) {
super(os);
this.encryptor = encryptor;
- iv = new byte[encryptor.getIvLength()];
}
@Override
@@ -159,7 +174,7 @@ public class SecureWALCellCodec extends WALCellCodec {
byte[] kvBuffer = kv.getBuffer();
int offset = kv.getOffset();
- RNG.nextBytes(iv);
+ byte[] iv = nextIv();
encryptor.setIv(iv);
encryptor.reset();
@@ -196,6 +211,9 @@ public class SecureWALCellCodec extends WALCellCodec {
StreamUtils.writeRawVInt32(out, baos.size());
baos.writeTo(out);
+
+ // Increment IV given the final payload length
+ incrementIv(baos.size());
}
}
[5/6] git commit: HBASE-11446 Reduce the frequency of RNG calls in
SecureWALCellCodec#EncryptedKvEncoder
Posted by ap...@apache.org.
HBASE-11446 Reduce the frequency of RNG calls in SecureWALCellCodec#EncryptedKvEncoder
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/624a66ec
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/624a66ec
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/624a66ec
Branch: refs/heads/0.98
Commit: 624a66ecaadb0f3838c696fee035dff470a28e1c
Parents: 5e9a857
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Jul 4 10:45:19 2014 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Jul 4 10:50:23 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/io/crypto/Decryptor.java | 6 +++++
.../hadoop/hbase/io/crypto/Encryption.java | 22 ++++++++++-----
.../hadoop/hbase/io/crypto/Encryptor.java | 7 ++++-
.../apache/hadoop/hbase/io/crypto/aes/AES.java | 1 +
.../hbase/io/crypto/aes/AESDecryptor.java | 5 ++++
.../hbase/io/crypto/aes/AESEncryptor.java | 5 ++++
.../regionserver/wal/SecureWALCellCodec.java | 28 ++++++++++++++++----
7 files changed, 61 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/624a66ec/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java
index ff0fc8c..adb8121 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Decryptor.java
@@ -43,6 +43,12 @@ public interface Decryptor {
public int getIvLength();
/**
+ * Get the cipher's internal block size
+ * @return the cipher's internal block size
+ */
+ public int getBlockSize();
+
+ /**
* Set the initialization vector
* @param iv
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/624a66ec/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java
index 35935d9..7a4e36a 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java
@@ -536,16 +536,24 @@ public final class Encryption {
}
public static void incrementIv(byte[] iv) {
+ incrementIv(iv, 1);
+ }
+
+ public static void incrementIv(byte[] iv, int v) {
int length = iv.length;
boolean carry = true;
- for (int i = 0; i < length; i++) {
- if (carry) {
- iv[i] = (byte) ((iv[i] + 1) & 0xFF);
- carry = 0 == iv[i];
- } else {
- break;
+ // TODO: Optimize for v > 1, e.g. 16, 32
+ do {
+ for (int i = 0; i < length; i++) {
+ if (carry) {
+ iv[i] = (byte) ((iv[i] + 1) & 0xFF);
+ carry = 0 == iv[i];
+ } else {
+ break;
+ }
}
- }
+ v--;
+ } while (v > 0);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/624a66ec/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java
index 902a536..1163c3f 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryptor.java
@@ -43,6 +43,12 @@ public interface Encryptor {
public int getIvLength();
/**
+ * Get the cipher's internal block size
+ * @return the cipher's internal block size
+ */
+ public int getBlockSize();
+
+ /**
* Get the initialization vector
*/
public byte[] getIv();
@@ -63,5 +69,4 @@ public interface Encryptor {
* Reset state, reinitialize with the key and iv
*/
void reset();
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/624a66ec/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
index 86e5e78..fa20a8a 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java
@@ -53,6 +53,7 @@ public class AES extends Cipher {
public static final int KEY_LENGTH = 16;
public static final int KEY_LENGTH_BITS = KEY_LENGTH * 8;
+ public static final int BLOCK_SIZE = 16;
public static final int IV_LENGTH = 16;
public static final String CIPHER_MODE_KEY = "hbase.crypto.algorithm.aes.mode";
http://git-wip-us.apache.org/repos/asf/hbase/blob/624a66ec/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java
index 8dc6b9e..2cb0618 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESDecryptor.java
@@ -59,6 +59,11 @@ public class AESDecryptor implements Decryptor {
}
@Override
+ public int getBlockSize() {
+ return AES.BLOCK_SIZE;
+ }
+
+ @Override
public void setIv(byte[] iv) {
Preconditions.checkNotNull(iv, "IV cannot be null");
Preconditions.checkArgument(iv.length == AES.IV_LENGTH, "Invalid IV length");
http://git-wip-us.apache.org/repos/asf/hbase/blob/624a66ec/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java
index 38e6b8d..1c93fbd 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AESEncryptor.java
@@ -61,6 +61,11 @@ public class AESEncryptor implements Encryptor {
}
@Override
+ public int getBlockSize() {
+ return AES.BLOCK_SIZE;
+ }
+
+ @Override
public byte[] getIv() {
return iv;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/624a66ec/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
index 35b7193..e36d741 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.codec.KeyValueCodec;
import org.apache.hadoop.hbase.io.crypto.Decryptor;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.crypto.Encryptor;
import org.apache.hadoop.hbase.io.util.StreamUtils;
import org.apache.hadoop.hbase.util.Bytes;
@@ -40,8 +41,6 @@ import org.apache.hadoop.hbase.util.Bytes;
*/
public class SecureWALCellCodec extends WALCellCodec {
- private static final SecureRandom RNG = new SecureRandom();
-
private Encryptor encryptor;
private Decryptor decryptor;
@@ -139,7 +138,24 @@ public class SecureWALCellCodec extends WALCellCodec {
static class EncryptedKvEncoder extends KeyValueCodec.KeyValueEncoder {
private Encryptor encryptor;
- private byte[] iv;
+ private final ThreadLocal<byte[]> iv = new ThreadLocal<byte[]>() {
+ @Override
+ protected byte[] initialValue() {
+ byte[] iv = new byte[encryptor.getIvLength()];
+ new SecureRandom().nextBytes(iv);
+ return iv;
+ }
+ };
+
+ protected byte[] nextIv() {
+ byte[] b = iv.get(), ret = new byte[b.length];
+ System.arraycopy(b, 0, ret, 0, b.length);
+ return ret;
+ }
+
+ protected void incrementIv(int v) {
+ Encryption.incrementIv(iv.get(), 1 + (v / encryptor.getBlockSize()));
+ }
public EncryptedKvEncoder(OutputStream os) {
super(os);
@@ -148,7 +164,6 @@ public class SecureWALCellCodec extends WALCellCodec {
public EncryptedKvEncoder(OutputStream os, Encryptor encryptor) {
super(os);
this.encryptor = encryptor;
- iv = new byte[encryptor.getIvLength()];
}
@Override
@@ -159,7 +174,7 @@ public class SecureWALCellCodec extends WALCellCodec {
byte[] kvBuffer = kv.getBuffer();
int offset = kv.getOffset();
- RNG.nextBytes(iv);
+ byte[] iv = nextIv();
encryptor.setIv(iv);
encryptor.reset();
@@ -196,6 +211,9 @@ public class SecureWALCellCodec extends WALCellCodec {
StreamUtils.writeRawVInt32(out, baos.size());
baos.writeTo(out);
+
+ // Increment IV given the final payload length
+ incrementIv(baos.size());
}
}
[4/6] git commit: HBASE-11457 Increment HFile block encoding IVs
accounting for cipher's internal use
Posted by ap...@apache.org.
HBASE-11457 Increment HFile block encoding IVs accounting for cipher's internal use
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f36bf219
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f36bf219
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f36bf219
Branch: refs/heads/branch-1
Commit: f36bf219ec7827eaef251cac94253d50acf2c1ae
Parents: 0b692df
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Jul 4 10:45:32 2014 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Jul 4 10:49:03 2014 -0700
----------------------------------------------------------------------
.../hbase/io/encoding/HFileBlockDefaultEncodingContext.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f36bf219/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
index 0dc1c8a..7dfe857 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
@@ -187,7 +187,6 @@ public class HFileBlockDefaultEncodingContext implements
Preconditions.checkState(ivLength <= Byte.MAX_VALUE, "IV length out of range");
cryptoByteStream.write(ivLength);
if (ivLength > 0) {
- Encryption.incrementIv(iv);
encryptor.setIv(iv);
cryptoByteStream.write(iv);
}
@@ -197,6 +196,9 @@ public class HFileBlockDefaultEncodingContext implements
onDiskBytesWithHeader = cryptoByteStream.toByteArray();
+ // Increment the IV given the final block size
+ Encryption.incrementIv(iv, 1 + (onDiskBytesWithHeader.length / encryptor.getBlockSize()));
+
} else {
cryptoByteStream.write(0);
[2/6] git commit: HBASE-11457 Increment HFile block encoding IVs
accounting for cipher's internal use
Posted by ap...@apache.org.
HBASE-11457 Increment HFile block encoding IVs accounting for cipher's internal use
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/94d3dbea
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/94d3dbea
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/94d3dbea
Branch: refs/heads/master
Commit: 94d3dbea860390bb9f1dbd3d4859f1c574a7ecd2
Parents: 2416d79
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Jul 4 10:45:32 2014 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Jul 4 10:45:32 2014 -0700
----------------------------------------------------------------------
.../hbase/io/encoding/HFileBlockDefaultEncodingContext.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/94d3dbea/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
index 0dc1c8a..7dfe857 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
@@ -187,7 +187,6 @@ public class HFileBlockDefaultEncodingContext implements
Preconditions.checkState(ivLength <= Byte.MAX_VALUE, "IV length out of range");
cryptoByteStream.write(ivLength);
if (ivLength > 0) {
- Encryption.incrementIv(iv);
encryptor.setIv(iv);
cryptoByteStream.write(iv);
}
@@ -197,6 +196,9 @@ public class HFileBlockDefaultEncodingContext implements
onDiskBytesWithHeader = cryptoByteStream.toByteArray();
+ // Increment the IV given the final block size
+ Encryption.incrementIv(iv, 1 + (onDiskBytesWithHeader.length / encryptor.getBlockSize()));
+
} else {
cryptoByteStream.write(0);