You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2016/02/20 05:02:06 UTC
[1/2] accumulo git commit: ACCUMULO-4141 Prevent unnecessary reads of
rfile header
Repository: accumulo
Updated Branches:
refs/heads/1.7 4678cf928 -> 5093ae347
ACCUMULO-4141 Prevent unnecessary reads of rfile header
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b033b04f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b033b04f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b033b04f
Branch: refs/heads/1.7
Commit: b033b04fd01ebbc991944cf51fa748aa9456d140
Parents: eb0f9b4
Author: Keith Turner <ke...@deenlo.com>
Authored: Thu Feb 18 12:31:44 2016 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Fri Feb 19 22:04:28 2016 -0500
----------------------------------------------------------------------
.../accumulo/core/file/rfile/bcfile/BCFile.java | 68 +++++++++++++++-----
1 file changed, 51 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b033b04f/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index ecc0b90..4767d91 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -28,6 +28,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
@@ -54,6 +55,8 @@ import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.compress.Compressor;
import org.apache.hadoop.io.compress.Decompressor;
+import com.google.common.base.Charsets;
+
/**
* Block Compressed file, the underlying physical storage layer for TFile. BCFile provides the basic block level compression for the data block and meta blocks.
* It is separated from TFile as it may be used for other block-compressed file implementation.
@@ -521,6 +524,30 @@ public final class BCFile {
}
}
+ // sha256 of some random data
+ private static final byte[] NO_CPYPTO_KEY = "ce18cf53c4c5077f771249b38033fa14bcb31cca0e5e95a371ee72daa8342ea2".getBytes(Charsets.UTF_8);
+
+ // This class is used as a place holder in the cache for RFiles that have no crypto....
+ private static final BCFileCryptoModuleParameters NO_CRYPTO = new BCFileCryptoModuleParameters() {
+
+ @Override
+ public Map<String,String> getAllOptions() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public byte[] getEncryptedKey() {
+ return NO_CPYPTO_KEY;
+ }
+
+ @Override
+ public String getOpaqueKeyEncryptionKeyID() {
+ // NONE + sha256 of random data
+ return "NONE:a4007e6aefb095a5a47030cd6c850818fb3a685dc6e85ba1ecc5a44ba68b193b";
+ }
+
+ };
+
private static class BCFileCryptoModuleParameters extends CryptoModuleParameters {
public void write(DataOutput out) throws IOException {
@@ -888,17 +915,14 @@ public final class BCFile {
cryptoParams = (BCFileCryptoModuleParameters) secretKeyEncryptionStrategy.decryptSecretKey(cryptoParams);
} else if (cachedCryptoParams != null) {
- cryptoParams = new BCFileCryptoModuleParameters();
- cryptoParams.read(cachedCryptoParams);
-
- this.cryptoModule = CryptoModuleFactory.getCryptoModule(cryptoParams.getAllOptions().get(Property.CRYPTO_MODULE_CLASS.getKey()));
- this.secretKeyEncryptionStrategy = CryptoModuleFactory.getSecretKeyEncryptionStrategy(cryptoParams.getKeyEncryptionStrategyClass());
-
- // This call should put the decrypted session key within the cryptoParameters object
- // secretKeyEncryptionStrategy.decryptSecretKey(cryptoParameters);
-
- cryptoParams = (BCFileCryptoModuleParameters) secretKeyEncryptionStrategy.decryptSecretKey(cryptoParams);
-
+ setupCryptoFromCachedData(cachedCryptoParams);
+ } else {
+ // Place something in cache that indicates this file has no crypto metadata. See ACCUMULO-4141
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ NO_CRYPTO.write(dos);
+ dos.close();
+ cache.cacheMetaBlock(CRYPTO_BLOCK_NAME, baos.toByteArray());
}
if (cachedMetaIndex == null) {
@@ -931,15 +955,25 @@ public final class BCFile {
metaIndex = new MetaIndex(cachedMetaIndex);
dataIndex = new DataIndex(cachedDataIndex);
- cryptoParams = new BCFileCryptoModuleParameters();
- cryptoParams.read(cachedCryptoParams);
+ setupCryptoFromCachedData(cachedCryptoParams);
+ }
+ }
- this.cryptoModule = CryptoModuleFactory.getCryptoModule(cryptoParams.getAllOptions().get(Property.CRYPTO_MODULE_CLASS.getKey()));
- this.secretKeyEncryptionStrategy = CryptoModuleFactory.getSecretKeyEncryptionStrategy(cryptoParams.getKeyEncryptionStrategyClass());
+ private void setupCryptoFromCachedData(BlockRead cachedCryptoParams) throws IOException {
+ BCFileCryptoModuleParameters params = new BCFileCryptoModuleParameters();
+ params.read(cachedCryptoParams);
- // This call should put the decrypted session key within the cryptoParameters object
- cryptoParams = (BCFileCryptoModuleParameters) secretKeyEncryptionStrategy.decryptSecretKey(cryptoParams);
+ if (Arrays.equals(params.getEncryptedKey(), NO_CRYPTO.getEncryptedKey())
+ && NO_CRYPTO.getOpaqueKeyEncryptionKeyID().equals(params.getOpaqueKeyEncryptionKeyID())) {
+ this.cryptoParams = null;
+ this.cryptoModule = null;
+ this.secretKeyEncryptionStrategy = null;
+ } else {
+ this.cryptoModule = CryptoModuleFactory.getCryptoModule(params.getAllOptions().get(Property.CRYPTO_MODULE_CLASS.getKey()));
+ this.secretKeyEncryptionStrategy = CryptoModuleFactory.getSecretKeyEncryptionStrategy(params.getKeyEncryptionStrategyClass());
+ // This call should put the decrypted session key within the cryptoParameters object
+ cryptoParams = (BCFileCryptoModuleParameters) secretKeyEncryptionStrategy.decryptSecretKey(params);
}
}
[2/2] accumulo git commit: Merge branch '1.6' into 1.7
Posted by kt...@apache.org.
Merge branch '1.6' into 1.7
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5093ae34
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5093ae34
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5093ae34
Branch: refs/heads/1.7
Commit: 5093ae347eb9e37d4fe4d172537f7ee17f777595
Parents: 4678cf9 b033b04
Author: Keith Turner <kt...@apache.org>
Authored: Fri Feb 19 22:44:52 2016 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Fri Feb 19 22:44:52 2016 -0500
----------------------------------------------------------------------
.../accumulo/core/file/rfile/bcfile/BCFile.java | 67 +++++++++++++++-----
1 file changed, 50 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5093ae34/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 91d61e1,4767d91..3764603
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@@ -26,8 -26,9 +26,10 @@@ import java.io.DataOutputStream
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
++import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
+ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
@@@ -519,6 -524,30 +521,30 @@@ public final class BCFile
}
}
+ // sha256 of some random data
- private static final byte[] NO_CPYPTO_KEY = "ce18cf53c4c5077f771249b38033fa14bcb31cca0e5e95a371ee72daa8342ea2".getBytes(Charsets.UTF_8);
++ private static final byte[] NO_CPYPTO_KEY = "ce18cf53c4c5077f771249b38033fa14bcb31cca0e5e95a371ee72daa8342ea2".getBytes(StandardCharsets.UTF_8);
+
+ // This class is used as a place holder in the cache for RFiles that have no crypto....
+ private static final BCFileCryptoModuleParameters NO_CRYPTO = new BCFileCryptoModuleParameters() {
+
+ @Override
+ public Map<String,String> getAllOptions() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public byte[] getEncryptedKey() {
+ return NO_CPYPTO_KEY;
+ }
+
+ @Override
+ public String getOpaqueKeyEncryptionKeyID() {
+ // NONE + sha256 of random data
+ return "NONE:a4007e6aefb095a5a47030cd6c850818fb3a685dc6e85ba1ecc5a44ba68b193b";
+ }
+
+ };
+
private static class BCFileCryptoModuleParameters extends CryptoModuleParameters {
public void write(DataOutput out) throws IOException {