You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2017/01/10 18:35:59 UTC
zeppelin git commit: [ZEPPELIN-1848] add option for S3 KMS key region
Repository: zeppelin
Updated Branches:
refs/heads/master a89cb1047 -> 31085cc03
[ZEPPELIN-1848] add option for S3 KMS key region
### What is this PR for?
When using S3 storage layer with encryption keys, currently only keys created in `us-east-1` region can be used. This PR adds ability to set target region for AWS KMS keys.
### What type of PR is it?
Improvement
### Todos
* [x] - add region to awsClient
* [x] - add conf for region
* [x] - tested with aws account `us-west-2` region
### What is the Jira issue?
[ZEPPELIN-1848](https://issues.apache.org/jira/browse/ZEPPELIN-1848)
### How should this be tested?
1. set up S3 storage as in [here](https://zeppelin.apache.org/docs/0.7.0-SNAPSHOT/storage/storage.html#notebook-storage-in-s3)
2. add region variable with `export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION="us-west-2"` in `conf/zeppelin-env.sh`
3. start Zeppelin and read/write S3
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? updated
Author: Khalid Huseynov <kh...@gmail.com>
Closes #1860 from khalidhuseynov/feat/s3-repo-kms-region and squashes the following commits:
712025f [Khalid Huseynov] add missing vars to .cmd conf
35c015a [Khalid Huseynov] align # in .sh conf
40ae2f1 [Khalid Huseynov] refactor and keep backward compatibility
303f16d [Khalid Huseynov] add documentation
929d401 [Khalid Huseynov] add property to .site
d5808cd [Khalid Huseynov] add env vars to .sh
3110193 [Khalid Huseynov] add crypt conf to s3 repo
da14298 [Khalid Huseynov] add property to ZeppelinConfiguration
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/31085cc0
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/31085cc0
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/31085cc0
Branch: refs/heads/master
Commit: 31085cc03ee0595ab4ee05195bc477be3b6a2a88
Parents: a89cb10
Author: Khalid Huseynov <kh...@gmail.com>
Authored: Fri Jan 6 20:51:47 2017 -0800
Committer: Lee moon soo <mo...@apache.org>
Committed: Tue Jan 10 10:35:55 2017 -0800
----------------------------------------------------------------------
conf/zeppelin-env.cmd.template | 3 +++
conf/zeppelin-env.sh.template | 2 ++
conf/zeppelin-site.xml.template | 10 ++++++++++
docs/storage/storage.md | 19 ++++++++++++++++++-
.../zeppelin/conf/ZeppelinConfiguration.java | 5 +++++
.../zeppelin/notebook/repo/S3NotebookRepo.java | 17 ++++++++++++++++-
6 files changed, 54 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/31085cc0/conf/zeppelin-env.cmd.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template
index cb7f87c..5fc3acf 100644
--- a/conf/zeppelin-env.cmd.template
+++ b/conf/zeppelin-env.cmd.template
@@ -31,6 +31,9 @@ REM set ZEPPELIN_NOTEBOOK_HOMESCREEN REM Id of notebook to be displayed in home
REM set ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE REM hide homescreen notebook from list when this value set to "true". default "false"
REM set ZEPPELIN_NOTEBOOK_S3_BUCKET REM Bucket where notebook saved
REM set ZEPPELIN_NOTEBOOK_S3_USER REM User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
+REM set ZEPPELIN_NOTEBOOK_S3_ENDPOINT REM Endpoint of the bucket
+REM set ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID REM AWS KMS key ID
+REM set ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION REM AWS KMS key region
REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default.
REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0.
REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/31085cc0/conf/zeppelin-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template
index cc0a83e..64db29d 100644
--- a/conf/zeppelin-env.sh.template
+++ b/conf/zeppelin-env.sh.template
@@ -33,6 +33,8 @@
# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved
# export ZEPPELIN_NOTEBOOK_S3_ENDPOINT # Endpoint of the bucket
# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
+# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID # AWS KMS key ID
+# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION # AWS KMS key region
# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default.
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/31085cc0/conf/zeppelin-site.xml.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index beaebc5..7faacac 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -108,6 +108,16 @@
</property>
-->
+<!-- provide region of your KMS key -->
+<!-- See http://docs.aws.amazon.com/general/latest/gr/rande.html#kms_region for region codes names -->
+<!--
+<property>
+ <name>zeppelin.notebook.s3.kmsKeyRegion</name>
+ <value>us-east-1</value>
+ <description>AWS KMS key region in your AWS account</description>
+</property>
+-->
+
<!-- Use a custom encryption materials provider to encrypt data -->
<!-- No configuration is given to the provider, so you must use system properties or another means to configure -->
<!-- See https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/EncryptionMaterialsProvider.html -->
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/31085cc0/docs/storage/storage.md
----------------------------------------------------------------------
diff --git a/docs/storage/storage.md b/docs/storage/storage.md
index 76012ff..20c6312 100644
--- a/docs/storage/storage.md
+++ b/docs/storage/storage.md
@@ -130,6 +130,23 @@ Or using the following setting in **zeppelin-site.xml**:
</property>
```
+In order to set custom KMS key region, set the following environment variable in the file **zeppelin-env.sh**:
+
+```
+export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION = kms-key-region
+```
+
+Or using the following setting in **zeppelin-site.xml**:
+
+```
+<property>
+ <name>zeppelin.notebook.s3.kmsKeyRegion</name>
+ <value>target-region</value>
+ <description>AWS KMS key region in your AWS account</description>
+</property>
+```
+Format of `target-region` is described in more details [here](http://docs.aws.amazon.com/general/latest/gr/rande.html#kms_region) in second `Region` column (e.g. `us-east-1`).
+
#### Custom Encryption Materials Provider class
You may use a custom [``EncryptionMaterialsProvider``](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/EncryptionMaterialsProvider.html) class as long as it is available in the classpath and able to initialize itself from system properties or another mechanism. To use this, set the following environment variable in the file **zeppelin-env.sh**:
@@ -238,4 +255,4 @@ export ZEPPELINHUB_API_TOKEN = ZeppelinHub token
export ZEPPELINHUB_API_ADDRESS = address of ZeppelinHub service (e.g. https://www.zeppelinhub.com)
```
-You can get more information on generating `token` and using authentication on the corresponding [help page](http://help.zeppelinhub.com/zeppelin_integration/#add-a-new-zeppelin-instance-and-generate-a-token).
\ No newline at end of file
+You can get more information on generating `token` and using authentication on the corresponding [help page](http://help.zeppelinhub.com/zeppelin_integration/#add-a-new-zeppelin-instance-and-generate-a-token).
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/31085cc0/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 9401854..0c3ecac 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -369,6 +369,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID);
}
+ public String getS3KMSKeyRegion() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION);
+ }
+
public String getS3EncryptionMaterialsProviderClass() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP);
}
@@ -579,6 +583,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_S3_USER("zeppelin.notebook.s3.user", "user"),
ZEPPELIN_NOTEBOOK_S3_EMP("zeppelin.notebook.s3.encryptionMaterialsProvider", null),
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID("zeppelin.notebook.s3.kmsKeyID", null),
+ ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null),
ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null),
ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"),
ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/31085cc0/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
index a0de433..26781b8 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
@@ -31,6 +31,7 @@ import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.notebook.Note;
@@ -48,12 +49,15 @@ import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
+import com.amazonaws.services.s3.model.CryptoConfiguration;
import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.regions.Region;
+import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.google.gson.Gson;
@@ -91,13 +95,24 @@ public class S3NotebookRepo implements NotebookRepo {
// always use the default provider chain
AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
+ CryptoConfiguration cryptoConf = null;
+ String keyRegion = conf.getS3KMSKeyRegion();
+ if (StringUtils.isNotBlank(keyRegion)) {
+ cryptoConf = new CryptoConfiguration();
+ cryptoConf.setAwsKmsRegion(Region.getRegion(Regions.fromName(keyRegion)));
+ }
+
// see if we should be encrypting data in S3
String kmsKeyID = conf.getS3KMSKeyID();
if (kmsKeyID != null) {
// use the AWS KMS to encrypt data
KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID);
- this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
+ if (cryptoConf != null) {
+ this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cryptoConf);
+ } else {
+ this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
+ }
}
else if (conf.getS3EncryptionMaterialsProviderClass() != null) {
// use a custom encryption materials provider class