You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2017/02/05 06:55:17 UTC
zeppelin git commit: [ZEPPELIN-1465] Add an option to allow S3
server-side encryption
Repository: zeppelin
Updated Branches:
refs/heads/master e9caebcfa -> 5bb38c89a
[ZEPPELIN-1465] Add an option to allow S3 server-side encryption
### What is this PR for?
Provide a configuration option that will cause the S3 Notebook repo to request server-side encryption of saved notebooks.
### What type of PR is it?
Improvement
### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-1465
### How should this be tested?
Enable the configuration option, save a notebook in zeppelin, and confirm in the AWS S3 Console that the related file was saved with AES-256 encryption on the server-side. (Properties tab, Detail section)
### Questions:
* Does the licenses files need update?
No
* Is there breaking changes for older versions?
No.
* Does this needs documentation?
I added mentions of the new option in existing documentation.
Thank you!
Author: Jeff Plourde <jp...@cyft.io>
Closes #1969 from jeff-cyft/s3_sse and squashes the following commits:
26f5264 [Jeff Plourde] code style - remove tab
3c657ac [Jeff Plourde] Configuration option to request S3 SSE when notebooks are saved.
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/5bb38c89
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/5bb38c89
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/5bb38c89
Branch: refs/heads/master
Commit: 5bb38c89ae67f95858547f73d0e833ef91b3d6ee
Parents: e9caebc
Author: Jeff Plourde <jp...@cyft.io>
Authored: Thu Feb 2 11:09:57 2017 -0500
Committer: Lee moon soo <mo...@apache.org>
Committed: Sun Feb 5 15:55:08 2017 +0900
----------------------------------------------------------------------
conf/zeppelin-env.cmd.template | 1 +
conf/zeppelin-env.sh.template | 1 +
conf/zeppelin-site.xml.template | 8 ++++++++
docs/install/configuration.md | 6 ++++++
docs/storage/storage.md | 18 ++++++++++++++++++
.../zeppelin/conf/ZeppelinConfiguration.java | 5 +++++
.../zeppelin/notebook/repo/S3NotebookRepo.java | 15 ++++++++++++++-
7 files changed, 53 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/conf/zeppelin-env.cmd.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template
index 5fc3acf..1bbde86 100644
--- a/conf/zeppelin-env.cmd.template
+++ b/conf/zeppelin-env.cmd.template
@@ -34,6 +34,7 @@ REM set ZEPPELIN_NOTEBOOK_S3_USER REM User in bucket where notebook
REM set ZEPPELIN_NOTEBOOK_S3_ENDPOINT REM Endpoint of the bucket
REM set ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID REM AWS KMS key ID
REM set ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION REM AWS KMS key region
+REM set ZEPPELIN_NOTEBOOK_S3_SSE REM Server-side encryption enabled for notebooks
REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default.
REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0.
REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/conf/zeppelin-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template
index 7e777b6..46fd481 100644
--- a/conf/zeppelin-env.sh.template
+++ b/conf/zeppelin-env.sh.template
@@ -35,6 +35,7 @@
# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID # AWS KMS key ID
# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION # AWS KMS key region
+# export ZEPPELIN_NOTEBOOK_S3_SSE # Server-side encryption enabled for notebooks
# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default.
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/conf/zeppelin-site.xml.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index bd8d7dd..abaff30 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -129,6 +129,14 @@
</property>
-->
+<!-- Server-side encryption enabled for notebooks -->
+<!--
+<property>
+ <name>zeppelin.notebook.s3.sse</name>
+ <value>true</value>
+ <description>Server-side encryption enabled for notebooks</description>
+</property>
+-->
<!-- If using Azure for storage use the following settings -->
<!--
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/docs/install/configuration.md
----------------------------------------------------------------------
diff --git a/docs/install/configuration.md b/docs/install/configuration.md
index befb520..a8ebf54 100644
--- a/docs/install/configuration.md
+++ b/docs/install/configuration.md
@@ -195,6 +195,12 @@ If both are defined, then the **environment variables** will take priority.
<td>Class name of a custom S3 encryption materials provider implementation to use for encrypting data in S3 (optional)</td>
</tr>
<tr>
+ <td>ZEPPELIN_NOTEBOOK_S3_SSE</td>
+ <td>zeppelin.notebook.s3.sse</td>
+ <td>false</td>
+ <td>Save notebooks to S3 with server-side encryption enabled</td>
+ </tr>
+ <tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</td>
<td>zeppelin.notebook.azure.connectionString</td>
<td></td>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/docs/storage/storage.md
----------------------------------------------------------------------
diff --git a/docs/storage/storage.md b/docs/storage/storage.md
index 0ab01da..73388da 100644
--- a/docs/storage/storage.md
+++ b/docs/storage/storage.md
@@ -165,6 +165,24 @@ Or using the following setting in **zeppelin-site.xml**:
<description>Custom encryption materials provider used to encrypt notebook data in S3</description>
```
+#### Enable server-side encryption
+
+To request server-side encryption of notebooks, set the following environment variable in the file **zeppelin-env.sh**:
+
+```
+export ZEPPELIN_NOTEBOOK_S3_SSE = true
+```
+
+Or using the following setting in **zeppelin-site.xml**:
+
+```
+<property>
+ <name>zeppelin.notebook.s3.sse</name>
+ <value>true</value>
+ <description>Server-side encryption enabled for notebooks</description>
+</property>
+```
+
</br>
## Notebook Storage in Azure <a name="Azure"></a>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 2c8d91c..0708719 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -380,6 +380,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP);
}
+ public boolean isS3ServerSideEncryption() {
+ return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_S3_SSE);
+ }
+
public String getInterpreterListPath() {
return getRelativeDir(String.format("%s/interpreter-list", getConfDir()));
}
@@ -587,6 +591,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_S3_EMP("zeppelin.notebook.s3.encryptionMaterialsProvider", null),
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID("zeppelin.notebook.s3.kmsKeyID", null),
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null),
+ ZEPPELIN_NOTEBOOK_S3_SSE("zeppelin.notebook.s3.sse", false),
ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null),
ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"),
ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
index 26781b8..bd7fe1a 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
@@ -55,6 +55,7 @@ import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
@@ -86,12 +87,14 @@ public class S3NotebookRepo implements NotebookRepo {
private final AmazonS3 s3client;
private final String bucketName;
private final String user;
+ private final boolean useServerSideEncryption;
private final ZeppelinConfiguration conf;
public S3NotebookRepo(ZeppelinConfiguration conf) throws IOException {
this.conf = conf;
bucketName = conf.getBucketName();
user = conf.getUser();
+ useServerSideEncryption = conf.isS3ServerSideEncryption();
// always use the default provider chain
AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
@@ -234,7 +237,17 @@ public class S3NotebookRepo implements NotebookRepo {
Writer writer = new OutputStreamWriter(new FileOutputStream(file));
writer.write(json);
writer.close();
- s3client.putObject(new PutObjectRequest(bucketName, key, file));
+
+ PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file);
+
+ if (useServerSideEncryption) {
+ // Request server-side encryption.
+ ObjectMetadata objectMetadata = new ObjectMetadata();
+ objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
+ putRequest.setMetadata(objectMetadata);
+ }
+
+ s3client.putObject(putRequest);
}
catch (AmazonClientException ace) {
throw new IOException("Unable to store note in S3: " + ace, ace);