You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2020/06/23 03:13:27 UTC
[zeppelin] branch master updated: [ZEPPELIN-4896] Add S3 Canned ACL
Option for S3NotebookRepo
This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new 5b3b819 [ZEPPELIN-4896] Add S3 Canned ACL Option for S3NotebookRepo
5b3b819 is described below
commit 5b3b819e7a127e95ea9a03ec841264e4d169f21a
Author: Nathaniel Troutman <tr...@amazon.com>
AuthorDate: Sat Jun 20 00:08:59 2020 -0700
[ZEPPELIN-4896] Add S3 Canned ACL Option for S3NotebookRepo
### What is this PR for?
When S3NotebookRepo saves a notebook into S3 it does so without granting the bucket owner full control of the object, this means that owning AWS account cannot read the notebook, only the creating AWS account. This causes issues when a notebook bucket is shared across AWS accounts.
This PR introduces the configuration value "zeppelin.notebook.s3.cannedAcl" which
will override the default object permissions when saving a notebook to S3. This allows
granting the bucket owner full control when the writer and the bucket owner are
different AWS accounts.
### What type of PR is it?
Improvement/Feature
### Todos
* [ ] - N/A
### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-4896
### How should this be tested?
#### Unit-Tests
Unfortunately Unit-Tests are not possible as gaul.S3Proxy does not support Object ACL throwing a NotImplemented exception if you attempt to use PutRequest.withCannedAcl for any value other than public or private (see https://github.com/gaul/s3proxy#limitations).
#### Manual Testing
1. Have two AWS accounts, "account-a" and "account-b"
2. Create a bucket "notebook-bucket" in "account-a" and grant "account-b" permissions to write to it
3. Setup zeppelin-site.xml for S3NotebookRepor WITHOUT Canned ACL Feature and AWS Credentials for "account-b"
4. Launch zeppelin, create a new note titled 'WithoutACL', and shutdown zeppelin
5. Verify that the permissions on the S3 Object do NOT grant the bucket owner "account-a" permissions
```
aws s3api get-object-acl --bucket notebook-bucket --key test-user/notebook/WithoutACL_2FD4NFYTU.zpln
{
"Owner": {
"DisplayName": "account-b",
"ID": "1e9...e4"
},
"Grants": [
{
"Grantee": {
"DisplayName": "account-b",
"ID": "1e9...e4",
"Type": "CanonicalUser"
},
"Permission": "FULL_CONTROL"
}
]
}
```
6. Edit zeppelin-site.xml enabling "zeppelin.notebook.s3.cannedAcl" as "BucketOwnerFullControl"
7. Launch zeppelin, create a new note titled "WithACL", and shutdown zeppelin
8. Verify that the permissions on the S3 object DO GRANT the bucket owner, "account-a", full control
```
aws s3api get-object-acl --bucket notebook-bucket --key test-user/notebook/WithACL_2FCXTUS3M.zpln
{
"Owner": {
"DisplayName": "account-b",
"ID": "1e9...e4"
},
"Grants": [
{
"Grantee": {
"DisplayName": "account-b",
"ID": "1e9...e4",
"Type": "CanonicalUser"
},
"Permission": "FULL_CONTROL"
},
{
"Grantee": {
"DisplayName": "account-a",
"ID": "f60...ee",
"Type": "CanonicalUser"
},
"Permission": "FULL_CONTROL"
}
]
}
```
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this need documentation? Yes, added relevant documentation along side existing S3 Notebook Repo documentation.
Author: Nathaniel Troutman <tr...@amazon.com>
Closes #3811 from ntroutman/canned-acl and squashes the following commits:
6ebe608a1 [Nathaniel Troutman] [ZEPPELIN-4896] Add S3 Canned ACL Option for S3NotebookRepo
---
conf/zeppelin-site.xml.template | 9 +++++++++
docs/setup/operation/configuration.md | 6 ++++++
docs/setup/storage/storage.md | 21 +++++++++++++++++++++
.../apache/zeppelin/conf/ZeppelinConfiguration.java | 5 +++++
.../zeppelin/notebook/repo/OldS3NotebookRepo.java | 9 ++++++++-
.../zeppelin/notebook/repo/S3NotebookRepo.java | 8 ++++++++
.../src/interfaces/message-common.interface.ts | 1 +
7 files changed, 58 insertions(+), 1 deletion(-)
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index d355003..a40b9ec 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -183,6 +183,15 @@
</property>
-->
+<!-- S3 Object Permissions (Canned ACL) for notebooks -->
+<!--
+<property>
+ <name>zeppelin.notebook.s3.cannedAcl</name>
+ <value>BucketOwnerFullControl</value>
+ <description>Saves notebooks in S3 with the given Canned Access Control List.</description>
+</property>
+-->
+
<!-- Optional override to control which signature algorithm should be used to sign AWS requests -->
<!-- Set this property to "S3SignerType" if your AWS S3 compatible APIs support only AWS Signature Version 2 such as Ceph. -->
<!--
diff --git a/docs/setup/operation/configuration.md b/docs/setup/operation/configuration.md
index 3c509b0..2a8b811 100644
--- a/docs/setup/operation/configuration.md
+++ b/docs/setup/operation/configuration.md
@@ -264,6 +264,12 @@ If both are defined, then the **environment variables** will take priority.
<td>Save notebooks to S3 with server-side encryption enabled</td>
</tr>
<tr>
+ <td><h6 class="properties">ZEPPELIN_NOTEBOOK_S3_CANNED_ACL</h6></td>
+ <td><h6 class="properties">zeppelin.notebook.s3.cannedAcl</h6></td>
+ <td></td>
+ <td>Save notebooks to S3 with the given [Canned ACL](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/CannedAccessControlList.html) which determines the S3 permissions.</td>
+ </tr>
+ <tr>
<td><h6 class="properties">ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE</h6></td>
<td><h6 class="properties">zeppelin.notebook.s3.signerOverride</h6></td>
<td></td>
diff --git a/docs/setup/storage/storage.md b/docs/setup/storage/storage.md
index c1e0997..7867990 100644
--- a/docs/setup/storage/storage.md
+++ b/docs/setup/storage/storage.md
@@ -206,6 +206,27 @@ Or using the following setting in **zeppelin-site.xml**:
</br>
+### S3 Object Permissions
+
+S3 allows writing objects into buckets owned by a different account than the requestor, when this happens S3 by default does not grant the bucket owner permissions to the written object. Setting the Canned ACL when communicating with S3 determines the permissions of notebooks saved in S3. Allowed values for Canned ACL are found [here](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/CannedAccessControlList.html), the most frequent value is "BucketOwne [...]
+
+
+```bash
+export ZEPPELIN_NOTEBOOK_S3_CANNED_ACL=BucketOwnerFullControl
+```
+
+Or using the following setting in **zeppelin-site.xml**:
+
+```xml
+<property>
+ <name>zeppelin.notebook.s3.cannedAcl</name>
+ <value>BucketOwnerFullControl</value>
+ <description>Saves notebooks in S3 with the given Canned Access Control List.</description>
+</property>
+```
+
+</br>
+
## Notebook Storage in Azure <a name="Azure"></a>
Using `AzureNotebookRepo` you can connect your Zeppelin with your Azure account for notebook storage.
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 5637e1c..dcb8619 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -492,6 +492,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE);
}
+ public String getS3CannedAcl() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_CANNED_ACL);
+ }
+
public String getOSSBucketName() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_OSS_BUCKET);
}
@@ -943,6 +947,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null),
ZEPPELIN_NOTEBOOK_S3_SSE("zeppelin.notebook.s3.sse", false),
ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE("zeppelin.notebook.s3.signerOverride", null),
+ ZEPPELIN_NOTEBOOK_S3_CANNED_ACL("zeppelin.notebook.s3.cannedAcl", null),
ZEPPELIN_NOTEBOOK_OSS_BUCKET("zeppelin.notebook.oss.bucket", "zeppelin"),
ZEPPELIN_NOTEBOOK_OSS_ENDPOINT("zeppelin.notebook.oss.endpoint", "http://oss-cn-hangzhou.aliyuncs.com"),
ZEPPELIN_NOTEBOOK_OSS_ACCESSKEYID("zeppelin.notebook.oss.accesskeyid", null),
diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/OldS3NotebookRepo.java b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/OldS3NotebookRepo.java
index 5e4b653..42bcc6e 100644
--- a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/OldS3NotebookRepo.java
+++ b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/OldS3NotebookRepo.java
@@ -27,6 +27,7 @@ import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
+import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CryptoConfiguration;
import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.GetObjectRequest;
@@ -84,6 +85,7 @@ public class OldS3NotebookRepo implements OldNotebookRepo {
private String bucketName;
private String user;
private boolean useServerSideEncryption;
+ private CannedAccessControlList objectCannedAcl;
private ZeppelinConfiguration conf;
public OldS3NotebookRepo() {
@@ -95,6 +97,9 @@ public class OldS3NotebookRepo implements OldNotebookRepo {
bucketName = conf.getS3BucketName();
user = conf.getS3User();
useServerSideEncryption = conf.isS3ServerSideEncryption();
+ if (StringUtils.isNotBlank(conf.getS3CannedAcl())) {
+ objectCannedAcl = CannedAccessControlList.valueOf(conf.getS3CannedAcl());
+ }
// always use the default provider chain
AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
@@ -244,7 +249,9 @@ public class OldS3NotebookRepo implements OldNotebookRepo {
objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
putRequest.setMetadata(objectMetadata);
}
-
+ if (objectCannedAcl != null) {
+ putRequest.withCannedAcl(objectCannedAcl);
+ }
s3client.putObject(putRequest);
}
catch (AmazonClientException ace) {
diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
index 8df3149..d936b79 100644
--- a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
+++ b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
@@ -47,6 +47,7 @@ import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
+import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CryptoConfiguration;
import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.GetObjectRequest;
@@ -84,6 +85,7 @@ public class S3NotebookRepo implements NotebookRepo {
private String bucketName;
private String user;
private boolean useServerSideEncryption;
+ private CannedAccessControlList objectCannedAcl;
private ZeppelinConfiguration conf;
private String rootFolder;
@@ -97,6 +99,9 @@ public class S3NotebookRepo implements NotebookRepo {
user = conf.getS3User();
rootFolder = user + "/notebook";
useServerSideEncryption = conf.isS3ServerSideEncryption();
+ if (StringUtils.isNotBlank(conf.getS3CannedAcl())) {
+ objectCannedAcl = CannedAccessControlList.valueOf(conf.getS3CannedAcl());
+ }
// always use the default provider chain
AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
@@ -237,6 +242,9 @@ public class S3NotebookRepo implements NotebookRepo {
objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
putRequest.setMetadata(objectMetadata);
}
+ if (objectCannedAcl != null) {
+ putRequest.withCannedAcl(objectCannedAcl);
+ }
s3client.putObject(putRequest);
}
catch (AmazonClientException ace) {
diff --git a/zeppelin-web-angular/projects/zeppelin-sdk/src/interfaces/message-common.interface.ts b/zeppelin-web-angular/projects/zeppelin-sdk/src/interfaces/message-common.interface.ts
index 0a5ad6d..c0873c9 100644
--- a/zeppelin-web-angular/projects/zeppelin-sdk/src/interfaces/message-common.interface.ts
+++ b/zeppelin-web-angular/projects/zeppelin-sdk/src/interfaces/message-common.interface.ts
@@ -117,6 +117,7 @@ export interface ConfigurationsInfo {
'zeppelin.interpreter.localRepo': string;
'zeppelin.notebook.collaborative.mode.enable': string;
'zeppelin.search.use.disk': string;
+ 'zeppelin.notebook.s3.cannedAcl': string;
};
}