You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2020/10/30 15:50:57 UTC
[zeppelin] branch branch-0.9 updated: [ZEPPELIN-5105] Add an option
to allow S3 path based access
This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new 3bc097d [ZEPPELIN-5105] Add an option to allow S3 path based access
3bc097d is described below
commit 3bc097d99f80c583ada3b07d21550fbefa7ee7e0
Author: jenil.shah <je...@ishisystems.com>
AuthorDate: Fri Oct 23 13:48:21 2020 +0530
[ZEPPELIN-5105] Add an option to allow S3 path based access
### What is this PR for?
Provide a configuration option that will allow to use path based access for S3 Notebook repo
### What type of PR is it?
Improvement
### Todos
* [x] - Task
### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-5105
### How should this be tested?
Start minio cluster without MINIO_DOMAIN. Configure minio s3 endpoint in zeppelin for notebook repo.Enable the configuration option, save a notebook in zeppelin, and confirm in the minio that the related file was saved.
### Questions:
Does the licenses files need update?
No
Is there breaking changes for older versions?
No.
Does this needs documentation?
I added mentions of the new option in existing documentation.
Thank you!
Author: jenil.shah <je...@ishisystems.com>
Closes #3953 from jenil-shah135/master and squashes the following commits:
b46820be3 [jenil.shah] S3 Service Bucket Access Using Path Style
(cherry picked from commit 70ea189d08d8d5d17db85201b1ecc335086d18e3)
Signed-off-by: Jeff Zhang <zj...@apache.org>
---
conf/zeppelin-env.cmd.template | 1 +
conf/zeppelin-env.sh.template | 1 +
conf/zeppelin-site.xml.template | 9 +++++++++
docs/setup/operation/configuration.md | 6 ++++++
docs/setup/storage/storage.md | 20 ++++++++++++++++++++
.../apache/zeppelin/conf/ZeppelinConfiguration.java | 5 +++++
.../zeppelin/notebook/repo/S3NotebookRepo.java | 2 ++
.../src/interfaces/message-common.interface.ts | 1 +
8 files changed, 45 insertions(+)
diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template
index 54cf8d5..83b610e 100644
--- a/conf/zeppelin-env.cmd.template
+++ b/conf/zeppelin-env.cmd.template
@@ -37,6 +37,7 @@ REM set ZEPPELIN_NOTEBOOK_S3_ENDPOINT REM Endpoint of the bucket
REM set ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID REM AWS KMS key ID
REM set ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION REM AWS KMS key region
REM set ZEPPELIN_NOTEBOOK_S3_SSE REM Server-side encryption enabled for notebooks
+REM set ZEPPELIN_NOTEBOOK_S3_PATH_STYLE_ACCESS REM Path style access for S3 bucket
REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default.
REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0.
REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading
diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template
index f266c11..7c4a38b 100644
--- a/conf/zeppelin-env.sh.template
+++ b/conf/zeppelin-env.sh.template
@@ -43,6 +43,7 @@
# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID # AWS KMS key ID
# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION # AWS KMS key region
# export ZEPPELIN_NOTEBOOK_S3_SSE # Server-side encryption enabled for notebooks
+# export ZEPPELIN_NOTEBOOK_S3_PATH_STYLE_ACCESS # Path style access for S3 bucket
# export ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR # GCS "directory" (prefix) under which notebooks are saved. E.g. gs://example-bucket/path/to/dir
# export GOOGLE_APPLICATION_CREDENTIALS # Provide a service account key file for GCS and BigQuery API calls (overrides application default credentials)
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index f435742..81b4a91 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -195,6 +195,15 @@
</property>
-->
+<!-- Path style access for S3 bucket -->
+<!--
+<property>
+ <name>zeppelin.notebook.s3.pathStyleAccess</name>
+ <value>true</value>
+ <description>Path style access for S3 bucket</description>
+</property>
+-->
+
<!-- S3 Object Permissions (Canned ACL) for notebooks -->
<!--
<property>
diff --git a/docs/setup/operation/configuration.md b/docs/setup/operation/configuration.md
index 19cfe12..4f7c734 100644
--- a/docs/setup/operation/configuration.md
+++ b/docs/setup/operation/configuration.md
@@ -270,6 +270,12 @@ If both are defined, then the **environment variables** will take priority.
<td>Save notebooks to S3 with the given [Canned ACL](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/CannedAccessControlList.html) which determines the S3 permissions.</td>
</tr>
<tr>
+ <td><h6 class="properties">ZEPPELIN_NOTEBOOK_S3_PATH_STYLE_ACCESS</h6></td>
+ <td><h6 class="properties">zeppelin.notebook.s3.pathStyleAccess</h6></td>
+ <td>false</td>
+ <td>Access S3 bucket using path style</td>
+ </tr>
+ <tr>
<td><h6 class="properties">ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE</h6></td>
<td><h6 class="properties">zeppelin.notebook.s3.signerOverride</h6></td>
<td></td>
diff --git a/docs/setup/storage/storage.md b/docs/setup/storage/storage.md
index 7867990..f53fa6b 100644
--- a/docs/setup/storage/storage.md
+++ b/docs/setup/storage/storage.md
@@ -227,6 +227,26 @@ Or using the following setting in **zeppelin-site.xml**:
</br>
+#### S3 Enable Path Style Access
+
+To request path style s3 bucket access, set the following environment variable in the file **zeppelin-env.sh**:
+
+```bash
+export ZEPPELIN_NOTEBOOK_S3_PATH_STYLE_ACCESS=true
+```
+
+Or using the following setting in **zeppelin-site.xml**:
+
+```xml
+<property>
+ <name>zeppelin.notebook.s3.pathStyleAccess</name>
+ <value>true</value>
+ <description>Path Style S3 bucket access enabled for notebook repo</description>
+</property>
+```
+
+</br>
+
## Notebook Storage in Azure <a name="Azure"></a>
Using `AzureNotebookRepo` you can connect your Zeppelin with your Azure account for notebook storage.
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 1e19f77..62466e6 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -508,6 +508,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
public String getS3SignerOverride() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE);
}
+
+ public boolean isS3PathStyleAccess() {
+ return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_S3_PATH_STYLE_ACCESS);
+ }
public String getS3CannedAcl() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_CANNED_ACL);
@@ -988,6 +992,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR("zeppelin.notebook.gcs.dir", ""),
ZEPPELIN_NOTEBOOK_GCS_CREDENTIALS_FILE("zeppelin.notebook.google.credentialsJsonFilePath", null),
ZEPPELIN_NOTEBOOK_S3_BUCKET("zeppelin.notebook.s3.bucket", "zeppelin"),
+ ZEPPELIN_NOTEBOOK_S3_PATH_STYLE_ACCESS("zeppelin.notebook.s3.pathStyleAccess", false),
ZEPPELIN_NOTEBOOK_S3_ENDPOINT("zeppelin.notebook.s3.endpoint", "s3.amazonaws.com"),
ZEPPELIN_NOTEBOOK_S3_TIMEOUT("zeppelin.notebook.s3.timeout", "120000"),
ZEPPELIN_NOTEBOOK_S3_USER("zeppelin.notebook.s3.user", "user"),
diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
index d936b79..c4905d7 100644
--- a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
+++ b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
@@ -46,6 +46,7 @@ import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.S3ClientOptions;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CryptoConfiguration;
@@ -130,6 +131,7 @@ public class S3NotebookRepo implements NotebookRepo {
// regular S3
this.s3client = new AmazonS3Client(credentialsProvider, cliConf);
}
+ s3client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(conf.isS3PathStyleAccess()).build());
// set S3 endpoint to use
s3client.setEndpoint(conf.getS3Endpoint());
diff --git a/zeppelin-web-angular/projects/zeppelin-sdk/src/interfaces/message-common.interface.ts b/zeppelin-web-angular/projects/zeppelin-sdk/src/interfaces/message-common.interface.ts
index c0873c9..b0b813d 100644
--- a/zeppelin-web-angular/projects/zeppelin-sdk/src/interfaces/message-common.interface.ts
+++ b/zeppelin-web-angular/projects/zeppelin-sdk/src/interfaces/message-common.interface.ts
@@ -118,6 +118,7 @@ export interface ConfigurationsInfo {
'zeppelin.notebook.collaborative.mode.enable': string;
'zeppelin.search.use.disk': string;
'zeppelin.notebook.s3.cannedAcl': string;
+ 'zeppelin.notebook.s3.pathStyleAccess': string;
};
}