You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2018/11/06 00:37:56 UTC
zeppelin git commit: [ZEPPELIN-3825] Allow custom service account for
GCSNotebookRepo
Repository: zeppelin
Updated Branches:
refs/heads/master d3028c21d -> 81e7030f7
[ZEPPELIN-3825] Allow custom service account for GCSNotebookRepo
### What is this PR for?
The current implementation uses the default google applications service account to establish a connection with GCS. We cannot specify a custom service account for the connection with the existing implementation. This PR enables the same.
* Added ZEPPELIN_NOTEBOOK_GCS_CREDENTIALS_FILE to ConfVars
* Added functionality in GCSNotebookRepo and OldGCSNotebookRepo
to get credentials from CREDENTIALS_FILE
* Updated doc string and documentation
### What type of PR is it?
[Improvement]
### Todos
* None
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3825
### How should this be tested?
* Create a new service account that has access to write to google cloud storage.
* Disable Google Storage access to the default application service account.
* Build and deploy zeppelin after updating zeppelin.notebook.gcs.credentialsJsonFilePath in zeppelin-site.xml
* Validate that new notes are being written to GCS bucket
### Screenshots (if appropriate)
* None
### Questions:
* Does the licenses files need update? -No
* Is there breaking changes for older versions? - No
* Does this needs documentation? - Yes. Existing documentation has been updated to reflect the same.
Author: sanjaykumar <sa...@shopkick.com>
Closes #3207 from 88sanjay/ZEPPELIN-3825 and squashes the following commits:
15c8aa0fb [sanjaykumar] Merge branch 'master' into ZEPPELIN-3825
84b8da4d5 [sanjaykumar] ZEPPELIN-3825. Allow custom service account for GCSNotebookRepo
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/81e7030f
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/81e7030f
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/81e7030f
Branch: refs/heads/master
Commit: 81e7030f7f8cdf31a95dac141265dc5499ecaf4a
Parents: d3028c2
Author: sanjaykumar <sa...@shopkick.com>
Authored: Mon Nov 5 11:55:55 2018 -0500
Committer: Lee moon soo <mo...@apache.org>
Committed: Mon Nov 5 16:37:52 2018 -0800
----------------------------------------------------------------------
conf/zeppelin-site.xml.template | 9 +++++++++
docs/setup/storage/storage.md | 18 ++++++++++++++++--
.../zeppelin/conf/ZeppelinConfiguration.java | 1 +
.../zeppelin/notebook/repo/GCSNotebookRepo.java | 15 +++++++++++++--
.../notebook/repo/OldGCSNotebookRepo.java | 15 +++++++++++++--
5 files changed, 52 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81e7030f/conf/zeppelin-site.xml.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index 9d9a99f..3920fb9 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -85,6 +85,15 @@
</property>
<property>
+ <name>zeppelin.notebook.gcs.credentialsJsonFilePath</name>
+ <value>path/to/key.json</value>
+ <description>
+ Path to GCS credential key file for authentication with Google Storage.
+ </description>
+</property>
+
+
+<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.GCSNotebookRepo</value>
<description>notebook persistence layer implementation</description>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81e7030f/docs/setup/storage/storage.md
----------------------------------------------------------------------
diff --git a/docs/setup/storage/storage.md b/docs/setup/storage/storage.md
index e826460..6ef3453 100644
--- a/docs/setup/storage/storage.md
+++ b/docs/setup/storage/storage.md
@@ -317,7 +317,7 @@ Or, if you want to simultaneously use your local git storage with GCS, use the f
### Google Cloud API Authentication
Note: On Google App Engine, Google Cloud Shell, and Google Compute Engine, these
-steps are not necessary, as build-in credentials are used by default.
+steps are not necessary if you are using the default built in service account.
For more information, see [Application Default Credentials](https://cloud.google.com/docs/authentication/production)
@@ -351,11 +351,25 @@ for authentication with GCS, you will need a JSON service account key file.
`/path/to/my/key.json`), and give it appropriate permissions. Ensure at
least the user running the zeppelin daemon can read it.
-Then, point `GOOGLE_APPLICATION_CREDENTIALS` at your new key file in **zeppelin-env.sh**. For example:
+ If you wish to set this as your default credential file to access Google Services,
+ point `GOOGLE_APPLICATION_CREDENTIALS` at your new key file in **zeppelin-env.sh**. For example:
```bash
export GOOGLE_APPLICATION_CREDENTIALS=/path/to/my/key.json
```
+If you do not want to use this key file as default credential file and want to specify a custom key
+file for authentication with GCS, update the following property :
+
+```xml
+<property>
+ <name>zeppelin.notebook.google.credentialsJsonFilePath</name>
+ <value>path/to/key.json</value>
+ <description>
+ Path to GCS credential key file for authentication with Google Storage.
+ </description>
+</property>
+```
+
</br>
## Notebook Storage in ZeppelinHub <a name="ZeppelinHub"></a>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81e7030f/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index b76634a..4e2b8c3 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -738,6 +738,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
// whether homescreen notebook will be hidden from notebook list or not
ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE("zeppelin.notebook.homescreen.hide", false),
ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR("zeppelin.notebook.gcs.dir", ""),
+ ZEPPELIN_NOTEBOOK_GCS_CREDENTIALS_FILE("zeppelin.notebook.google.credentialsJsonFilePath", null),
ZEPPELIN_NOTEBOOK_S3_BUCKET("zeppelin.notebook.s3.bucket", "zeppelin"),
ZEPPELIN_NOTEBOOK_S3_ENDPOINT("zeppelin.notebook.s3.endpoint", "s3.amazonaws.com"),
ZEPPELIN_NOTEBOOK_S3_TIMEOUT("zeppelin.notebook.s3.timeout", "120000"),
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81e7030f/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java
index ee269df..e882b53 100644
--- a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java
+++ b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java
@@ -17,6 +17,8 @@
package org.apache.zeppelin.notebook.repo;
+import com.google.auth.Credentials;
+import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
@@ -29,6 +31,8 @@ import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.gson.JsonParseException;
+
+import java.io.FileInputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
@@ -54,7 +58,9 @@ import org.slf4j.LoggerFactory;
* object store, so this "directory" should not itself be an object. Instead, it represents the base
* path for the note.json files.
*
- * Authentication is provided by google-auth-library-java.
+ * Authentication is provided by google-auth-library-java. A custom json key file path
+ * can be specified by zeppelin.notebook.google.credentialsJsonFilePath to connect with GCS
+ * If not specified the GOOGLE_APPLICATION_CREDENTIALS will be used to connect to GCS.
* @see <a href="https://github.com/google/google-auth-library-java">
* google-auth-library-java</a>.
*/
@@ -113,7 +119,12 @@ public class GCSNotebookRepo implements NotebookRepo {
this.notePathPattern = Pattern.compile("^(.+\\.zpln)$");
}
- this.storage = StorageOptions.getDefaultInstance().getService();
+ Credentials credentials = GoogleCredentials.getApplicationDefault();
+ String credentialJsonPath = zConf.getString(ConfVars.ZEPPELIN_NOTEBOOK_GCS_CREDENTIALS_FILE);
+ if (credentialJsonPath != null) {
+ credentials = GoogleCredentials.fromStream(new FileInputStream(credentialJsonPath));
+ }
+ this.storage = StorageOptions.newBuilder().setCredentials(credentials).build().getService();
}
private BlobId makeBlobId(String noteId, String notePath) throws IOException {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81e7030f/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/OldGCSNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/OldGCSNotebookRepo.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/OldGCSNotebookRepo.java
index a0851e6..4394676 100644
--- a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/OldGCSNotebookRepo.java
+++ b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/OldGCSNotebookRepo.java
@@ -17,6 +17,8 @@
package org.apache.zeppelin.notebook.repo;
+import com.google.auth.Credentials;
+import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
@@ -39,6 +41,7 @@ import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -56,7 +59,9 @@ import java.util.regex.Pattern;
* object store, so this "directory" should not itself be an object. Instead, it represents the base
* path for the note.json files.
*
- * Authentication is provided by google-auth-library-java.
+ * Authentication is provided by google-auth-library-java. A custom json key file path
+ * can be specified by zeppelin.notebook.google.credentialsJsonFilePath to connect with GCS
+ * If not specified the GOOGLE_APPLICATION_CREDENTIALS will be used to connect to GCS.
* @see <a href="https://github.com/google/google-auth-library-java">
* google-auth-library-java</a>.
*/
@@ -115,7 +120,13 @@ public class OldGCSNotebookRepo implements OldNotebookRepo {
this.noteNamePattern = Pattern.compile("^([^/]+)/note\\.json$");
}
- this.storage = StorageOptions.getDefaultInstance().getService();
+
+ Credentials credentials = GoogleCredentials.getApplicationDefault();
+ String credentialJsonPath = zConf.getString(ConfVars.ZEPPELIN_NOTEBOOK_GCS_CREDENTIALS_FILE);
+ if (credentialJsonPath != null) {
+ credentials = GoogleCredentials.fromStream(new FileInputStream(credentialJsonPath));
+ }
+ this.storage = StorageOptions.newBuilder().setCredentials(credentials).build().getService();
}
private BlobId makeBlobId(String noteId) {