You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2015/08/10 22:19:48 UTC
incubator-zeppelin git commit: Allow instance profile authentication
with S3
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master cf9541f8d -> addc12866
Allow instance profile authentication with S3
This PR generalizes authentication with S3 access (for storing notebooks) a bit. Before the only way to authenticate was to set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables. This change uses DefaultAWSCredentialsProviderChain for authentication, which allows instance profiles on EC2 instances for authentication with S3.
Author: Corey Huang <co...@gmail.com>
Closes #184 from cdfhuang/s3_instance_profiles and squashes the following commits:
237eab2 [Corey Huang] Use credential provider directly to avoid AWS token expiration with instance profiles
2fb5de0 [Corey Huang] Fix comment error
d0a0b03 [Corey Huang] Allow instance profile authentication with S3
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/addc1286
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/addc1286
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/addc1286
Branch: refs/heads/master
Commit: addc12866d57a2b58d2b7638d83e53f5cc14029d
Parents: cf9541f
Author: Corey Huang <co...@gmail.com>
Authored: Fri Aug 7 18:14:04 2015 +0000
Committer: Lee moon soo <mo...@apache.org>
Committed: Mon Aug 10 13:19:44 2015 -0700
----------------------------------------------------------------------
.../org/apache/zeppelin/conf/Credentials.java | 43 --------------------
.../zeppelin/notebook/repo/S3NotebookRepo.java | 26 +++++++++---
2 files changed, 21 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/addc1286/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java
deleted file mode 100644
index 87248a6..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.zeppelin.conf;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
-
-/**
- *
- * @author vgmartinez
- *
- */
-public class Credentials {
- static String aws_access_key_id = System.getenv("AWS_ACCESS_KEY_ID");
- static String aws_secret_access_key = System.getenv("AWS_SECRET_ACCESS_KEY");
-
- private static AWSCredentials credentials = new BasicAWSCredentials(aws_access_key_id,
- aws_secret_access_key);
-
- public AWSCredentials getCredentials() {
- return credentials;
- }
-
- public static void setCredentials(AWSCredentials credentials) {
- Credentials.credentials = credentials;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/addc1286/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
index 0b90262..bb9e5d1 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
@@ -27,7 +27,6 @@ import java.util.LinkedList;
import java.util.List;
import org.apache.commons.io.IOUtils;
-import org.apache.zeppelin.conf.Credentials;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.notebook.Note;
@@ -39,6 +38,8 @@ import org.slf4j.LoggerFactory;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
+import com.amazonaws.auth.AWSCredentialsProviderChain;
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.GetObjectRequest;
@@ -55,14 +56,29 @@ import com.google.gson.GsonBuilder;
* @author vgmartinez
*
*/
-public class S3NotebookRepo implements NotebookRepo{
+public class S3NotebookRepo implements NotebookRepo {
Logger logger = LoggerFactory.getLogger(S3NotebookRepo.class);
- Credentials aws = new Credentials();
+
+ // Use a credential provider chain so that instance profiles can be utilized
+ // on an EC2 instance. The order of locations where credentials are searched
+ // is documented here
+ //
+ // http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/
+ // auth/DefaultAWSCredentialsProviderChain.html
+ //
+ // In summary, the order is:
+ //
+ // 1. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+ // 2. Java System Properties - aws.accessKeyId and aws.secretKey
+ // 3. Credential profiles file at the default location (~/.aws/credentials)
+ // shared by all AWS SDKs and the AWS CLI
+ // 4. Instance profile credentials delivered through the Amazon EC2 metadata service
+ private AmazonS3 s3client = new AmazonS3Client(new DefaultAWSCredentialsProviderChain());
+
private static String bucketName = "";
- String user = "";
+ private String user = "";
- AmazonS3 s3client = new AmazonS3Client(aws.getCredentials());
private ZeppelinConfiguration conf;