You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by ga...@apache.org on 2016/01/14 00:19:12 UTC
[1/2] jclouds git commit: JCLOUDS-480: AWS S3 v4 signature
Repository: jclouds
Updated Branches:
refs/heads/master c20fcb8cd -> 8bddbb496
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java b/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java
index e8009e4..75dd648 100644
--- a/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java
+++ b/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignature.java
@@ -16,243 +16,11 @@
*/
package org.jclouds.s3.filters;
-import static com.google.common.base.Charsets.UTF_8;
-import static com.google.common.collect.Iterables.get;
-import static com.google.common.io.BaseEncoding.base64;
-import static com.google.common.io.ByteStreams.readBytes;
-import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AUTH_TAG;
-import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
-import static org.jclouds.crypto.Macs.asByteProcessor;
-import static org.jclouds.http.utils.Queries.queryParser;
-import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_SERVICE_PATH;
-import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
-import static org.jclouds.util.Strings2.toInputStream;
-
-import java.util.Collection;
-import java.util.Locale;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import javax.annotation.Resource;
-import javax.inject.Inject;
-import javax.inject.Named;
-import javax.inject.Provider;
-import javax.inject.Singleton;
-
-import org.jclouds.Constants;
-import org.jclouds.aws.domain.SessionCredentials;
-import org.jclouds.crypto.Crypto;
-import org.jclouds.date.TimeStamp;
-import org.jclouds.domain.Credentials;
-import org.jclouds.http.HttpException;
-import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRequestFilter;
-import org.jclouds.http.HttpUtils;
-import org.jclouds.http.internal.SignatureWire;
-import org.jclouds.logging.Logger;
-import org.jclouds.rest.RequestSigner;
-import org.jclouds.s3.util.S3Utils;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
-import com.google.common.base.Supplier;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Ordering;
-import com.google.common.collect.SortedSetMultimap;
-import com.google.common.collect.TreeMultimap;
-import com.google.common.io.ByteProcessor;
-import com.google.common.net.HttpHeaders;
/**
* Signs the S3 request.
*/
-@Singleton
-public class RequestAuthorizeSignature implements HttpRequestFilter, RequestSigner {
-
- private static final Collection<String> FIRST_HEADERS_TO_SIGN = ImmutableList.of(HttpHeaders.DATE);
-
- private static final Set<String> SIGNED_PARAMETERS = ImmutableSet.of("acl", "torrent", "logging", "location", "policy",
- "requestPayment", "versioning", "versions", "versionId", "notification", "uploadId", "uploads",
- "partNumber", "website", "response-content-type", "response-content-language", "response-expires",
- "response-cache-control", "response-content-disposition", "response-content-encoding", "delete");
-
- private final SignatureWire signatureWire;
- private final Supplier<Credentials> creds;
- private final Provider<String> timeStampProvider;
- private final Crypto crypto;
- private final HttpUtils utils;
-
- @Resource
- @Named(Constants.LOGGER_SIGNATURE)
- Logger signatureLog = Logger.NULL;
-
- private final String authTag;
- private final String headerTag;
- private final String servicePath;
- private final boolean isVhostStyle;
-
- @Inject
- public RequestAuthorizeSignature(SignatureWire signatureWire, @Named(PROPERTY_AUTH_TAG) String authTag,
- @Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
- @Named(PROPERTY_S3_SERVICE_PATH) String servicePath, @Named(PROPERTY_HEADER_TAG) String headerTag,
- @org.jclouds.location.Provider Supplier<Credentials> creds,
- @TimeStamp Provider<String> timeStampProvider, Crypto crypto, HttpUtils utils) {
- this.isVhostStyle = isVhostStyle;
- this.servicePath = servicePath;
- this.headerTag = headerTag;
- this.authTag = authTag;
- this.signatureWire = signatureWire;
- this.creds = creds;
- this.timeStampProvider = timeStampProvider;
- this.crypto = crypto;
- this.utils = utils;
- }
-
- public HttpRequest filter(HttpRequest request) throws HttpException {
- request = replaceDateHeader(request);
- Credentials current = creds.get();
- if (current instanceof SessionCredentials) {
- request = replaceSecurityTokenHeader(request, SessionCredentials.class.cast(current));
- }
- String signature = calculateSignature(createStringToSign(request));
- request = replaceAuthorizationHeader(request, signature);
- utils.logRequest(signatureLog, request, "<<");
- return request;
- }
-
- HttpRequest replaceSecurityTokenHeader(HttpRequest request, SessionCredentials current) {
- return request.toBuilder().replaceHeader("x-amz-security-token", current.getSessionToken()).build();
- }
-
- protected HttpRequest replaceAuthorizationHeader(HttpRequest request, String signature) {
- request = request.toBuilder()
- .replaceHeader(HttpHeaders.AUTHORIZATION, authTag + " " + creds.get().identity + ":" + signature).build();
- return request;
- }
-
- HttpRequest replaceDateHeader(HttpRequest request) {
- request = request.toBuilder().replaceHeader(HttpHeaders.DATE, timeStampProvider.get()).build();
- return request;
- }
-
- public String createStringToSign(HttpRequest request) {
- utils.logRequest(signatureLog, request, ">>");
- SortedSetMultimap<String, String> canonicalizedHeaders = TreeMultimap.create();
- StringBuilder buffer = new StringBuilder();
- // re-sign the request
- appendMethod(request, buffer);
- appendPayloadMetadata(request, buffer);
- appendHttpHeaders(request, canonicalizedHeaders);
-
- // Remove default date timestamp if "x-amz-date" is set.
- if (canonicalizedHeaders.containsKey("x-" + headerTag + "-date")) {
- canonicalizedHeaders.removeAll("date");
- }
-
- appendAmzHeaders(canonicalizedHeaders, buffer);
- appendBucketName(request, buffer);
- appendUriPath(request, buffer);
- if (signatureWire.enabled())
- signatureWire.output(buffer.toString());
- return buffer.toString();
- }
-
- String calculateSignature(String toSign) throws HttpException {
- String signature = sign(toSign);
- if (signatureWire.enabled())
- signatureWire.input(toInputStream(signature));
- return signature;
- }
-
- public String sign(String toSign) {
- try {
- ByteProcessor<byte[]> hmacSHA1 = asByteProcessor(crypto.hmacSHA1(creds.get().credential.getBytes(UTF_8)));
- return base64().encode(readBytes(toInputStream(toSign), hmacSHA1));
- } catch (Exception e) {
- throw new HttpException("error signing request", e);
- }
- }
-
- void appendMethod(HttpRequest request, StringBuilder toSign) {
- toSign.append(request.getMethod()).append("\n");
- }
-
- @VisibleForTesting
- void appendAmzHeaders(SortedSetMultimap<String, String> canonicalizedHeaders, StringBuilder toSign) {
- for (Entry<String, String> header : canonicalizedHeaders.entries()) {
- String key = header.getKey();
- if (key.startsWith("x-" + headerTag + "-")) {
- toSign.append(String.format("%s:%s\n", key.toLowerCase(), header.getValue()));
- }
- }
- }
-
- void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) {
- // note that we fall back to headers, and some requests such as ?uploads do not have a
- // payload, yet specify payload related parameters
- buffer.append(
- request.getPayload() == null ? Strings.nullToEmpty(request.getFirstHeaderOrNull("Content-MD5")) :
- HttpUtils.nullToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata()
- .getContentMD5())).append("\n");
- buffer.append(
- Strings.nullToEmpty(request.getPayload() == null ? request.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE)
- : request.getPayload().getContentMetadata().getContentType())).append("\n");
- for (String header : FIRST_HEADERS_TO_SIGN)
- buffer.append(HttpUtils.nullToEmpty(request.getHeaders().get(header))).append("\n");
- }
-
- @VisibleForTesting
- void appendHttpHeaders(HttpRequest request, SortedSetMultimap<String, String> canonicalizedHeaders) {
- Multimap<String, String> headers = request.getHeaders();
- for (Entry<String, String> header : headers.entries()) {
- if (header.getKey() == null)
- continue;
- String key = header.getKey().toString().toLowerCase(Locale.getDefault());
- // Ignore any headers that are not particularly interesting.
- if (key.equalsIgnoreCase(HttpHeaders.CONTENT_TYPE) || key.equalsIgnoreCase("Content-MD5")
- || key.equalsIgnoreCase(HttpHeaders.DATE) || key.startsWith("x-" + headerTag + "-")) {
- canonicalizedHeaders.put(key, header.getValue());
- }
- }
- }
-
- @VisibleForTesting
- void appendBucketName(HttpRequest req, StringBuilder toSign) {
- String bucketName = S3Utils.getBucketName(req);
-
- // If we have a payload/bucket/container that is not all lowercase, vhost-style URLs are not an option and must be
- // automatically converted to their path-based equivalent. This should only be possible for AWS-S3 since it is
- // the only S3 implementation configured to allow uppercase payload/bucket/container names.
- //
- // http://code.google.com/p/jclouds/issues/detail?id=992
- if (isVhostStyle && bucketName != null && bucketName.equals(bucketName.toLowerCase()))
- toSign.append(servicePath).append(bucketName);
- }
-
- @VisibleForTesting
- void appendUriPath(HttpRequest request, StringBuilder toSign) {
-
- toSign.append(request.getEndpoint().getRawPath());
-
- // ...however, there are a few exceptions that must be included in the
- // signed URI.
- if (request.getEndpoint().getQuery() != null) {
- Multimap<String, String> params = queryParser().apply(request.getEndpoint().getQuery());
- char separator = '?';
- for (String paramName : Ordering.natural().sortedCopy(params.keySet())) {
- // Skip any parameters that aren't part of the canonical signed string
- if (!SIGNED_PARAMETERS.contains(paramName))
- continue;
- toSign.append(separator).append(paramName);
- String paramValue = get(params.get(paramName), 0);
- if (paramValue != null) {
- toSign.append("=").append(paramValue);
- }
- separator = '&';
- }
- }
- }
+public interface RequestAuthorizeSignature extends HttpRequestFilter {
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV2.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV2.java b/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV2.java
new file mode 100644
index 0000000..dab003c
--- /dev/null
+++ b/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV2.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.collect.Iterables.get;
+import static com.google.common.io.BaseEncoding.base64;
+import static com.google.common.io.ByteStreams.readBytes;
+import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AUTH_TAG;
+import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
+import static org.jclouds.crypto.Macs.asByteProcessor;
+import static org.jclouds.http.utils.Queries.queryParser;
+import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_SERVICE_PATH;
+import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
+import static org.jclouds.util.Strings2.toInputStream;
+
+import java.util.Collection;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Resource;
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Provider;
+import javax.inject.Singleton;
+
+import org.jclouds.Constants;
+import org.jclouds.aws.domain.SessionCredentials;
+import org.jclouds.crypto.Crypto;
+import org.jclouds.date.TimeStamp;
+import org.jclouds.domain.Credentials;
+import org.jclouds.http.HttpException;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpUtils;
+import org.jclouds.http.internal.SignatureWire;
+import org.jclouds.logging.Logger;
+import org.jclouds.rest.RequestSigner;
+import org.jclouds.s3.util.S3Utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.SortedSetMultimap;
+import com.google.common.collect.TreeMultimap;
+import com.google.common.io.ByteProcessor;
+import com.google.common.net.HttpHeaders;
+
+/**
+ * AWS Sign V2
+ */
+@Singleton
+public class RequestAuthorizeSignatureV2 implements RequestAuthorizeSignature, RequestSigner {
+ private static final Collection<String> FIRST_HEADERS_TO_SIGN = ImmutableList.of(HttpHeaders.DATE);
+
+ private static final Set<String> SIGNED_PARAMETERS = ImmutableSet.of("acl", "torrent", "logging", "location",
+ "policy", "requestPayment", "versioning", "versions", "versionId", "notification", "uploadId", "uploads",
+ "partNumber", "website", "response-content-type", "response-content-language", "response-expires",
+ "response-cache-control", "response-content-disposition", "response-content-encoding", "delete");
+
+ private final SignatureWire signatureWire;
+ private final Supplier<Credentials> creds;
+ private final Provider<String> timeStampProvider;
+ private final Crypto crypto;
+ private final HttpUtils utils;
+
+ @Resource
+ @Named(Constants.LOGGER_SIGNATURE)
+ Logger signatureLog = Logger.NULL;
+
+ private final String authTag;
+ private final String headerTag;
+ private final String servicePath;
+ private final boolean isVhostStyle;
+
+ @Inject
+ public RequestAuthorizeSignatureV2(SignatureWire signatureWire, @Named(PROPERTY_AUTH_TAG) String authTag,
+ @Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
+ @Named(PROPERTY_S3_SERVICE_PATH) String servicePath, @Named(PROPERTY_HEADER_TAG) String headerTag,
+ @org.jclouds.location.Provider Supplier<Credentials> creds,
+ @TimeStamp Provider<String> timeStampProvider, Crypto crypto, HttpUtils utils) {
+ this.isVhostStyle = isVhostStyle;
+ this.servicePath = servicePath;
+ this.headerTag = headerTag;
+ this.authTag = authTag;
+ this.signatureWire = signatureWire;
+ this.creds = creds;
+ this.timeStampProvider = timeStampProvider;
+ this.crypto = crypto;
+ this.utils = utils;
+ }
+
+ public HttpRequest filter(HttpRequest request) throws HttpException {
+ request = replaceDateHeader(request);
+ Credentials current = creds.get();
+ if (current instanceof SessionCredentials) {
+ request = replaceSecurityTokenHeader(request, SessionCredentials.class.cast(current));
+ }
+ String signature = calculateSignature(createStringToSign(request));
+ request = replaceAuthorizationHeader(request, signature);
+ utils.logRequest(signatureLog, request, "<<");
+ return request;
+ }
+
+ HttpRequest replaceSecurityTokenHeader(HttpRequest request, SessionCredentials current) {
+ return request.toBuilder().replaceHeader("x-amz-security-token", current.getSessionToken()).build();
+ }
+
+ protected HttpRequest replaceAuthorizationHeader(HttpRequest request, String signature) {
+ request = request.toBuilder()
+ .replaceHeader(HttpHeaders.AUTHORIZATION,
+ authTag + " " + creds.get().identity + ":" + signature).build();
+ return request;
+ }
+
+ HttpRequest replaceDateHeader(HttpRequest request) {
+ request = request.toBuilder().replaceHeader(HttpHeaders.DATE, timeStampProvider.get()).build();
+ return request;
+ }
+
+ public String createStringToSign(HttpRequest request) {
+ utils.logRequest(signatureLog, request, ">>");
+ SortedSetMultimap<String, String> canonicalizedHeaders = TreeMultimap.create();
+ StringBuilder buffer = new StringBuilder();
+ // re-sign the request
+ appendMethod(request, buffer);
+ appendPayloadMetadata(request, buffer);
+ appendHttpHeaders(request, canonicalizedHeaders);
+
+ // Remove default date timestamp if "x-amz-date" is set.
+ if (canonicalizedHeaders.containsKey("x-" + headerTag + "-date")) {
+ canonicalizedHeaders.removeAll("date");
+ }
+
+ appendAmzHeaders(canonicalizedHeaders, buffer);
+ appendBucketName(request, buffer);
+ appendUriPath(request, buffer);
+ if (signatureWire.enabled()) {
+ signatureWire.output(buffer.toString());
+ }
+ return buffer.toString();
+ }
+
+ String calculateSignature(String toSign) throws HttpException {
+ String signature = sign(toSign);
+ if (signatureWire.enabled()) {
+ signatureWire.input(toInputStream(signature));
+ }
+ return signature;
+ }
+
+ public String sign(String toSign) {
+ try {
+ ByteProcessor<byte[]> hmacSHA1 = asByteProcessor(
+ crypto.hmacSHA1(creds.get().credential.getBytes(UTF_8)));
+ return base64().encode(readBytes(toInputStream(toSign), hmacSHA1));
+ } catch (Exception e) {
+ throw new HttpException("error signing request", e);
+ }
+ }
+
+ void appendMethod(HttpRequest request, StringBuilder toSign) {
+ toSign.append(request.getMethod()).append("\n");
+ }
+
+ @VisibleForTesting
+ void appendAmzHeaders(SortedSetMultimap<String, String> canonicalizedHeaders, StringBuilder toSign) {
+ for (Map.Entry<String, String> header : canonicalizedHeaders.entries()) {
+ String key = header.getKey();
+ if (key.startsWith("x-" + headerTag + "-")) {
+ toSign.append(String.format("%s:%s\n", key.toLowerCase(), header.getValue()));
+ }
+ }
+ }
+
+ void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) {
+ // note that we fall back to headers, and some requests such as ?uploads do not have a
+ // payload, yet specify payload related parameters
+ buffer.append(
+ request.getPayload() == null ? Strings.nullToEmpty(request.getFirstHeaderOrNull("Content-MD5")) :
+ HttpUtils.nullToEmpty(
+ request.getPayload() == null ? null : request.getPayload().getContentMetadata()
+ .getContentMD5())).append("\n");
+ buffer.append(
+ Strings.nullToEmpty(
+ request.getPayload() == null ? request.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE)
+ : request.getPayload().getContentMetadata().getContentType())).append("\n");
+ for (String header : FIRST_HEADERS_TO_SIGN)
+ buffer.append(HttpUtils.nullToEmpty(request.getHeaders().get(header))).append("\n");
+ }
+
+ @VisibleForTesting
+ void appendHttpHeaders(HttpRequest request, SortedSetMultimap<String, String> canonicalizedHeaders) {
+ Multimap<String, String> headers = request.getHeaders();
+ for (Map.Entry<String, String> header : headers.entries()) {
+ if (header.getKey() == null) {
+ continue;
+ }
+ String key = header.getKey().toString().toLowerCase(Locale.getDefault());
+ // Ignore any headers that are not particularly interesting.
+ if (key.equalsIgnoreCase(HttpHeaders.CONTENT_TYPE) || key.equalsIgnoreCase("Content-MD5")
+ || key.equalsIgnoreCase(HttpHeaders.DATE) || key.startsWith("x-" + headerTag + "-")) {
+ canonicalizedHeaders.put(key, header.getValue());
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void appendBucketName(HttpRequest req, StringBuilder toSign) {
+ String bucketName = S3Utils.getBucketName(req);
+
+ // If we have a payload/bucket/container that is not all lowercase, vhost-style URLs are not an option and must be
+ // automatically converted to their path-based equivalent. This should only be possible for AWS-S3 since it is
+ // the only S3 implementation configured to allow uppercase payload/bucket/container names.
+ //
+ // http://code.google.com/p/jclouds/issues/detail?id=992
+ if (isVhostStyle && bucketName != null && bucketName.equals(bucketName.toLowerCase())) {
+ toSign.append(servicePath).append(bucketName);
+ }
+ }
+
+ @VisibleForTesting
+ void appendUriPath(HttpRequest request, StringBuilder toSign) {
+
+ toSign.append(request.getEndpoint().getRawPath());
+
+ // ...however, there are a few exceptions that must be included in the
+ // signed URI.
+ if (request.getEndpoint().getQuery() != null) {
+ Multimap<String, String> params = queryParser().apply(request.getEndpoint().getQuery());
+ char separator = '?';
+ for (String paramName : Ordering.natural().sortedCopy(params.keySet())) {
+ // Skip any parameters that aren't part of the canonical signed string
+ if (!SIGNED_PARAMETERS.contains(paramName)) {
+ continue;
+ }
+ toSign.append(separator).append(paramName);
+ String paramValue = get(params.get(paramName), 0);
+ if (paramValue != null) {
+ toSign.append("=").append(paramValue);
+ }
+ separator = '&';
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4.java b/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4.java
new file mode 100644
index 0000000..4e4edbd
--- /dev/null
+++ b/apis/s3/src/main/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+import com.google.common.reflect.TypeToken;
+import com.google.inject.Singleton;
+import org.jclouds.http.HttpException;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.io.Payload;
+import org.jclouds.rest.internal.GeneratedHttpRequest;
+import org.jclouds.s3.S3Client;
+
+import javax.inject.Inject;
+
+@Singleton
+public class RequestAuthorizeSignatureV4 implements RequestAuthorizeSignature {
+
+ private static final String PUT_OBJECT_METHOD = "putObject";
+ private static final TypeToken<S3Client> S3_CLIENT_TYPE = new TypeToken<S3Client>() {
+ };
+
+ private final Aws4SignerForAuthorizationHeader signerForAuthorizationHeader;
+ private final Aws4SignerForChunkedUpload signerForChunkedUpload;
+ private final Aws4SignerForQueryString signerForQueryString;
+
+ @Inject
+ public RequestAuthorizeSignatureV4(Aws4SignerForAuthorizationHeader signerForAuthorizationHeader,
+ Aws4SignerForChunkedUpload signerForChunkedUpload,
+ Aws4SignerForQueryString signerForQueryString) {
+ this.signerForAuthorizationHeader = signerForAuthorizationHeader;
+ this.signerForChunkedUpload = signerForChunkedUpload;
+ this.signerForQueryString = signerForQueryString;
+ }
+
+ @Override
+ public HttpRequest filter(HttpRequest request) throws HttpException {
+ // request use chunked upload
+ if (useChunkedUpload(request)) {
+ return signForChunkedUpload(request);
+ }
+ return signForAuthorizationHeader(request);
+ }
+
+ /**
+ * returns true, if use AWS S3 chunked upload.
+ */
+ protected boolean useChunkedUpload(HttpRequest request) {
+ // only S3Client putObject method, payload not null, content-length > 0 and cannot repeatable
+ if (!GeneratedHttpRequest.class.isAssignableFrom(request.getClass())) {
+ return false;
+ }
+ GeneratedHttpRequest req = GeneratedHttpRequest.class.cast(request);
+
+ // s3 client type and method name is putObject
+ if (S3_CLIENT_TYPE.equals(req.getInvocation().getInvokable().getOwnerType()) &&
+ !PUT_OBJECT_METHOD.equals(req.getInvocation().getInvokable().getName())) {
+ return false;
+ }
+
+ Payload payload = req.getPayload();
+
+ // check payload null or payload.contentMetadata null
+ if (payload == null || payload.getContentMetadata() == null) {
+ return false;
+ }
+
+ Long contentLength = payload.getContentMetadata().getContentLength();
+
+ if (contentLength == null) {
+ return false;
+ }
+
+ return contentLength > 0l && !payload.isRepeatable();
+ }
+
+ protected HttpRequest signForAuthorizationHeader(HttpRequest request) {
+ return signerForAuthorizationHeader.sign(request);
+ }
+
+ protected HttpRequest signForChunkedUpload(HttpRequest request) {
+ return signerForChunkedUpload.sign(request);
+ }
+
+ // Authenticating Requests by Using Query Parameters (AWS Signature Version 4)
+
+ /**
+ * Using query parameters to authenticate requests is useful when you want to express a request entirely in a URL.
+ * This method is also referred as presigning a URL. Presigned URLs enable you to grant temporary access to your
+ * Amazon S3 resources. The end user can then enter the presigned URL in his or her browser to access the specific
+ * Amazon S3 resource. You can also use presigned URLs to embed clickable links in HTML.
+ * <p/>
+ * For example, you might store videos in an Amazon S3 bucket and make them available on your website by using presigned URLs.
+ * Identifies the version of AWS Signature and the algorithm that you used to calculate the signature.
+ */
+ public HttpRequest signForTemporaryAccess(HttpRequest request, long timeInSeconds) {
+ return signerForQueryString.sign(request, timeInSeconds);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/reference/S3Constants.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/reference/S3Constants.java b/apis/s3/src/main/java/org/jclouds/s3/reference/S3Constants.java
index a376ae3..7569573 100644
--- a/apis/s3/src/main/java/org/jclouds/s3/reference/S3Constants.java
+++ b/apis/s3/src/main/java/org/jclouds/s3/reference/S3Constants.java
@@ -31,6 +31,7 @@ public final class S3Constants {
public static final String DELIMITER = "delimiter";
public static final String PROPERTY_S3_SERVICE_PATH = "jclouds.s3.service-path";
public static final String PROPERTY_S3_VIRTUAL_HOST_BUCKETS = "jclouds.s3.virtual-host-buckets";
+ public static final String PROPERTY_JCLOUDS_S3_CHUNKED_SIZE = "jclouds.s3.chunked.size";
private S3Constants() {
throw new AssertionError("intentionally unimplemented");
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/test/java/org/jclouds/s3/filters/AwsHostNameUtilsTest.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/test/java/org/jclouds/s3/filters/AwsHostNameUtilsTest.java b/apis/s3/src/test/java/org/jclouds/s3/filters/AwsHostNameUtilsTest.java
new file mode 100644
index 0000000..6e160cd
--- /dev/null
+++ b/apis/s3/src/test/java/org/jclouds/s3/filters/AwsHostNameUtilsTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.net.URI;
+
+/**
+ * Tests parser region and service
+ */
+public class AwsHostNameUtilsTest {
+ @Test
+ public void testParseRegion() {
+ Assert.assertEquals(
+ AwsHostNameUtils.parseRegionName("test.s3.cn-north-1.amazonaws.com.cn", "s3"),
+ "cn-north-1"
+ );
+
+
+ }
+
+ @Test
+ // default region
+ public void testParseDefaultRegion() {
+ Assert.assertEquals(
+ AwsHostNameUtils.parseRegionName("s3.amazonaws.com", "s3"),
+ "us-east-1"
+ );
+ }
+
+ @Test
+ // test s3 service
+ public void testParseService() {
+ Assert.assertEquals(
+ AwsHostNameUtils.parseServiceName(URI.create("https://s3.amazonaws.com")),
+ "s3"
+ );
+
+
+ Assert.assertEquals(
+ AwsHostNameUtils.parseServiceName(URI.create("https://test-bucket.s3.cn-north-1.amazonaws.com.cn")),
+ "s3"
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java
deleted file mode 100644
index 794701d..0000000
--- a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureTest.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.jclouds.s3.filters;
-
-import static org.jclouds.Constants.PROPERTY_SESSION_INTERVAL;
-import static org.jclouds.reflect.Reflection2.method;
-import static org.testng.Assert.assertEquals;
-
-import java.net.URI;
-import java.util.Properties;
-
-import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest;
-import org.jclouds.http.HttpRequest;
-import org.jclouds.rest.internal.GeneratedHttpRequest;
-import org.jclouds.s3.S3Client;
-import org.jclouds.s3.domain.AccessControlList;
-import org.jclouds.s3.domain.CannedAccessPolicy;
-import org.jclouds.s3.domain.S3Object;
-import org.jclouds.s3.internal.BaseS3ClientTest;
-import org.jclouds.s3.options.PutObjectOptions;
-import org.jclouds.s3.reference.S3Headers;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.SortedSetMultimap;
-import com.google.common.collect.TreeMultimap;
-import com.google.common.net.HttpHeaders;
-/**
- * Tests behavior of {@code RequestAuthorizeSignature}
- */
-// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
-@Test(groups = "unit", testName = "RequestAuthorizeSignatureTest")
-public class RequestAuthorizeSignatureTest extends BaseS3ClientTest<S3Client> {
- String bucketName = "bucket";
-
- @DataProvider(parallel = true)
- public Object[][] dataProvider() throws NoSuchMethodException {
- return new Object[][] { { listOwnedBuckets() }, { putObject() }, { putBucketAcl() }
-
- };
- }
-
- /**
- * NOTE this test is dependent on how frequently the timestamp updates. At the time of writing,
- * this was once per second. If this timestamp update interval is increased, it could make this
- * test appear to hang for a long time.
- */
- @Test(threadPoolSize = 3, dataProvider = "dataProvider", timeOut = 10000)
- void testIdempotent(HttpRequest request) {
- request = filter.filter(request);
- String signature = request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION);
- String date = request.getFirstHeaderOrNull(HttpHeaders.DATE);
- int iterations = 1;
- while (request.getFirstHeaderOrNull(HttpHeaders.DATE).equals(date)) {
- date = request.getFirstHeaderOrNull(HttpHeaders.DATE);
- request = filter.filter(request);
- if (request.getFirstHeaderOrNull(HttpHeaders.DATE).equals(date))
- assert signature.equals(request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION)) : String.format(
- "sig: %s != %s on attempt %s", signature, request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION),
- iterations);
- else
- iterations++;
-
- }
- System.out.printf("%s: %d iterations before the timestamp updated %n", Thread.currentThread().getName(),
- iterations);
- }
-
- @Test
- void testAppendBucketNameHostHeader() throws SecurityException, NoSuchMethodException {
- GeneratedHttpRequest request = processor.createRequest(
- method(S3Client.class, "getBucketLocation", String.class),
- ImmutableList.<Object> of("bucket"));
- StringBuilder builder = new StringBuilder();
- filter.appendBucketName(request, builder);
- assertEquals(builder.toString(), "");
- }
-
- @Test
- void testAclQueryString() throws SecurityException, NoSuchMethodException {
- HttpRequest request = putBucketAcl();
- StringBuilder builder = new StringBuilder();
- filter.appendUriPath(request, builder);
- assertEquals(builder.toString(), "/" + bucketName + "?acl");
- }
-
- private GeneratedHttpRequest putBucketAcl() throws NoSuchMethodException {
- return processor.createRequest(
- method(S3Client.class, "putBucketACL", String.class, AccessControlList.class),
- ImmutableList.<Object> of("bucket",
- AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PRIVATE, "1234")));
- }
-
- // "?acl", "?location", "?logging", "?uploads", or "?torrent"
-
- @Test
- void testAppendBucketNameHostHeaderService() throws SecurityException, NoSuchMethodException {
- HttpRequest request = listOwnedBuckets();
- StringBuilder builder = new StringBuilder();
- filter.appendBucketName(request, builder);
- assertEquals(builder.toString(), "");
- }
-
- private GeneratedHttpRequest listOwnedBuckets() throws NoSuchMethodException {
- return processor.createRequest(method(S3Client.class, "listOwnedBuckets"),
- ImmutableList.of());
- }
-
- @Test
- void testHeadersGoLowercase() throws SecurityException, NoSuchMethodException {
- HttpRequest request = putObject();
- SortedSetMultimap<String, String> canonicalizedHeaders = TreeMultimap.create();
- filter.appendHttpHeaders(request, canonicalizedHeaders);
- StringBuilder builder = new StringBuilder();
- filter.appendAmzHeaders(canonicalizedHeaders, builder);
- assertEquals(builder.toString(), S3Headers.USER_METADATA_PREFIX + "adrian:foo\n");
- }
-
- private HttpRequest putObject() throws NoSuchMethodException {
- S3Object object = blobToS3Object.apply(BindBlobToMultipartFormTest.TEST_BLOB);
- object.getMetadata().getUserMetadata().put("Adrian", "foo");
- return processor.createRequest(method(S3Client.class, "putObject", String.class,
- S3Object.class, PutObjectOptions[].class), ImmutableList.<Object> of("bucket", object));
- }
-
- @Test
- void testAppendBucketNameInURIPath() throws SecurityException, NoSuchMethodException {
- GeneratedHttpRequest request = processor.createRequest(
- method(S3Client.class, "getBucketLocation", String.class),
- ImmutableList.<Object> of(bucketName));
- URI uri = request.getEndpoint();
- assertEquals(uri.getHost(), "localhost");
- assertEquals(uri.getPath(), "/" + bucketName);
- }
-
- @Override
- protected Properties setupProperties() {
- Properties overrides = super.setupProperties();
- overrides.setProperty(PROPERTY_SESSION_INTERVAL, 1 + "");
- return overrides;
- }
-}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV2Test.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV2Test.java b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV2Test.java
new file mode 100644
index 0000000..1733a1b
--- /dev/null
+++ b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV2Test.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+import static org.jclouds.Constants.PROPERTY_SESSION_INTERVAL;
+import static org.jclouds.reflect.Reflection2.method;
+import static org.testng.Assert.assertEquals;
+
+import java.net.URI;
+import java.util.Properties;
+
+import org.jclouds.blobstore.binders.BindBlobToMultipartFormTest;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.rest.internal.GeneratedHttpRequest;
+import org.jclouds.s3.S3Client;
+import org.jclouds.s3.domain.AccessControlList;
+import org.jclouds.s3.domain.CannedAccessPolicy;
+import org.jclouds.s3.domain.S3Object;
+import org.jclouds.s3.internal.BaseS3ClientTest;
+import org.jclouds.s3.options.PutObjectOptions;
+import org.jclouds.s3.reference.S3Headers;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.SortedSetMultimap;
+import com.google.common.collect.TreeMultimap;
+import com.google.common.net.HttpHeaders;
+
+/**
+ * Tests behavior of {@code RequestAuthorizeSignatureV2}
+ */
+// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
+@Test(groups = "unit", testName = "RequestAuthorizeSignatureV2Test")
+public class RequestAuthorizeSignatureV2Test extends BaseS3ClientTest<S3Client> {
+ String bucketName = "bucket";
+
+ @DataProvider(parallel = true)
+ public Object[][] dataProvider() throws NoSuchMethodException {
+ return new Object[][]{{listOwnedBuckets()}, {putObject()}, {putBucketAcl()}
+
+ };
+ }
+
+ /**
+ * NOTE this test is dependent on how frequently the timestamp updates. At the time of writing,
+ * this was once per second. If this timestamp update interval is increased, it could make this
+ * test appear to hang for a long time.
+ */
+ @Test(threadPoolSize = 3, dataProvider = "dataProvider", timeOut = 10000)
+ void testIdempotent(HttpRequest request) {
+ request = filter.filter(request);
+ String signature = request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION);
+ String date = request.getFirstHeaderOrNull(HttpHeaders.DATE);
+ int iterations = 1;
+ while (request.getFirstHeaderOrNull(HttpHeaders.DATE).equals(date)) {
+ date = request.getFirstHeaderOrNull(HttpHeaders.DATE);
+ request = filter.filter(request);
+ if (request.getFirstHeaderOrNull(HttpHeaders.DATE).equals(date))
+ assert signature.equals(request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION)) : String.format(
+ "sig: %s != %s on attempt %s", signature, request.getFirstHeaderOrNull(HttpHeaders.AUTHORIZATION),
+ iterations);
+ else
+ iterations++;
+
+ }
+ System.out.printf("%s: %d iterations before the timestamp updated %n", Thread.currentThread().getName(),
+ iterations);
+ }
+
+ @Test
+ void testAppendBucketNameHostHeader() throws SecurityException, NoSuchMethodException {
+ GeneratedHttpRequest request = processor.createRequest(
+ method(S3Client.class, "getBucketLocation", String.class),
+ ImmutableList.<Object>of("bucket"));
+ StringBuilder builder = new StringBuilder();
+ ((RequestAuthorizeSignatureV2) filter).appendBucketName(request, builder);
+ assertEquals(builder.toString(), "");
+ }
+
+ @Test
+ void testAclQueryString() throws SecurityException, NoSuchMethodException {
+ HttpRequest request = putBucketAcl();
+ StringBuilder builder = new StringBuilder();
+ ((RequestAuthorizeSignatureV2) filter).appendUriPath(request, builder);
+ assertEquals(builder.toString(), "/" + bucketName + "?acl");
+ }
+
+ private GeneratedHttpRequest putBucketAcl() throws NoSuchMethodException {
+ return processor.createRequest(
+ method(S3Client.class, "putBucketACL", String.class, AccessControlList.class),
+ ImmutableList.<Object>of("bucket",
+ AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PRIVATE, "1234")));
+ }
+
+ // "?acl", "?location", "?logging", "?uploads", or "?torrent"
+
+ @Test
+ void testAppendBucketNameHostHeaderService() throws SecurityException, NoSuchMethodException {
+ HttpRequest request = listOwnedBuckets();
+ StringBuilder builder = new StringBuilder();
+ ((RequestAuthorizeSignatureV2) filter).appendBucketName(request, builder);
+ assertEquals(builder.toString(), "");
+ }
+
+ private GeneratedHttpRequest listOwnedBuckets() throws NoSuchMethodException {
+ return processor.createRequest(method(S3Client.class, "listOwnedBuckets"),
+ ImmutableList.of());
+ }
+
+ @Test
+ void testHeadersGoLowercase() throws SecurityException, NoSuchMethodException {
+ HttpRequest request = putObject();
+ SortedSetMultimap<String, String> canonicalizedHeaders = TreeMultimap.create();
+ ((RequestAuthorizeSignatureV2) filter).appendHttpHeaders(request, canonicalizedHeaders);
+ StringBuilder builder = new StringBuilder();
+ ((RequestAuthorizeSignatureV2) filter).appendAmzHeaders(canonicalizedHeaders, builder);
+ assertEquals(builder.toString(), S3Headers.USER_METADATA_PREFIX + "adrian:foo\n");
+ }
+
+ private HttpRequest putObject() throws NoSuchMethodException {
+ S3Object object = blobToS3Object.apply(BindBlobToMultipartFormTest.TEST_BLOB);
+ object.getMetadata().getUserMetadata().put("Adrian", "foo");
+ return processor.createRequest(method(S3Client.class, "putObject", String.class,
+ S3Object.class, PutObjectOptions[].class), ImmutableList.<Object>of("bucket", object));
+ }
+
+ @Test
+ void testAppendBucketNameInURIPath() throws SecurityException, NoSuchMethodException {
+ GeneratedHttpRequest request = processor.createRequest(
+ method(S3Client.class, "getBucketLocation", String.class),
+ ImmutableList.<Object>of(bucketName));
+ URI uri = request.getEndpoint();
+ assertEquals(uri.getHost(), "localhost");
+ assertEquals(uri.getPath(), "/" + bucketName);
+ }
+
+ @Override
+ protected Properties setupProperties() {
+ Properties overrides = super.setupProperties();
+ overrides.setProperty(PROPERTY_SESSION_INTERVAL, 1 + "");
+ return overrides;
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4ChunkedUploadTest.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4ChunkedUploadTest.java b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4ChunkedUploadTest.java
new file mode 100644
index 0000000..39a6fe1
--- /dev/null
+++ b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4ChunkedUploadTest.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableList;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import com.google.common.net.HttpHeaders;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import org.jclouds.Constants;
+import org.jclouds.ContextBuilder;
+import org.jclouds.date.DateService;
+import org.jclouds.date.TimeStamp;
+import org.jclouds.domain.Credentials;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.io.Payload;
+import org.jclouds.io.Payloads;
+import org.jclouds.logging.config.NullLoggingModule;
+import org.jclouds.reflect.Invocation;
+import org.jclouds.rest.ConfiguresHttpApi;
+import org.jclouds.rest.internal.BaseRestApiTest;
+import org.jclouds.rest.internal.GeneratedHttpRequest;
+import org.jclouds.s3.S3ApiMetadata;
+import org.jclouds.s3.S3Client;
+import org.jclouds.s3.config.S3HttpApiModule;
+import org.jclouds.s3.domain.S3Object;
+import org.jclouds.s3.options.PutObjectOptions;
+import org.jclouds.util.Closeables2;
+import org.testng.annotations.Test;
+
+import javax.inject.Named;
+import javax.xml.ws.http.HTTPException;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.Date;
+
+import static com.google.common.io.BaseEncoding.base16;
+import static org.jclouds.reflect.Reflection2.method;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
+
+/**
+ * Tests behavior of {@code RequestAuthorizeSignature}
+ */
+// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
+@Test(groups = "unit", testName = "RequestAuthorizeSignatureV4ChunkedUploadTest")
+public class RequestAuthorizeSignatureV4ChunkedUploadTest {
+ private static final String CONTENT_SEED =
+ "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc tortor metus, sagittis eget augue ut,\n"
+ + "feugiat vehicula risus. Integer tortor mauris, vehicula nec mollis et, consectetur eget tortor. In ut\n"
+ + "elit sagittis, ultrices est ut, iaculis turpis. In hac habitasse platea dictumst. Donec laoreet tellus\n"
+ + "at auctor tempus. Praesent nec diam sed urna sollicitudin vehicula eget id est. Vivamus sed laoreet\n"
+ + "lectus. Aliquam convallis condimentum risus, vitae porta justo venenatis vitae. Phasellus vitae nunc\n"
+ + "varius, volutpat quam nec, mollis urna. Donec tempus, nisi vitae gravida facilisis, sapien sem malesuada\n"
+ + "purus, id semper libero ipsum condimentum nulla. Suspendisse vel mi leo. Morbi pellentesque placerat congue.\n"
+ + "Nunc sollicitudin nunc diam, nec hendrerit dui commodo sed. Duis dapibus commodo elit, id commodo erat\n"
+ + "congue id. Aliquam erat volutpat.\n";
+
+ private static final String CHUKED_UPLOAD_PAYLOAD_SHA256 = "2b6da230b03189254b2ceafe689c5298cfdd288869e80b2b9369da8f8f0a3d99";
+
+ private static final String PUT_OBJECT_AUTHORIZATION = "AWS4-HMAC-SHA256 "
+ + "Credential=AKIAPAEBI3QI4EXAMPLE/20150203/cn-north-1/s3/aws4_request, "
+ + "SignedHeaders=content-encoding;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class, "
+ + "Signature=3db48b3d786d599e8e785ba66030e8a9249c678a52f2432bf6fd44c97cb3145f";
+
+
+ private static final String IDENTITY = "AKIAPAEBI3QI4EXAMPLE";
+ private static final String CREDENTIAL = "oHkkcPcOjJnoAXpjT8GXdNeBjo6Ru7QeFExAmPlE";
+ private static final String TIMESTAMP = "Thu, 03 Feb 2015 07:11:11 GMT";
+
+ private static final String BUCKET_NAME = "test-bucket";
+ private static final String OBJECT_NAME = "ExampleChunkedObject.txt";
+
+ @ConfiguresHttpApi
+ private static final class TestS3HttpApiModule extends S3HttpApiModule<S3Client> {
+ @Override
+ protected String provideTimeStamp(@TimeStamp Supplier<String> cache) {
+ return TIMESTAMP;
+ }
+
+ @Override
+ protected Supplier<Date> provideTimeStampCacheDate(
+ @Named(Constants.PROPERTY_SESSION_INTERVAL) long seconds,
+ @TimeStamp final Supplier<String> timestamp,
+ final DateService dateService) {
+ return Suppliers.ofInstance(dateService.rfc822DateParse(TIMESTAMP));
+ }
+ }
+
+ public static Injector injector(Credentials creds) {
+ return ContextBuilder.newBuilder(new S3ApiMetadata())
+ .credentialsSupplier(Suppliers.<Credentials>ofInstance(creds))
+ .modules(ImmutableList.<Module>of(new BaseRestApiTest.MockModule(), new NullLoggingModule(),
+ new TestS3HttpApiModule()))
+ .buildInjector();
+ }
+
+ public static RequestAuthorizeSignatureV4 filter(Credentials creds) {
+ return injector(creds).getInstance(RequestAuthorizeSignatureV4.class);
+ }
+
+ Credentials temporaryCredentials = new Credentials.Builder()
+ .identity(IDENTITY)
+ .credential(CREDENTIAL)
+ .build();
+
+
+ @Test
+ void testPutObjectWithChunkedUpload() {
+ Invocation invocation = Invocation.create(
+ method(S3Client.class, "putObject", String.class, S3Object.class, PutObjectOptions[].class),
+ ImmutableList.<Object>of(BUCKET_NAME));
+ byte[] content = make65KPayload().getBytes(Charset.forName("UTF-8"));
+ HttpRequest putObject = GeneratedHttpRequest.builder().invocation(invocation)
+ .method("PUT")
+ .endpoint("https://" + BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn/" + OBJECT_NAME)
+ .addHeader(HttpHeaders.HOST, BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn")
+ .addHeader("x-amz-storage-class", "REDUCED_REDUNDANCY")
+ .build();
+ Payload payload = Payloads.newInputStreamPayload(new ByteArrayInputStream(content));
+ payload.getContentMetadata().setContentLength((long) content.length);
+ payload.getContentMetadata().setContentType("text/plain");
+ putObject.setPayload(payload);
+ HttpRequest filtered = filter(temporaryCredentials).filter(putObject);
+ assertEquals(filtered.getFirstHeaderOrNull("Authorization"), PUT_OBJECT_AUTHORIZATION);
+ assertEquals(filtered.getPayload().getClass(), ChunkedUploadPayload.class);
+
+ InputStream is = null;
+ try {
+ is = filtered.getPayload().openStream();
+ assertEquals(base16().lowerCase().encode(hash(is)), CHUKED_UPLOAD_PAYLOAD_SHA256);
+ } catch (IOException e) {
+ fail("open stream error", e);
+ } finally {
+ Closeables2.closeQuietly(is);
+ }
+ }
+
+ /**
+ * Want sample to upload 3 chunks for our selected chunk size of 64K; one
+ * full size chunk, one partial chunk and then the 0-byte terminator chunk.
+ * This routine just takes 1K of seed text and turns it into a 65K-or-so
+ * string for sample use.
+ */
+ private static String make65KPayload() {
+ StringBuilder oneKSeed = new StringBuilder();
+ while (oneKSeed.length() < 1024) {
+ oneKSeed.append(CONTENT_SEED);
+ }
+
+ // now scale up to meet/exceed our requirement
+ StringBuilder output = new StringBuilder();
+ for (int i = 0; i < 66; i++) {
+ output.append(oneKSeed);
+ }
+ return output.toString();
+ }
+
+ /**
+ * hash input with sha256
+ *
+ * @param input
+ * @return hash result
+ * @throws HTTPException
+ */
+ private static byte[] hash(InputStream input) {
+ try {
+ Hasher hasher = Hashing.sha256().newHasher();
+ byte[] buffer = new byte[4096];
+ int r;
+ while ((r = input.read(buffer)) != -1) {
+ hasher.putBytes(buffer, 0, r);
+ }
+ return hasher.hash().asBytes();
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to compute hash while signing request: " + e.getMessage(), e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4Test.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4Test.java b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4Test.java
new file mode 100644
index 0000000..9494a86
--- /dev/null
+++ b/apis/s3/src/test/java/org/jclouds/s3/filters/RequestAuthorizeSignatureV4Test.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+import static org.jclouds.reflect.Reflection2.method;
+import static org.testng.Assert.assertEquals;
+
+import java.util.Date;
+
+import javax.inject.Named;
+
+import org.jclouds.Constants;
+import org.jclouds.ContextBuilder;
+import org.jclouds.date.DateService;
+import org.jclouds.date.TimeStamp;
+import org.jclouds.domain.Credentials;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.options.GetOptions;
+import org.jclouds.io.Payload;
+import org.jclouds.io.Payloads;
+import org.jclouds.logging.config.NullLoggingModule;
+import org.jclouds.reflect.Invocation;
+import org.jclouds.rest.ConfiguresHttpApi;
+import org.jclouds.rest.internal.BaseRestApiTest;
+import org.jclouds.rest.internal.GeneratedHttpRequest;
+import org.jclouds.s3.S3ApiMetadata;
+import org.jclouds.s3.S3Client;
+import org.jclouds.s3.config.S3HttpApiModule;
+import org.jclouds.s3.domain.S3Object;
+import org.jclouds.s3.options.PutObjectOptions;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableList;
+import com.google.common.net.HttpHeaders;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+/**
+ * Tests behavior of {@code RequestAuthorizeSignature}
+ */
+// NOTE:without testName, this will not call @Before* and fail w/NPE during surefire
+@Test(groups = "unit", testName = "RequestAuthorizeSignatureV4Test")
+public class RequestAuthorizeSignatureV4Test {
+ private static final String IDENTITY = "AKIAPAEBI3QI4EXAMPLE";
+ private static final String CREDENTIAL = "oHkkcPcOjJnoAXpjT8GXdNeBjo6Ru7QeFExAmPlE";
+ private static final String TIMESTAMP = "Thu, 03 Feb 2015 07:11:11 GMT";
+
+ private static final String GET_BUCKET_LOCATION_SIGNATURE_RESULT = "AWS4-HMAC-SHA256 "
+ + "Credential=AKIAPAEBI3QI4EXAMPLE/20150203/cn-north-1/s3/aws4_request, "
+ + "SignedHeaders=host;x-amz-content-sha256;x-amz-date, "
+ + "Signature=5634847b3ad6a857887ab0ccff2fcaf3d35ef3dc549a3c27ebc0f584a80494c3";
+
+ private static final String GET_OBJECT_RESULT = "AWS4-HMAC-SHA256 "
+ + "Credential=AKIAPAEBI3QI4EXAMPLE/20150203/cn-north-1/s3/aws4_request, "
+ + "SignedHeaders=host;x-amz-content-sha256;x-amz-date, "
+ + "Signature=fbd1d0f04a72907fb20ecd771644afd62cb689f91d26e9471b7a234531ec4718";
+
+ private static final String GET_OBJECT_ACL_RESULT = "AWS4-HMAC-SHA256 "
+ + "Credential=AKIAPAEBI3QI4EXAMPLE/20150203/cn-north-1/s3/aws4_request, "
+ + "SignedHeaders=host;x-amz-content-sha256;x-amz-date, "
+ + "Signature=52d7f31d249032b59781fe69c8124ff4bf209be3f374b28657a60d906c752381";
+
+ private static final String PUT_OBJECT_CONTENT = "text sign";
+
+ private static final String PUT_OBJECT_RESULT = "AWS4-HMAC-SHA256 "
+ + "Credential=AKIAPAEBI3QI4EXAMPLE/20150203/cn-north-1/s3/aws4_request, "
+ + "SignedHeaders=content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, "
+ + "Signature=090f1bb1db984221ae1a20c5d12a82820a0d74b4be85f20daa1431604f41df08";
+
+ private static final String BUCKET_NAME = "test-bucket";
+ private static final String OBJECT_NAME = "ExampleObject.txt";
+
+ @ConfiguresHttpApi
+ private static final class TestS3HttpApiModule extends S3HttpApiModule<S3Client> {
+ @Override
+ protected String provideTimeStamp(@TimeStamp Supplier<String> cache) {
+ return TIMESTAMP;
+ }
+
+ @Override
+ protected Supplier<Date> provideTimeStampCacheDate(
+ @Named(Constants.PROPERTY_SESSION_INTERVAL) long seconds,
+ @TimeStamp final Supplier<String> timestamp,
+ final DateService dateService) {
+ return Suppliers.ofInstance(dateService.rfc822DateParse(TIMESTAMP));
+ }
+ }
+
+ public static Injector injector(Credentials creds) {
+ return ContextBuilder.newBuilder(new S3ApiMetadata())
+ .credentialsSupplier(Suppliers.<Credentials>ofInstance(creds))
+ .modules(ImmutableList.<Module>of(new BaseRestApiTest.MockModule(), new NullLoggingModule(),
+ new TestS3HttpApiModule()))
+ .buildInjector();
+ }
+
+ public static RequestAuthorizeSignatureV4 filter(Credentials creds) {
+ return injector(creds).getInstance(RequestAuthorizeSignatureV4.class);
+ }
+
+ Credentials temporaryCredentials = new Credentials.Builder()
+ .identity(IDENTITY)
+ .credential(CREDENTIAL)
+ .build();
+
+
+ @Test
+ void testGetBucketLocationSignature() {
+ Invocation invocation = Invocation.create(method(S3Client.class, "getBucketLocation", String.class),
+ ImmutableList.<Object>of(BUCKET_NAME));
+
+ HttpRequest getBucketLocation = GeneratedHttpRequest.builder().method("GET")
+ .invocation(invocation)
+ .endpoint("https://" + BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn/")
+ .addHeader(HttpHeaders.HOST, BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn")
+ .addQueryParam("location", "")
+ .build();
+ HttpRequest filtered = filter(temporaryCredentials).filter(getBucketLocation);
+ assertEquals(filtered.getFirstHeaderOrNull("Authorization"), GET_BUCKET_LOCATION_SIGNATURE_RESULT);
+ }
+
+ @Test
+ void testGetObjectSignature() {
+ Invocation invocation = Invocation.create(method(S3Client.class, "getObject", String.class,
+ String.class, GetOptions[].class),
+ ImmutableList.<Object>of(BUCKET_NAME, OBJECT_NAME, new GetOptions[0]));
+
+ HttpRequest getObject = GeneratedHttpRequest.builder().method("GET")
+ .invocation(invocation)
+ .endpoint("https://" + BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn/" + OBJECT_NAME)
+ .addHeader(HttpHeaders.HOST, BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn")
+ .build();
+
+ HttpRequest filtered = filter(temporaryCredentials).filter(getObject);
+ assertEquals(filtered.getFirstHeaderOrNull("Authorization"), GET_OBJECT_RESULT);
+
+ }
+
+ @Test
+ void testGetObjectACLSignature() {
+
+ Invocation invocation = Invocation.create(method(S3Client.class, "getObjectACL", String.class, String.class),
+ ImmutableList.<Object>of(BUCKET_NAME));
+
+ HttpRequest getObjectACL = GeneratedHttpRequest.builder().method("GET")
+ .invocation(invocation)
+ .endpoint("https://" + BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn/" + OBJECT_NAME)
+ .addHeader(HttpHeaders.HOST, BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn")
+ .addQueryParam("acl", "")
+ .build();
+
+ HttpRequest filtered = filter(temporaryCredentials).filter(getObjectACL);
+ assertEquals(filtered.getFirstHeaderOrNull("Authorization"), GET_OBJECT_ACL_RESULT);
+ }
+
+ @Test
+ void testPutObjectSignature() {
+ Invocation invocation = Invocation.create(method(S3Client.class, "putObject", String.class, S3Object.class,
+ PutObjectOptions[].class),
+ ImmutableList.<Object>of(BUCKET_NAME));
+
+ Payload payload = Payloads.newStringPayload(PUT_OBJECT_CONTENT);
+ payload.getContentMetadata().setContentType("text/plain");
+
+ HttpRequest putObject = GeneratedHttpRequest.builder().method("PUT")
+ .invocation(invocation)
+ .endpoint("https://" + BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn/" + OBJECT_NAME)
+ .addHeader(HttpHeaders.HOST, BUCKET_NAME + ".s3.cn-north-1.amazonaws.com.cn")
+ .addHeader("x-amz-storage-class", "REDUCED_REDUNDANCY")
+ .payload(payload)
+ .build();
+
+ HttpRequest filtered = filter(temporaryCredentials).filter(putObject);
+ assertEquals(filtered.getFirstHeaderOrNull("Authorization"), PUT_OBJECT_RESULT);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/test/java/org/jclouds/s3/internal/BaseS3ClientTest.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/test/java/org/jclouds/s3/internal/BaseS3ClientTest.java b/apis/s3/src/test/java/org/jclouds/s3/internal/BaseS3ClientTest.java
index 63d4481..ee48a0f 100644
--- a/apis/s3/src/test/java/org/jclouds/s3/internal/BaseS3ClientTest.java
+++ b/apis/s3/src/test/java/org/jclouds/s3/internal/BaseS3ClientTest.java
@@ -26,6 +26,7 @@ import org.jclouds.s3.S3ApiMetadata;
import org.jclouds.s3.S3Client;
import org.jclouds.s3.blobstore.functions.BlobToObject;
import org.jclouds.s3.filters.RequestAuthorizeSignature;
+import org.jclouds.s3.filters.RequestAuthorizeSignatureV2;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -47,7 +48,7 @@ public abstract class BaseS3ClientTest<T extends S3Client> extends BaseRestAnnot
protected void setupFactory() throws IOException {
super.setupFactory();
blobToS3Object = injector.getInstance(BlobToObject.class);
- filter = injector.getInstance(RequestAuthorizeSignature.class);
+ filter = injector.getInstance(RequestAuthorizeSignatureV2.class);
}
public BaseS3ClientTest() {
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobRequestSigner.java
----------------------------------------------------------------------
diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobRequestSigner.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobRequestSigner.java
index 39649af..aa967c9 100644
--- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobRequestSigner.java
+++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobRequestSigner.java
@@ -33,7 +33,7 @@ import org.jclouds.reflect.Invocation;
import org.jclouds.rest.internal.RestAnnotationProcessor;
import org.jclouds.s3.blobstore.S3BlobRequestSigner;
import org.jclouds.s3.blobstore.functions.BlobToObject;
-import org.jclouds.s3.filters.RequestAuthorizeSignature;
+import org.jclouds.s3.filters.RequestAuthorizeSignatureV2;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
@@ -44,7 +44,7 @@ import com.google.inject.Provider;
public class AWSS3BlobRequestSigner extends S3BlobRequestSigner<AWSS3Client> {
public static final String TEMPORARY_SIGNATURE_PARAM = "Signature";
- private final RequestAuthorizeSignature authSigner;
+ private final RequestAuthorizeSignatureV2 authSigner;
private final String identity;
private final DateService dateService;
private final Provider<String> timeStampProvider;
@@ -53,7 +53,7 @@ public class AWSS3BlobRequestSigner extends S3BlobRequestSigner<AWSS3Client> {
public AWSS3BlobRequestSigner(RestAnnotationProcessor processor, BlobToObject blobToObject,
BlobToHttpGetOptions blob2HttpGetOptions, Class<AWSS3Client> interfaceClass,
@org.jclouds.location.Provider Supplier<Credentials> credentials,
- RequestAuthorizeSignature authSigner, @TimeStamp Provider<String> timeStampProvider,
+ RequestAuthorizeSignatureV2 authSigner, @TimeStamp Provider<String> timeStampProvider,
DateService dateService) throws SecurityException, NoSuchMethodException {
super(processor, blobToObject, blob2HttpGetOptions, interfaceClass);
this.authSigner = authSigner;
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobRequestSignerV4.java
----------------------------------------------------------------------
diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobRequestSignerV4.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobRequestSignerV4.java
new file mode 100644
index 0000000..f470999
--- /dev/null
+++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobRequestSignerV4.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.aws.s3.blobstore;
+
+import static org.jclouds.blobstore.util.BlobStoreUtils.cleanRequest;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.jclouds.aws.s3.AWSS3Client;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.reflect.Invocation;
+import org.jclouds.rest.internal.RestAnnotationProcessor;
+import org.jclouds.s3.blobstore.S3BlobRequestSigner;
+import org.jclouds.s3.blobstore.functions.BlobToObject;
+import org.jclouds.s3.filters.RequestAuthorizeSignatureV4;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+
+public class AWSS3BlobRequestSignerV4 extends S3BlobRequestSigner<AWSS3Client> {
+
+ private final RequestAuthorizeSignatureV4 authSigner;
+
+ @Inject
+ public AWSS3BlobRequestSignerV4(RestAnnotationProcessor processor, BlobToObject blobToObject,
+ BlobToHttpGetOptions blob2HttpGetOptions, Class<AWSS3Client> interfaceClass,
+ RequestAuthorizeSignatureV4 authSigner) throws SecurityException, NoSuchMethodException {
+ super(processor, blobToObject, blob2HttpGetOptions, interfaceClass);
+ this.authSigner = authSigner;
+ }
+
+ @Override
+ public HttpRequest signGetBlob(String container, String name, long timeInSeconds) {
+ checkNotNull(container, "container");
+ checkNotNull(name, "name");
+ HttpRequest request = processor.apply(Invocation.create(getMethod, ImmutableList.<Object>of(container, name)));
+ request = authSigner.signForTemporaryAccess(request, timeInSeconds);
+ return cleanRequest(request);
+ }
+
+ @Override
+ public HttpRequest signPutBlob(String container, Blob blob, long timeInSeconds) {
+ checkNotNull(container, "container");
+ checkNotNull(blob, "blob");
+ HttpRequest request = processor.apply(Invocation.create(createMethod,
+ ImmutableList.<Object>of(container, blobToObject.apply(blob))));
+ request = authSigner.signForTemporaryAccess(request, timeInSeconds);
+ return cleanRequest(request);
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/providers/aws-s3/src/main/java/org/jclouds/aws/s3/filters/AWSRequestAuthorizeSignature.java
----------------------------------------------------------------------
diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/filters/AWSRequestAuthorizeSignature.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/filters/AWSRequestAuthorizeSignature.java
index 86c1bb9..f494ba1 100644
--- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/filters/AWSRequestAuthorizeSignature.java
+++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/filters/AWSRequestAuthorizeSignature.java
@@ -34,13 +34,13 @@ import org.jclouds.domain.Credentials;
import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpUtils;
import org.jclouds.http.internal.SignatureWire;
-import org.jclouds.s3.filters.RequestAuthorizeSignature;
+import org.jclouds.s3.filters.RequestAuthorizeSignatureV2;
import com.google.common.base.Supplier;
/** Signs the AWS S3 request, supporting temporary signatures. */
@Singleton
-public class AWSRequestAuthorizeSignature extends RequestAuthorizeSignature {
+public class AWSRequestAuthorizeSignature extends RequestAuthorizeSignatureV2 {
@Inject
public AWSRequestAuthorizeSignature(SignatureWire signatureWire, @Named(PROPERTY_AUTH_TAG) String authTag,
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/providers/aws-s3/src/main/java/org/jclouds/aws/s3/filters/AWSRequestAuthorizeSignatureV4.java
----------------------------------------------------------------------
diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/filters/AWSRequestAuthorizeSignatureV4.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/filters/AWSRequestAuthorizeSignatureV4.java
new file mode 100644
index 0000000..6cb3bf1
--- /dev/null
+++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/filters/AWSRequestAuthorizeSignatureV4.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.aws.s3.filters;
+
+import static org.jclouds.http.utils.Queries.queryParser;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_SIGNATURE_PARAM;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.http.HttpRequest;
+import org.jclouds.s3.filters.Aws4SignerForAuthorizationHeader;
+import org.jclouds.s3.filters.Aws4SignerForChunkedUpload;
+import org.jclouds.s3.filters.Aws4SignerForQueryString;
+import org.jclouds.s3.filters.RequestAuthorizeSignatureV4;
+
+/**
+ * Signs the AWS S3 request, supporting temporary signatures.
+ */
+@Singleton
+public class AWSRequestAuthorizeSignatureV4 extends RequestAuthorizeSignatureV4 {
+
+ @Inject
+ public AWSRequestAuthorizeSignatureV4(Aws4SignerForAuthorizationHeader signerForAuthorizationHeader,
+ Aws4SignerForChunkedUpload signerForChunkedUpload,
+ Aws4SignerForQueryString signerForQueryString) {
+ super(signerForAuthorizationHeader, signerForChunkedUpload, signerForQueryString);
+ }
+
+ @Override
+ protected HttpRequest signForAuthorizationHeader(HttpRequest request) {
+ /*
+ * Only add the Authorization header if the query string doesn't already contain
+ * the 'X-Amz-Signature' parameter, otherwise S3 will fail the request complaining about
+ * duplicate authentication methods. The 'Signature' parameter will be added for signed URLs
+ * with expiration.
+ */
+
+ if (queryParser().apply(request.getEndpoint().getQuery()).containsKey(AMZ_SIGNATURE_PARAM)) {
+ return request;
+ }
+ return super.signForAuthorizationHeader(request);
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerV4ExpectTest.java
----------------------------------------------------------------------
diff --git a/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerV4ExpectTest.java b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerV4ExpectTest.java
new file mode 100644
index 0000000..cdfccf1
--- /dev/null
+++ b/providers/aws-s3/src/test/java/org/jclouds/aws/s3/blobstore/AWSS3BlobSignerV4ExpectTest.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.aws.s3.blobstore;
+
+import static org.jclouds.Constants.PROPERTY_CREDENTIAL;
+import static org.jclouds.Constants.PROPERTY_IDENTITY;
+import static org.testng.Assert.assertEquals;
+
+import java.util.Date;
+import java.util.Properties;
+
+import javax.inject.Named;
+
+import org.jclouds.Constants;
+import org.jclouds.aws.s3.AWSS3ApiMetadata;
+import org.jclouds.aws.s3.AWSS3ProviderMetadata;
+import org.jclouds.aws.s3.blobstore.config.AWSS3BlobStoreContextModule;
+import org.jclouds.aws.s3.config.AWSS3HttpApiModule;
+import org.jclouds.aws.s3.filters.AWSRequestAuthorizeSignatureV4;
+import org.jclouds.blobstore.BlobRequestSigner;
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.date.DateService;
+import org.jclouds.date.TimeStamp;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.providers.ProviderMetadata;
+import org.jclouds.rest.ConfiguresHttpApi;
+import org.jclouds.s3.blobstore.S3BlobSignerExpectTest;
+import org.jclouds.s3.filters.RequestAuthorizeSignature;
+import org.testng.SkipException;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.net.HttpHeaders;
+import com.google.inject.Module;
+import com.google.inject.Scopes;
+
+@Test(groups = "unit", testName = "AWSS3BlobSignerV4ExpectTest")
+public class AWSS3BlobSignerV4ExpectTest extends S3BlobSignerExpectTest {
+ private static final String IDENTITY = "AKIAPAEBI3QI4EXAMPLE";
+ private static final String CREDENTIAL = "oHkkcPcOjJnoAXpjT8GXdNeBjo6Ru7QeFExAmPlE";
+ private static final String TIMESTAMP = "Thu, 03 Feb 2015 07:11:11 GMT";
+
+ private static final String BUCKET_NAME = "test-bucket";
+ private static final String OBJECT_NAME = "ExampleObject.txt";
+ private static final String HOST = BUCKET_NAME + ".s3.amazonaws.com";
+
+ public AWSS3BlobSignerV4ExpectTest() {
+ provider = null;
+ }
+
+ @Override
+ protected HttpRequest getBlobWithTime() {
+ return HttpRequest.builder().method("GET")
+ .endpoint("https://" + HOST + "/" + OBJECT_NAME
+ + "?X-Amz-Algorithm=AWS4-HMAC-SHA256"
+ + "&X-Amz-Credential=AKIAPAEBI3QI4EXAMPLE/20150203/us-east-1/s3/aws4_request"
+ + "&X-Amz-Date=20150203T071111Z"
+ + "&X-Amz-Expires=86400"
+ + "&X-Amz-SignedHeaders=host"
+ + "&X-Amz-Signature=0bafb6a0d99c8b7c39abe5496e9897e8c442b09278f1a647267acb25e8d1c550")
+ .addHeader(HttpHeaders.HOST, HOST)
+ .build();
+ }
+
+ @Test
+ @Override
+ public void testSignGetBlobWithTime() {
+ BlobStore getBlobWithTime = requestsSendResponses(init());
+ HttpRequest compare = getBlobWithTime();
+ HttpRequest signedRequest = getBlobWithTime.getContext().getSigner().signGetBlob(BUCKET_NAME, OBJECT_NAME,
+ 86400l /* seconds */);
+ assertEquals(signedRequest, compare);
+ }
+
+ protected HttpRequest _putBlobWithTime() {
+ return HttpRequest.builder().method("PUT")
+ .endpoint("https://" + HOST + "/" + OBJECT_NAME
+ + "?X-Amz-Algorithm=AWS4-HMAC-SHA256"
+ + "&X-Amz-Credential=AKIAPAEBI3QI4EXAMPLE/20150203/us-east-1/s3/aws4_request"
+ + "&X-Amz-Date=20150203T071111Z"
+ + "&X-Amz-Expires=86400"
+ + "&X-Amz-SignedHeaders=host"
+ + "&X-Amz-Signature=41484fb83e0c51b289907979ff96b2c743f6faf8dc70fca1c6fa78d8aeda132f")
+ .addHeader(HttpHeaders.EXPECT, "100-continue")
+ .addHeader(HttpHeaders.HOST, HOST)
+ .build();
+ }
+
+ @Test
+ @Override
+ public void testSignPutBlobWithTime() throws Exception {
+ BlobStore signPutBloblWithTime = requestsSendResponses(init());
+ Blob blob = signPutBloblWithTime.blobBuilder(OBJECT_NAME).payload(text).contentType("text/plain").build();
+ HttpRequest compare = _putBlobWithTime();
+ compare.setPayload(blob.getPayload());
+ HttpRequest signedRequest = signPutBloblWithTime.getContext().getSigner().signPutBlob(BUCKET_NAME, blob,
+ 86400l /* seconds */);
+ assertEquals(signedRequest, compare);
+ }
+
+ @Override
+ protected HttpRequest putBlob() {
+ throw new SkipException("skip putBlob");
+ }
+
+ @Override
+ public void testSignPutBlob() {
+ throw new SkipException("skip testSignPutBlob");
+ }
+
+ @Override
+ public void testSignGetBlob() {
+ throw new SkipException("skip testSignGetBlob");
+ }
+
+ @Override
+ public void testSignGetBlobWithOptions() {
+ throw new SkipException("skip testSignGetBlobWithOptions");
+ }
+
+ @Override
+ public void testSignRemoveBlob() {
+ throw new SkipException("skip testSignRemoveBlob");
+ }
+
+ @Override
+ protected Module createModule() {
+ return new TestAWSS3SignerV4HttpApiModule();
+ }
+
+ @Override
+ protected Properties setupProperties() {
+ Properties props = super.setupProperties();
+ props.put(PROPERTY_IDENTITY, IDENTITY);
+ props.put(PROPERTY_CREDENTIAL, CREDENTIAL);
+ return props;
+ }
+
+ @Override
+ protected ProviderMetadata createProviderMetadata() {
+ AWSS3ApiMetadata.Builder apiBuilder = new AWSS3ApiMetadata().toBuilder();
+ apiBuilder.defaultModules(ImmutableSet.<Class<? extends Module>>of(TestAWSS3SignerV4HttpApiModule.class,
+ TestAWSS3BlobStoreContextModule.class));
+ return new AWSS3ProviderMetadata().toBuilder().apiMetadata(apiBuilder.build()).build();
+ }
+
+ public static final class TestAWSS3BlobStoreContextModule extends AWSS3BlobStoreContextModule {
+
+ @Override
+ protected void bindRequestSigner() {
+ // replace AWSS3BlobRequestSigner aws s3 with AWSS3BlobRequestSignerV4
+ bind(BlobRequestSigner.class).to(AWSS3BlobRequestSignerV4.class);
+ }
+
+ }
+
+ @ConfiguresHttpApi
+ public static final class TestAWSS3SignerV4HttpApiModule extends AWSS3HttpApiModule {
+ @Override
+ protected void configure() {
+ super.configure();
+ }
+
+ @Override
+ protected void bindRequestSigner() {
+ bind(RequestAuthorizeSignature.class).to(AWSRequestAuthorizeSignatureV4.class).in(Scopes.SINGLETON);
+ }
+
+ @Override
+ @TimeStamp
+ protected String provideTimeStamp(@TimeStamp Supplier<String> cache) {
+ return TIMESTAMP;
+ }
+
+ @Override
+ @TimeStamp
+ protected Supplier<Date> provideTimeStampCacheDate(
+ @Named(Constants.PROPERTY_SESSION_INTERVAL) long seconds,
+ @TimeStamp final Supplier<String> timestamp,
+ final DateService dateService) {
+ return Suppliers.ofInstance(dateService.rfc822DateParse(TIMESTAMP));
+ }
+ }
+}
[2/2] jclouds git commit: JCLOUDS-480: AWS S3 v4 signature
Posted by ga...@apache.org.
JCLOUDS-480: AWS S3 v4 signature
This includes support for chunked uploads.
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/8bddbb49
Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/8bddbb49
Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/8bddbb49
Branch: refs/heads/master
Commit: 8bddbb496a7b1a52f851e3e22626eaf644072973
Parents: c20fcb8
Author: Zhao Jin <zh...@unicall.cc>
Authored: Wed Feb 11 10:55:28 2015 +0800
Committer: Andrew Gaul <ga...@apache.org>
Committed: Wed Jan 13 15:08:18 2016 -0800
----------------------------------------------------------------------
.../main/java/org/jclouds/s3/S3ApiMetadata.java | 4 +
.../org/jclouds/s3/config/S3HttpApiModule.java | 43 +-
.../org/jclouds/s3/filters/Aws4SignerBase.java | 437 +++++++++++++++++++
.../Aws4SignerForAuthorizationHeader.java | 204 +++++++++
.../s3/filters/Aws4SignerForChunkedUpload.java | 254 +++++++++++
.../s3/filters/Aws4SignerForQueryString.java | 150 +++++++
.../jclouds/s3/filters/AwsHostNameUtils.java | 186 ++++++++
.../s3/filters/AwsSignatureV4Constants.java | 151 +++++++
.../s3/filters/ChunkedUploadException.java | 23 +
.../s3/filters/ChunkedUploadPayload.java | 217 +++++++++
.../s3/filters/RequestAuthorizeSignature.java | 234 +---------
.../s3/filters/RequestAuthorizeSignatureV2.java | 264 +++++++++++
.../s3/filters/RequestAuthorizeSignatureV4.java | 114 +++++
.../org/jclouds/s3/reference/S3Constants.java | 1 +
.../s3/filters/AwsHostNameUtilsTest.java | 61 +++
.../filters/RequestAuthorizeSignatureTest.java | 157 -------
.../RequestAuthorizeSignatureV2Test.java | 158 +++++++
...stAuthorizeSignatureV4ChunkedUploadTest.java | 199 +++++++++
.../RequestAuthorizeSignatureV4Test.java | 193 ++++++++
.../jclouds/s3/internal/BaseS3ClientTest.java | 3 +-
.../s3/blobstore/AWSS3BlobRequestSigner.java | 6 +-
.../s3/blobstore/AWSS3BlobRequestSignerV4.java | 66 +++
.../filters/AWSRequestAuthorizeSignature.java | 4 +-
.../filters/AWSRequestAuthorizeSignatureV4.java | 58 +++
.../blobstore/AWSS3BlobSignerV4ExpectTest.java | 201 +++++++++
25 files changed, 2990 insertions(+), 398 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java
index b5c2bac..14fab7b 100644
--- a/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java
+++ b/apis/s3/src/main/java/org/jclouds/s3/S3ApiMetadata.java
@@ -22,6 +22,7 @@ import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
import static org.jclouds.blobstore.reference.BlobStoreConstants.PROPERTY_BLOBSTORE_DIRECTORY_SUFFIX;
import static org.jclouds.blobstore.reference.BlobStoreConstants.PROPERTY_USER_METADATA_PREFIX;
import static org.jclouds.reflect.Reflection2.typeToken;
+import static org.jclouds.s3.reference.S3Constants.PROPERTY_JCLOUDS_S3_CHUNKED_SIZE;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_SERVICE_PATH;
import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
@@ -78,6 +79,9 @@ public class S3ApiMetadata extends BaseHttpApiMetadata {
properties.setProperty(PROPERTY_RELAX_HOSTNAME, "true");
properties.setProperty(PROPERTY_BLOBSTORE_DIRECTORY_SUFFIX, "/");
properties.setProperty(PROPERTY_USER_METADATA_PREFIX, String.format("x-${%s}-meta-", PROPERTY_HEADER_TAG));
+
+ // Chunk size must be at least 8 KB. We recommend a chunk size of a least 64 KB for better performance.
+ properties.setProperty(PROPERTY_JCLOUDS_S3_CHUNKED_SIZE, String.valueOf(64 * 1024));
return properties;
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/config/S3HttpApiModule.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/config/S3HttpApiModule.java b/apis/s3/src/main/java/org/jclouds/s3/config/S3HttpApiModule.java
index 69feb8e..e8eded5 100644
--- a/apis/s3/src/main/java/org/jclouds/s3/config/S3HttpApiModule.java
+++ b/apis/s3/src/main/java/org/jclouds/s3/config/S3HttpApiModule.java
@@ -17,6 +17,7 @@
package org.jclouds.s3.config;
import java.net.URI;
+import java.util.Date;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -33,6 +34,7 @@ import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.date.DateService;
import org.jclouds.date.TimeStamp;
import org.jclouds.http.HttpErrorHandler;
+import org.jclouds.http.HttpRequest;
import org.jclouds.http.HttpRetryHandler;
import org.jclouds.http.annotation.ClientError;
import org.jclouds.http.annotation.Redirection;
@@ -46,6 +48,7 @@ import org.jclouds.s3.S3Client;
import org.jclouds.s3.blobstore.functions.BucketsToStorageMetadata;
import org.jclouds.s3.domain.BucketMetadata;
import org.jclouds.s3.filters.RequestAuthorizeSignature;
+import org.jclouds.s3.filters.RequestAuthorizeSignatureV2;
import org.jclouds.s3.functions.GetRegionForBucket;
import org.jclouds.s3.handlers.ParseS3ErrorFromXmlContent;
import org.jclouds.s3.handlers.S3RedirectionRetryHandler;
@@ -181,13 +184,26 @@ public class S3HttpApiModule<S extends S3Client> extends AWSHttpApiModule<S> {
}
protected void bindRequestSigner() {
- bind(RequestAuthorizeSignature.class).in(Scopes.SINGLETON);
+ bind(RequestAuthorizeSignature.class).to(RequestAuthorizeSignatureV2.class).in(Scopes.SINGLETON);
}
@Provides
@Singleton
protected final RequestSigner provideRequestSigner(RequestAuthorizeSignature in) {
- return in;
+ if (in instanceof RequestSigner) {
+ return (RequestSigner) in;
+ }
+ return new RequestSigner() {
+ @Override
+ public String createStringToSign(HttpRequest input) {
+ return null;
+ }
+
+ @Override
+ public String sign(String toSign) {
+ return null;
+ }
+ };
}
@Override
@@ -222,4 +238,27 @@ public class S3HttpApiModule<S extends S3Client> extends AWSHttpApiModule<S> {
}
}, seconds, TimeUnit.SECONDS);
}
+
+ @Provides
+ @TimeStamp
+ protected Date provideTimeStampDate(@TimeStamp Supplier<Date> cache) {
+ return cache.get();
+ }
+
+ /**
+ * borrowing concurrency code to ensure that caching takes place properly
+ */
+ @Provides
+ @TimeStamp
+ @Singleton
+ protected Supplier<Date> provideTimeStampCacheDate(
+ @Named(Constants.PROPERTY_SESSION_INTERVAL) long seconds,
+ @TimeStamp final Supplier<String> timestamp,
+ final DateService dateService) {
+ return Suppliers.memoizeWithExpiration(new Supplier<Date>() {
+ public Date get() {
+ return dateService.rfc822DateParse(timestamp.get());
+ }
+ }, seconds, TimeUnit.SECONDS);
+ }
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerBase.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerBase.java b/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerBase.java
new file mode 100644
index 0000000..109df5e
--- /dev/null
+++ b/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerBase.java
@@ -0,0 +1,437 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.io.BaseEncoding.base16;
+import static com.google.common.io.ByteStreams.readBytes;
+import static org.jclouds.crypto.Macs.asByteProcessor;
+import static org.jclouds.http.utils.Queries.queryParser;
+import static org.jclouds.util.Strings2.toInputStream;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.security.InvalidKeyException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Locale;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TimeZone;
+
+import javax.inject.Inject;
+import javax.xml.ws.http.HTTPException;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.escape.Escaper;
+import com.google.common.hash.Hashing;
+import com.google.common.hash.HashingInputStream;
+import com.google.common.io.ByteProcessor;
+import com.google.common.io.ByteSource;
+import com.google.common.io.ByteStreams;
+import com.google.common.net.HttpHeaders;
+import com.google.common.net.PercentEscaper;
+import com.google.inject.ImplementedBy;
+import org.jclouds.crypto.Crypto;
+import org.jclouds.domain.Credentials;
+import org.jclouds.http.HttpException;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.internal.SignatureWire;
+import org.jclouds.io.Payload;
+import org.jclouds.providers.ProviderMetadata;
+
+/**
+ * Common methods and properties for all AWS4 signer variants
+ */
+public abstract class Aws4SignerBase {
+ private static final TimeZone GMT = TimeZone.getTimeZone("GMT");
+ protected static final DateFormat timestampFormat;
+ protected static final DateFormat dateFormat;
+
+ // Do not URL-encode any of the unreserved characters that RFC 3986 defines:
+ // A-Z, a-z, 0-9, hyphen (-), underscore (_), period (.), and tilde (~).
+ private static final Escaper AWS_URL_PARAMETER_ESCAPER;
+
+ static {
+ timestampFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'");
+ timestampFormat.setTimeZone(GMT);
+
+ dateFormat = new SimpleDateFormat("yyyyMMdd");
+ dateFormat.setTimeZone(GMT);
+
+ AWS_URL_PARAMETER_ESCAPER = new PercentEscaper("-_.~", false);
+ }
+
+ // Specifying a default for how to parse the service and region in this way allows
+ // tests or other downstream services to not have to use guice overrides.
+ @ImplementedBy(ServiceAndRegion.AWSServiceAndRegion.class)
+ public interface ServiceAndRegion {
+ String service();
+
+ String region(String host);
+
+ final class AWSServiceAndRegion implements ServiceAndRegion {
+ private final String service;
+
+ @Inject
+ AWSServiceAndRegion(ProviderMetadata provider) {
+ this(provider.getEndpoint());
+ }
+
+ AWSServiceAndRegion(String endpoint) {
+ this.service = AwsHostNameUtils.parseServiceName(URI.create(checkNotNull(endpoint, "endpoint")));
+ }
+
+ @Override
+ public String service() {
+ return service;
+ }
+
+ @Override
+ public String region(String host) {
+ return AwsHostNameUtils.parseRegionName(host, service());
+ }
+ }
+ }
+
+ protected final String headerTag;
+ protected final ServiceAndRegion serviceAndRegion;
+ protected final SignatureWire signatureWire;
+ protected final Supplier<Credentials> creds;
+ protected final Supplier<Date> timestampProvider;
+ protected final Crypto crypto;
+
+
+ protected Aws4SignerBase(SignatureWire signatureWire, String headerTag,
+ Supplier<Credentials> creds, Supplier<Date> timestampProvider,
+ ServiceAndRegion serviceAndRegion, Crypto crypto) {
+ this.signatureWire = signatureWire;
+ this.headerTag = headerTag;
+ this.creds = creds;
+ this.timestampProvider = timestampProvider;
+ this.serviceAndRegion = serviceAndRegion;
+ this.crypto = crypto;
+ }
+
+ protected String getContentType(HttpRequest request) {
+ Payload payload = request.getPayload();
+
+ // Default Content Type
+ String contentType = request.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE);
+ if (payload != null
+ && payload.getContentMetadata() != null
+ && payload.getContentMetadata().getContentType() != null) {
+ contentType = payload.getContentMetadata().getContentType();
+ }
+ return contentType;
+ }
+
+ protected String getContentLength(HttpRequest request) {
+ Payload payload = request.getPayload();
+
+ // Default Content Type
+ String contentLength = request.getFirstHeaderOrNull(HttpHeaders.CONTENT_LENGTH);
+ if (payload != null
+ && payload.getContentMetadata() != null
+ && payload.getContentMetadata().getContentType() != null) {
+ Long length = payload.getContentMetadata().getContentLength();
+ contentLength =
+ length == null ? contentLength : String.valueOf(payload.getContentMetadata().getContentLength());
+ }
+ return contentLength;
+ }
+
+ // append all of 'x-amz-*' headers
+ protected void appendAmzHeaders(HttpRequest request,
+ ImmutableMap.Builder<String, String> signedHeadersBuilder) {
+ for (Map.Entry<String, String> header : request.getHeaders().entries()) {
+ String key = header.getKey();
+ if (key.startsWith("x-" + headerTag + "-")) {
+ signedHeadersBuilder.put(key.toLowerCase(), header.getValue());
+ }
+ }
+ }
+
+ /**
+ * caluclate AWS signature key.
+ * <p>
+ * <code>
+ * DateKey = hmacSHA256(datestamp, "AWS4"+ secretKey)
+ * <br>
+ * DateRegionKey = hmacSHA256(region, DateKey)
+ * <br>
+ * DateRegionServiceKey = hmacSHA256(service, DateRegionKey)
+ * <br>
+ * SigningKey = hmacSHA256("aws4_request", DateRegionServiceKey)
+ * <br>
+ * <p/>
+ * </code>
+ * </p>
+ *
+ * @param secretKey AWS access secret key
+ * @param datestamp date yyyyMMdd
+ * @param region AWS region
+ * @param service AWS service
+ * @return SigningKey
+ */
+ protected byte[] signatureKey(String secretKey, String datestamp, String region, String service) {
+ byte[] kSecret = ("AWS4" + secretKey).getBytes(UTF_8);
+ byte[] kDate = hmacSHA256(datestamp, kSecret);
+ byte[] kRegion = hmacSHA256(region, kDate);
+ byte[] kService = hmacSHA256(service, kRegion);
+ byte[] kSigning = hmacSHA256("aws4_request", kService);
+ return kSigning;
+ }
+
+ /**
+ * hmac sha256
+ *
+ * @param toSign string to sign
+ * @param key hash key
+ */
+ protected byte[] hmacSHA256(String toSign, byte[] key) {
+ try {
+ return readBytes(toInputStream(toSign), hmacSHA256(crypto, key));
+ } catch (IOException e) {
+ throw new HttpException("read sign error", e);
+ } catch (InvalidKeyException e) {
+ throw new HttpException("invalid key", e);
+ }
+ }
+
+ public static ByteProcessor<byte[]> hmacSHA256(Crypto crypto, byte[] signatureKey) throws InvalidKeyException {
+ return asByteProcessor(crypto.hmacSHA256(signatureKey));
+ }
+
+ /**
+ * hash input with sha256
+ *
+ * @param input
+ * @return hash result
+ * @throws HTTPException
+ */
+ public static byte[] hash(InputStream input) throws HTTPException {
+ HashingInputStream his = new HashingInputStream(Hashing.sha256(), input);
+ try {
+ ByteStreams.copy(his, ByteStreams.nullOutputStream());
+ return his.hash().asBytes();
+ } catch (IOException e) {
+ throw new HttpException("Unable to compute hash while signing request: " + e.getMessage(), e);
+ }
+ }
+
+ /**
+ * hash input with sha256
+ *
+ * @param bytes input bytes
+ * @return hash result
+ * @throws HTTPException
+ */
+ public static byte[] hash(byte[] bytes) throws HTTPException {
+ try {
+ return ByteSource.wrap(bytes).hash(Hashing.sha256()).asBytes();
+ } catch (IOException e) {
+ throw new HttpException("Unable to compute hash while signing request: " + e.getMessage(), e);
+ }
+ }
+
+
+ /**
+ * hash string (encoding UTF_8) with sha256
+ *
+ * @param input input stream
+ * @return hash result
+ * @throws HTTPException
+ */
+ public static byte[] hash(String input) throws HTTPException {
+ return hash(new ByteArrayInputStream(input.getBytes(UTF_8)));
+ }
+
+ /**
+ * Examines the specified query string parameters and returns a
+ * canonicalized form.
+ * <p/>
+ * The canonicalized query string is formed by first sorting all the query
+ * string parameters, then URI encoding both the key and value and then
+ * joining them, in order, separating key value pairs with an '&'.
+ *
+ * @param queryString The query string parameters to be canonicalized.
+ * @return A canonicalized form for the specified query string parameters.
+ */
+ protected String getCanonicalizedQueryString(String queryString) {
+ Multimap<String, String> params = queryParser().apply(queryString);
+ SortedMap<String, String> sorted = Maps.newTreeMap();
+ if (params == null) {
+ return "";
+ }
+ Iterator<Map.Entry<String, String>> pairs = params.entries().iterator();
+ while (pairs.hasNext()) {
+ Map.Entry<String, String> pair = pairs.next();
+ String key = pair.getKey();
+ String value = pair.getValue();
+ sorted.put(urlEncode(key), urlEncode(value));
+ }
+
+ return Joiner.on("&").withKeyValueSeparator("=").join(sorted);
+ }
+
+ /**
+ * Encode a string for use in the path of a URL; uses URLEncoder.encode,
+ * (which encodes a string for use in the query portion of a URL), then
+ * applies some postfilters to fix things up per the RFC. Can optionally
+ * handle strings which are meant to encode a path (ie include '/'es
+ * which should NOT be escaped).
+ *
+ * @param value the value to encode
+ * @return the encoded value
+ */
+ public static String urlEncode(final String value) {
+ if (value == null) {
+ return "";
+ }
+ return AWS_URL_PARAMETER_ESCAPER.escape(value);
+ }
+
+ /**
+ * Lowercase base 16 encoding.
+ *
+ * @param bytes bytes
+ * @return base16 lower case hex string.
+ */
+ public static String hex(final byte[] bytes) {
+ return base16().lowerCase().encode(bytes);
+ }
+
+ /**
+ * Create a Canonical Request to sign
+ * <h4>Canonical Request</h4>
+ * <p>
+ * <code>
+ * <HTTPMethod>\n
+ * <br>
+ * <CanonicalURI>\n
+ * <br>
+ * <CanonicalQueryString>\n
+ * <br>
+ * <CanonicalHeaders>\n
+ * <br>
+ * <SignedHeaders>\n
+ * <br>
+ * <HashedPayload>
+ * </code>
+ * </p>
+ * <p><b>HTTPMethod</b> is one of the HTTP methods, for example GET, PUT, HEAD, and DELETE.</p>
+ * <p><b>CanonicalURI</b> is the URI-encoded version of the absolute path component of the URI—everything starting
+ * with the "/" that follows the domain name and up to the end of the string or to the question mark character ('?')
+ * if you have query string parameters.</p>
+ * <p><b>CanonicalQueryString</b> specifies the URI-encoded query string parameters. You URI-encode name and values
+ * individually. You must also sort the parameters in the canonical query string alphabetically by key name.
+ * The sorting occurs after encoding.</p>
+ * <p><b>CanonicalHeaders</b> is a list of request headers with their values. Individual header name and value pairs are
+ * separated by the newline character ("\n"). Header names must be in lowercase. Header value must be trim space.
+ * <br>
+ * The <b>CanonicalHeaders</b> list must include the following:
+ * HTTP host header.
+ * If the Content-Type header is present in the request, it must be added to the CanonicalHeaders list.
+ * Any x-amz-* headers that you plan to include in your request must also be added.</p>
+ * <p><b>SignedHeaders</b> is an alphabetically sorted, semicolon-separated list of lowercase request header names.
+ * The request headers in the list are the same headers that you included in the CanonicalHeaders string.</p>
+ * <p><b>HashedPayload</b> is the hexadecimal value of the SHA256 hash of the request payload. </p>
+ * <p>If there is no payload in the request, you compute a hash of the empty string as follows:
+ * <code>Hex(SHA256Hash(""))</code> The hash returns the following value:
+ * e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 </p>
+ *
+ * @param method http request method
+ * @param endpoint http request endpoing
+ * @param signedHeaders signed headers
+ * @param timestamp ISO8601 timestamp
+ * @param credentialScope credential scope
+ * @return string to sign
+ */
+ protected String createStringToSign(String method, URI endpoint, Map<String, String> signedHeaders,
+ String timestamp, String credentialScope, String hashedPayload) {
+
+ // lower case header keys
+ Map<String, String> lowerCaseHeaders = lowerCaseNaturalOrderKeys(signedHeaders);
+
+ StringBuilder canonicalRequest = new StringBuilder();
+
+ // HTTPRequestMethod + '\n' +
+ canonicalRequest.append(method).append("\n");
+
+ // CanonicalURI + '\n' +
+ canonicalRequest.append(endpoint.getPath()).append("\n");
+
+ // CanonicalQueryString + '\n' +
+ if (endpoint.getQuery() != null) {
+ canonicalRequest.append(getCanonicalizedQueryString(endpoint.getQuery()));
+ }
+ canonicalRequest.append("\n");
+
+ // CanonicalHeaders + '\n' +
+ for (Map.Entry<String, String> entry : lowerCaseHeaders.entrySet()) {
+ canonicalRequest.append(entry.getKey()).append(':').append(entry.getValue()).append('\n');
+ }
+ canonicalRequest.append("\n");
+
+ // SignedHeaders + '\n' +
+ canonicalRequest.append(Joiner.on(';').join(lowerCaseHeaders.keySet())).append('\n');
+
+ // HexEncode(Hash(Payload))
+ canonicalRequest.append(hashedPayload);
+
+ signatureWire.getWireLog().debug("<<", canonicalRequest);
+
+ // Create a String to Sign
+ StringBuilder toSign = new StringBuilder();
+ // Algorithm + '\n' +
+ toSign.append("AWS4-HMAC-SHA256").append('\n');
+ // RequestDate + '\n' +
+ toSign.append(timestamp).append('\n');
+ // CredentialScope + '\n' +
+ toSign.append(credentialScope).append('\n');
+ // HexEncode(Hash(CanonicalRequest))
+ toSign.append(hex(hash(canonicalRequest.toString())));
+
+ return toSign.toString();
+ }
+
+ /**
+ * change the keys but keep the values in-tact.
+ *
+ * @param in input map to transform
+ * @return immutableSortedMap with the new lowercase keys.
+ */
+ protected static Map<String, String> lowerCaseNaturalOrderKeys(Map<String, String> in) {
+ checkNotNull(in, "input map");
+ ImmutableSortedMap.Builder<String, String> returnVal = ImmutableSortedMap.<String, String>naturalOrder();
+ for (Map.Entry<String, String> entry : in.entrySet())
+ returnVal.put(entry.getKey().toLowerCase(Locale.US), entry.getValue());
+ return returnVal.build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForAuthorizationHeader.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForAuthorizationHeader.java b/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForAuthorizationHeader.java
new file mode 100644
index 0000000..31525e7
--- /dev/null
+++ b/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForAuthorizationHeader.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.net.HttpHeaders;
+import com.google.inject.Inject;
+import org.jclouds.aws.domain.SessionCredentials;
+import org.jclouds.crypto.Crypto;
+import org.jclouds.date.TimeStamp;
+import org.jclouds.domain.Credentials;
+import org.jclouds.http.HttpException;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.internal.SignatureWire;
+import org.jclouds.io.Payload;
+import org.jclouds.location.Provider;
+import org.jclouds.util.Closeables2;
+
+import javax.inject.Named;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Date;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.io.BaseEncoding.base16;
+import static com.google.common.net.HttpHeaders.AUTHORIZATION;
+import static com.google.common.net.HttpHeaders.CONTENT_MD5;
+import static com.google.common.net.HttpHeaders.DATE;
+import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_ALGORITHM_HMAC_SHA256;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_CONTENT_SHA256_HEADER;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_DATE_HEADER;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_SECURITY_TOKEN_HEADER;
+import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
+
+/**
+ * AWS4 signer sign requests to Amazon S3 using an 'Authorization' header.
+ */
+public class Aws4SignerForAuthorizationHeader extends Aws4SignerBase {
+ @Inject
+ public Aws4SignerForAuthorizationHeader(SignatureWire signatureWire,
+ @Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
+ @Named(PROPERTY_HEADER_TAG) String headerTag,
+ @Provider Supplier<Credentials> creds, @TimeStamp Supplier<Date> timestampProvider,
+ ServiceAndRegion serviceAndRegion, Crypto crypto) {
+ super(signatureWire, headerTag, creds, timestampProvider, serviceAndRegion, crypto);
+ }
+
+ protected HttpRequest sign(HttpRequest request) throws HttpException {
+ checkNotNull(request, "request is not ready to sign");
+ checkNotNull(request.getEndpoint(), "request is not ready to sign, request.endpoint not present.");
+
+ // get host from request endpoint.
+ String host = request.getEndpoint().getHost();
+
+ Date date = timestampProvider.get();
+ String timestamp = timestampFormat.format(date);
+ String datestamp = dateFormat.format(date);
+
+ String service = serviceAndRegion.service();
+ String region = serviceAndRegion.region(host);
+ String credentialScope = Joiner.on('/').join(datestamp, region, service, "aws4_request");
+
+ HttpRequest.Builder<?> requestBuilder = request.toBuilder() //
+ .removeHeader(AUTHORIZATION) // remove Authorization
+ .removeHeader(CONTENT_MD5) // aws s3 not allowed Content-MD5, use specs x-amz-content-sha256
+ .removeHeader(DATE); // remove date
+
+ ImmutableMap.Builder<String, String> signedHeadersBuilder = ImmutableSortedMap.<String, String>naturalOrder();
+
+ // Content Type
+ // content-type is not a required signing param. However, examples use this, so we include it to ease testing.
+ String contentType = getContentType(request);
+ if (!Strings.isNullOrEmpty(contentType)) {
+ requestBuilder.replaceHeader(HttpHeaders.CONTENT_TYPE, contentType);
+ signedHeadersBuilder.put(HttpHeaders.CONTENT_TYPE.toLowerCase(), contentType);
+ }
+
+ // Content-Length for PUT or POST request http method
+ String contentLength = getContentLength(request);
+ if (!Strings.isNullOrEmpty(contentLength)) {
+ requestBuilder.replaceHeader(HttpHeaders.CONTENT_LENGTH, contentLength);
+ signedHeadersBuilder.put(HttpHeaders.CONTENT_LENGTH.toLowerCase(), contentLength);
+ }
+
+ // host
+ requestBuilder.replaceHeader(HttpHeaders.HOST, host);
+ signedHeadersBuilder.put(HttpHeaders.HOST.toLowerCase(), host);
+
+ // user-agent
+ if (request.getHeaders().containsKey(HttpHeaders.USER_AGENT)) {
+ signedHeadersBuilder.put(HttpHeaders.USER_AGENT.toLowerCase(),
+ request.getFirstHeaderOrNull(HttpHeaders.USER_AGENT));
+ }
+
+ // all x-amz-* headers
+ appendAmzHeaders(request, signedHeadersBuilder);
+
+ // x-amz-security-token
+ Credentials credentials = creds.get();
+ if (credentials instanceof SessionCredentials) {
+ String token = SessionCredentials.class.cast(credentials).getSessionToken();
+ requestBuilder.replaceHeader(AMZ_SECURITY_TOKEN_HEADER, token);
+ signedHeadersBuilder.put(AMZ_SECURITY_TOKEN_HEADER.toLowerCase(), token);
+ }
+
+ // x-amz-content-sha256
+ String contentSha256 = getPayloadHash(request);
+ requestBuilder.replaceHeader(AMZ_CONTENT_SHA256_HEADER, contentSha256);
+ signedHeadersBuilder.put(AMZ_CONTENT_SHA256_HEADER.toLowerCase(), contentSha256);
+
+ // put x-amz-date
+ requestBuilder.replaceHeader(AMZ_DATE_HEADER, timestamp);
+ signedHeadersBuilder.put(AMZ_DATE_HEADER.toLowerCase(), timestamp);
+
+ ImmutableMap<String, String> signedHeaders = signedHeadersBuilder.build();
+
+ String stringToSign = createStringToSign(request.getMethod(), request.getEndpoint(), signedHeaders, timestamp,
+ credentialScope, contentSha256);
+ signatureWire.getWireLog().debug("<< " + stringToSign);
+
+ byte[] signatureKey = signatureKey(credentials.credential, datestamp, region, service);
+ String signature = base16().lowerCase().encode(hmacSHA256(stringToSign, signatureKey));
+
+ StringBuilder authorization = new StringBuilder(AMZ_ALGORITHM_HMAC_SHA256).append(" ");
+ authorization.append("Credential=").append(Joiner.on("/").join(credentials.identity, credentialScope))
+ .append(", ");
+ authorization.append("SignedHeaders=").append(Joiner.on(";").join(signedHeaders.keySet()))
+ .append(", ");
+ authorization.append("Signature=").append(signature);
+ return requestBuilder.replaceHeader(HttpHeaders.AUTHORIZATION, authorization.toString()).build();
+ }
+
+ protected String getPayloadHash(HttpRequest request) {
+ Payload payload = request.getPayload();
+ if (payload == null) {
+ // when payload is null.
+ return getEmptyPayloadContentHash();
+ }
+ return calculatePayloadContentHash(payload);
+ }
+
+ /**
+ * The hash returns the following value: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+ */
+ protected String getEmptyPayloadContentHash() {
+ return base16().lowerCase().encode(hash(new ByteArrayInputStream(new byte[0])));
+ }
+
+ /**
+ * in this time, payload ContentMetadata provided content hash md5, but aws required sha256.
+ */
+ protected String calculatePayloadContentHash(Payload payload) {
+ // use payload stream calculate content sha256
+ InputStream payloadStream;
+ try {
+ payloadStream = payload.openStream();
+ } catch (IOException e) {
+ throw new HttpException("unable to open payload stream to calculate AWS4 signature.");
+ }
+ try {
+ return base16().lowerCase().encode(hash(payloadStream));
+ } finally {
+ closeOrResetPayloadStream(payloadStream, payload.isRepeatable());
+ }
+ }
+
+ // some times, when use Multipart Payload and a part can not be repeatable, will happen some error...
+ void closeOrResetPayloadStream(InputStream payloadStream, boolean repeatable) {
+ // if payload stream can repeatable.
+ if (repeatable) {
+ Closeables2.closeQuietly(payloadStream);
+ } else {
+ try {
+ // reset unrepeatable payload stream
+ payloadStream.reset();
+ } catch (IOException e) {
+ // reset payload stream
+ throw new HttpException(
+ "unable to reset unrepeatable payload stream after calculating AWS4 signature.");
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForChunkedUpload.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForChunkedUpload.java b/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForChunkedUpload.java
new file mode 100644
index 0000000..e4dd6eb
--- /dev/null
+++ b/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForChunkedUpload.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_ALGORITHM_HMAC_SHA256;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_CONTENT_SHA256_HEADER;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_DATE_HEADER;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_DECODED_CONTENT_LENGTH_HEADER;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_SECURITY_TOKEN_HEADER;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.CHUNK_SIGNATURE_HEADER;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.CLRF;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.CONTENT_ENCODING_HEADER_AWS_CHUNKED;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.SIGNATURE_LENGTH;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.STREAMING_BODY_SHA256;
+import static org.jclouds.s3.reference.S3Constants.PROPERTY_JCLOUDS_S3_CHUNKED_SIZE;
+import static org.jclouds.util.Strings2.toInputStream;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.io.ByteStreams.readBytes;
+import static com.google.common.net.HttpHeaders.AUTHORIZATION;
+import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
+import static com.google.common.net.HttpHeaders.CONTENT_MD5;
+import static com.google.common.net.HttpHeaders.DATE;
+
+import java.io.IOException;
+import java.security.InvalidKeyException;
+import java.util.Date;
+
+import javax.inject.Named;
+
+import org.jclouds.aws.domain.SessionCredentials;
+import org.jclouds.crypto.Crypto;
+import org.jclouds.date.TimeStamp;
+import org.jclouds.domain.Credentials;
+import org.jclouds.http.HttpException;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.internal.SignatureWire;
+import org.jclouds.io.Payload;
+import org.jclouds.location.Provider;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.io.ByteProcessor;
+import com.google.common.net.HttpHeaders;
+import com.google.inject.Inject;
+
+/**
+ * AWS4 signer sign 'chunked' uploads.
+ */
+public class Aws4SignerForChunkedUpload extends Aws4SignerBase {
+
+ private final int userDataBlockSize;
+
+ @Inject
+ public Aws4SignerForChunkedUpload(SignatureWire signatureWire,
+ @Named(PROPERTY_HEADER_TAG) String headerTag,
+ @Named(PROPERTY_JCLOUDS_S3_CHUNKED_SIZE) int userDataBlockSize,
+ @Provider Supplier<Credentials> creds, @TimeStamp Supplier<Date> timestampProvider,
+ ServiceAndRegion serviceAndRegion, Crypto crypto) {
+ super(signatureWire, headerTag, creds, timestampProvider, serviceAndRegion, crypto);
+ this.userDataBlockSize = userDataBlockSize;
+ }
+
+ protected HttpRequest sign(HttpRequest request) throws HttpException {
+ checkNotNull(request, "request is not ready to sign");
+ checkNotNull(request.getEndpoint(), "request is not ready to sign, request.endpoint not present.");
+
+ Payload payload = request.getPayload();
+ // chunked upload required content-length.
+ Long contentLength = payload.getContentMetadata().getContentLength();
+
+ // check contentLength not null
+ checkNotNull(contentLength, "request is not ready to sign, payload contentLength not present.");
+
+ // get host from request endpoint.
+ String host = request.getEndpoint().getHost();
+
+ Date date = timestampProvider.get();
+ String timestamp = timestampFormat.format(date);
+ String datestamp = dateFormat.format(date);
+
+ String service = serviceAndRegion.service();
+ String region = serviceAndRegion.region(host);
+ String credentialScope = Joiner.on('/').join(datestamp, region, service, "aws4_request");
+
+ HttpRequest.Builder<?> requestBuilder = request.toBuilder() //
+ .removeHeader(AUTHORIZATION) // remove Authorization
+ .removeHeader(CONTENT_MD5) // aws s3 not allowed Content-MD5, use aws specs x-amz-content-sha256
+ .removeHeader(DATE) // remove Date
+ .removeHeader(CONTENT_LENGTH); // remove Content-Length
+
+ ImmutableMap.Builder<String, String> signedHeadersBuilder = ImmutableSortedMap.<String, String>naturalOrder();
+
+ // content-encoding
+ requestBuilder.replaceHeader(HttpHeaders.CONTENT_ENCODING, CONTENT_ENCODING_HEADER_AWS_CHUNKED);
+ signedHeadersBuilder.put(HttpHeaders.CONTENT_ENCODING.toLowerCase(), CONTENT_ENCODING_HEADER_AWS_CHUNKED);
+
+
+ // x-amz-decoded-content-length
+ requestBuilder.replaceHeader(AMZ_DECODED_CONTENT_LENGTH_HEADER, contentLength.toString());
+ signedHeadersBuilder.put(AMZ_DECODED_CONTENT_LENGTH_HEADER.toLowerCase(), contentLength.toString());
+
+ // how big is the overall request stream going to be once we add the signature
+ // 'headers' to each chunk?
+ long totalLength = calculateChunkedContentLength(contentLength, userDataBlockSize);
+ requestBuilder.replaceHeader(CONTENT_LENGTH, Long.toString(totalLength));
+ signedHeadersBuilder.put(CONTENT_LENGTH.toLowerCase(), Long.toString(totalLength));
+
+ // Content Type
+ // content-type is not a required signing param. However, examples use this, so we include it to ease testing.
+ String contentType = getContentType(request);
+ if (!Strings.isNullOrEmpty(contentType)) {
+ requestBuilder.replaceHeader(HttpHeaders.CONTENT_TYPE, contentType);
+ signedHeadersBuilder.put(HttpHeaders.CONTENT_TYPE.toLowerCase(), contentType);
+ } else {
+ requestBuilder.removeHeader(HttpHeaders.CONTENT_TYPE);
+ }
+
+ // host
+ requestBuilder.replaceHeader(HttpHeaders.HOST, host);
+ signedHeadersBuilder.put(HttpHeaders.HOST.toLowerCase(), host);
+
+ // user-agent, not a required signing param
+ if (request.getHeaders().containsKey(HttpHeaders.USER_AGENT)) {
+ signedHeadersBuilder.put(HttpHeaders.USER_AGENT.toLowerCase(),
+ request.getFirstHeaderOrNull(HttpHeaders.USER_AGENT));
+ }
+
+ // all x-amz-* headers
+ appendAmzHeaders(request, signedHeadersBuilder);
+
+ // x-amz-security-token
+ Credentials credentials = creds.get();
+ if (credentials instanceof SessionCredentials) {
+ String token = SessionCredentials.class.cast(credentials).getSessionToken();
+ requestBuilder.replaceHeader(AMZ_SECURITY_TOKEN_HEADER, token);
+ signedHeadersBuilder.put(AMZ_SECURITY_TOKEN_HEADER.toLowerCase(), token);
+ }
+
+ // x-amz-content-sha256
+ String contentSha256 = getPayloadHash();
+ requestBuilder.replaceHeader(AMZ_CONTENT_SHA256_HEADER, contentSha256);
+ signedHeadersBuilder.put(AMZ_CONTENT_SHA256_HEADER.toLowerCase(), contentSha256);
+
+ // put x-amz-date
+ requestBuilder.replaceHeader(AMZ_DATE_HEADER, timestamp);
+ signedHeadersBuilder.put(AMZ_DATE_HEADER.toLowerCase(), timestamp);
+
+ ImmutableMap<String, String> signedHeaders = signedHeadersBuilder.build();
+
+ String stringToSign = createStringToSign(request.getMethod(), request.getEndpoint(), signedHeaders, timestamp,
+ credentialScope, contentSha256);
+ signatureWire.getWireLog().debug("<< " + stringToSign);
+
+ byte[] signatureKey = signatureKey(credentials.credential, datestamp, region, service);
+
+ // init hmacSHA256 processor for seed signature and chunked block signature
+ ByteProcessor<byte[]> hmacSHA256;
+ try {
+ hmacSHA256 = hmacSHA256(crypto, signatureKey);
+ } catch (InvalidKeyException e) {
+ throw new ChunkedUploadException("invalid key", e);
+ }
+
+ // Calculating the Seed Signature
+ String signature;
+ try {
+ signature = hex(readBytes(toInputStream(stringToSign), hmacSHA256));
+ } catch (IOException e) {
+ throw new ChunkedUploadException("hmac sha256 seed signature error", e);
+ }
+
+ StringBuilder authorization = new StringBuilder(AMZ_ALGORITHM_HMAC_SHA256).append(" ");
+ authorization.append("Credential=").append(Joiner.on("/").join(credentials.identity, credentialScope))
+ .append(", ");
+ authorization.append("SignedHeaders=").append(Joiner.on(";").join(signedHeaders.keySet()))
+ .append(", ");
+ authorization.append("Signature=").append(signature);
+
+ // replace request payload with chunked upload payload
+ ChunkedUploadPayload chunkedPayload = new ChunkedUploadPayload(payload, userDataBlockSize, timestamp,
+ credentialScope, hmacSHA256, signature);
+
+ return request.toBuilder()
+ .replaceHeader(HttpHeaders.AUTHORIZATION, authorization.toString())
+ .payload(chunkedPayload)
+ .build();
+
+ }
+
+ // for seed signature, value: STREAMING-AWS4-HMAC-SHA256-PAYLOAD
+ protected String getPayloadHash() {
+ return STREAMING_BODY_SHA256;
+ }
+
+ /**
+ * Calculates the expanded payload size of our data when it is chunked
+ *
+ * @param originalLength The true size of the data payload to be uploaded
+ * @param chunkSize The size of each chunk we intend to send; each chunk will be
+ * prefixed with signed header data, expanding the overall size
+ * by a determinable amount
+ * @return The overall payload size to use as content-length on a chunked
+ * upload
+ */
+ public static long calculateChunkedContentLength(long originalLength, long chunkSize) {
+ checkArgument(originalLength > 0, "Nonnegative content length expected.");
+
+ long maxSizeChunks = originalLength / chunkSize;
+ long remainingBytes = originalLength % chunkSize;
+ return maxSizeChunks * calculateChunkHeaderLength(chunkSize)
+ + (remainingBytes > 0 ? calculateChunkHeaderLength(remainingBytes) : 0)
+ + calculateChunkHeaderLength(0);
+ }
+
+ /**
+ * Returns the size of a chunk header, which only varies depending on the
+ * selected chunk size
+ *
+ * @param chunkDataSize The intended size of each chunk; this is placed into the chunk
+ * header
+ * @return The overall size of the header that will prefix the user data in
+ * each chunk
+ */
+ private static long calculateChunkHeaderLength(long chunkDataSize) {
+ return Long.toHexString(chunkDataSize).length()
+ + CHUNK_SIGNATURE_HEADER.length()
+ + SIGNATURE_LENGTH
+ + CLRF.length()
+ + chunkDataSize
+ + CLRF.length();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForQueryString.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForQueryString.java b/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForQueryString.java
new file mode 100644
index 0000000..2f1abe4
--- /dev/null
+++ b/apis/s3/src/main/java/org/jclouds/s3/filters/Aws4SignerForQueryString.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_ALGORITHM_PARAM;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_CONTENT_SHA256_HEADER;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_CREDENTIAL_PARAM;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_DATE_HEADER;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_DATE_PARAM;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_EXPIRES_PARAM;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_SECURITY_TOKEN_PARAM;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_SIGNATURE_PARAM;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AMZ_SIGNEDHEADERS_PARAM;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.AUTHORIZATION_HEADER;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.UNSIGNED_PAYLOAD;
+import static org.jclouds.s3.reference.S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.io.BaseEncoding.base16;
+
+import java.util.Date;
+
+import javax.inject.Named;
+
+import org.jclouds.aws.domain.SessionCredentials;
+import org.jclouds.crypto.Crypto;
+import org.jclouds.date.TimeStamp;
+import org.jclouds.domain.Credentials;
+import org.jclouds.http.HttpException;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.Uris;
+import org.jclouds.http.internal.SignatureWire;
+import org.jclouds.location.Provider;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.inject.Inject;
+
+/**
+ * AWS4 signer sign requests to Amazon S3 using query string parameters.
+ */
+public class Aws4SignerForQueryString extends Aws4SignerBase {
+ @Inject
+ public Aws4SignerForQueryString(SignatureWire signatureWire,
+ @Named(PROPERTY_S3_VIRTUAL_HOST_BUCKETS) boolean isVhostStyle,
+ @Named(PROPERTY_HEADER_TAG) String headerTag,
+ @Provider Supplier<Credentials> creds, @TimeStamp Supplier<Date> timestampProvider,
+ ServiceAndRegion serviceAndRegion, Crypto crypto) {
+ super(signatureWire, headerTag, creds, timestampProvider, serviceAndRegion, crypto);
+ }
+
+
+ protected HttpRequest sign(HttpRequest request, long timeInSeconds) throws HttpException {
+ checkNotNull(request, "request is not ready to sign");
+ checkNotNull(request.getEndpoint(), "request is not ready to sign, request.endpoint not present.");
+
+ // get host from request endpoint.
+ String host = request.getEndpoint().getHost();
+
+ Date date = timestampProvider.get();
+ String timestamp = timestampFormat.format(date);
+ String datestamp = dateFormat.format(date);
+
+ String service = serviceAndRegion.service();
+ String region = serviceAndRegion.region(host);
+ String credentialScope = Joiner.on('/').join(datestamp, region, service, "aws4_request");
+
+ // different with signature with Authorization header
+ HttpRequest.Builder<?> requestBuilder = request.toBuilder() //
+ // sign for temporary access use query string parameter:
+ // X-Amz-Algorithm, X-Amz-Credential, X-Amz-Date, X-Amz-Expires, X-Amz-SignedHeaders, X-Amz-Signature
+ // remove Authorization, x-amz-content-sha256, X-Amz-Date headers
+ .removeHeader(AUTHORIZATION_HEADER)
+ .removeHeader(AMZ_CONTENT_SHA256_HEADER)
+ .removeHeader(AMZ_DATE_HEADER);
+
+ ImmutableMap.Builder<String, String> signedHeadersBuilder = ImmutableSortedMap.<String, String>naturalOrder(); //
+ Uris.UriBuilder endpointBuilder = Uris.uriBuilder(request.getEndpoint());
+
+
+ // Canonical Headers
+ // must include the HTTP host header.
+ // If you plan to include any of the x-amz-* headers, these headers must also be added for signature calculation.
+ // You can optionally add all other headers that you plan to include in your request.
+ // For added security, you should sign as many headers as possible.
+
+ // HOST
+ signedHeadersBuilder.put("host", host);
+ ImmutableMap<String, String> signedHeaders = signedHeadersBuilder.build();
+
+ Credentials credentials = creds.get();
+
+ if (credentials instanceof SessionCredentials) {
+ String token = SessionCredentials.class.cast(credentials).getSessionToken();
+ // different with signature with Authorization header
+ endpointBuilder.replaceQuery(AMZ_SECURITY_TOKEN_PARAM, token);
+ }
+
+ // X-Amz-Algorithm=HMAC-SHA256
+ endpointBuilder.replaceQuery(AMZ_ALGORITHM_PARAM, AwsSignatureV4Constants.AMZ_ALGORITHM_HMAC_SHA256);
+
+ // X-Amz-Credential=<your-access-key-id>/<date>/<AWS-region>/<AWS-service>/aws4_request.
+ String credential = Joiner.on("/").join(credentials.identity, credentialScope);
+ endpointBuilder.replaceQuery(AMZ_CREDENTIAL_PARAM, credential);
+
+ // X-Amz-Date=ISO 8601 format, for example, 20130721T201207Z
+ endpointBuilder.replaceQuery(AMZ_DATE_PARAM, timestamp);
+
+ // X-Amz-Expires=time in seconds
+ endpointBuilder.replaceQuery(AMZ_EXPIRES_PARAM, String.valueOf(timeInSeconds));
+
+ // X-Amz-SignedHeaders=HTTP host header is required.
+ endpointBuilder.replaceQuery(AMZ_SIGNEDHEADERS_PARAM, Joiner.on(';').join(signedHeaders.keySet()));
+
+ String stringToSign = createStringToSign(request.getMethod(), endpointBuilder.build(), signedHeaders, timestamp, credentialScope,
+ getPayloadHash());
+
+ signatureWire.getWireLog().debug("<< " + stringToSign);
+
+
+ byte[] signatureKey = signatureKey(credentials.credential, datestamp, region, service);
+ String signature = base16().lowerCase().encode(hmacSHA256(stringToSign, signatureKey));
+
+ // X-Amz-Signature=Signature
+ endpointBuilder.replaceQuery(AMZ_SIGNATURE_PARAM, signature);
+
+ return requestBuilder.endpoint(endpointBuilder.build()).build();
+ }
+
+ protected String getPayloadHash() {
+ return UNSIGNED_PAYLOAD;
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/filters/AwsHostNameUtils.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/AwsHostNameUtils.java b/apis/s3/src/main/java/org/jclouds/s3/filters/AwsHostNameUtils.java
new file mode 100644
index 0000000..e133aa3
--- /dev/null
+++ b/apis/s3/src/main/java/org/jclouds/s3/filters/AwsHostNameUtils.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jclouds.s3.filters;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class AwsHostNameUtils {
+
+ private static final Pattern S3_ENDPOINT_PATTERN = Pattern.compile("^(?:.+\\.)?s3[.-]([a-z0-9-]+)$");
+
+ private static final Pattern STANDARD_CLOUDSEARCH_ENDPOINT_PATTERN = Pattern.compile("^(?:.+\\.)?([a-z0-9-]+)\\.cloudsearch$");
+
+ private static final Pattern EXTENDED_CLOUDSEARCH_ENDPOINT_PATTERN = Pattern.compile("^(?:.+\\.)?([a-z0-9-]+)\\.cloudsearch\\..+");
+
+ private static final ImmutableMap<String, String> HOST_REGEX_TO_REGION_MAPPINGS = new ImmutableMap.Builder<String, String>()
+ .put("(.+\\.)?s3\\.amazonaws\\.com", "us-east-1")
+ .put("(.+\\.)?s3-external-1\\.amazonaws\\.com", "us-east-1")
+ .put("(.+\\.)?s3-fips-us-gov-west-1\\.amazonaws\\.com", "us-gov-west-1")
+ .build();
+
+ /**
+ * Attempts to parse the region name from an endpoint based on conventions
+ * about the endpoint format.
+ *
+ * @param host the hostname to parse
+ * @param serviceHint an optional hint about the service for the endpoint
+ * @return the region parsed from the hostname, or
+ * "us-east-1" if no region information
+ * could be found
+ */
+ public static String parseRegionName(final String host, final String serviceHint) {
+
+ String regionNameInInternalConfig = parseRegionNameByInternalConfig(host);
+ if (regionNameInInternalConfig != null) {
+ return regionNameInInternalConfig;
+ }
+
+ if (host.endsWith(".amazonaws.com")) {
+ int index = host.length() - ".amazonaws.com".length();
+ return parseStandardRegionName(host.substring(0, index));
+ }
+
+ if (serviceHint != null) {
+ if (serviceHint.equals("cloudsearch")
+ && !host.startsWith("cloudsearch.")) {
+
+ // CloudSearch domains use the nonstandard domain format
+ // [domain].[region].cloudsearch.[suffix].
+
+ Matcher matcher = EXTENDED_CLOUDSEARCH_ENDPOINT_PATTERN
+ .matcher(host);
+
+ if (matcher.matches()) {
+ return matcher.group(1);
+ }
+ }
+
+ // If we have a service hint, look for 'service.[region]' or
+ // 'service-[region]' in the endpoint's hostname.
+ Pattern pattern = Pattern.compile(
+ "^(?:.+\\.)?"
+ + Pattern.quote(serviceHint)
+ + "[.-]([a-z0-9-]+)\\."
+ );
+
+ Matcher matcher = pattern.matcher(host);
+ if (matcher.find()) {
+ return matcher.group(1);
+ }
+ }
+
+ // Endpoint is totally non-standard; guess us-east-1 for lack of a
+ // better option.
+
+ return "us-east-1";
+ }
+
+ /**
+ * Parses the region name from a standard (*.amazonaws.com) endpoint.
+ *
+ * @param fragment the portion of the endpoint excluding
+ * ".amazonaws.com"
+ * @return the parsed region name (or "us-east-1" as a
+ * best guess if we can't tell for sure)
+ */
+ private static String parseStandardRegionName(final String fragment) {
+
+ Matcher matcher = S3_ENDPOINT_PATTERN.matcher(fragment);
+ if (matcher.matches()) {
+ // host was 'bucket.s3-[region].amazonaws.com'.
+ return matcher.group(1);
+ }
+
+ matcher = STANDARD_CLOUDSEARCH_ENDPOINT_PATTERN.matcher(fragment);
+ if (matcher.matches()) {
+ // host was 'domain.[region].cloudsearch.amazonaws.com'.
+ return matcher.group(1);
+ }
+
+ int index = fragment.lastIndexOf('.');
+ if (index == -1) {
+ // host was 'service.amazonaws.com', guess us-east-1
+ // for lack of a better option.
+ return "us-east-1";
+ }
+
+ // host was 'service.[region].amazonaws.com'.
+ String region = fragment.substring(index + 1);
+
+ // Special case for iam.us-gov.amazonaws.com, which is actually
+ // us-gov-west-1.
+ if ("us-gov".equals(region)) {
+ region = "us-gov-west-1";
+ }
+
+ return region;
+ }
+
+ /**
+ * @return the configured region name if the given host name matches any of
+ * the host-to-region mappings in the internal config; otherwise
+ * return null.
+ */
+ private static String parseRegionNameByInternalConfig(String host) {
+ for (Map.Entry<String, String> mapping : HOST_REGEX_TO_REGION_MAPPINGS.entrySet()) {
+ String hostNameRegex = mapping.getKey();
+ if (host.matches(hostNameRegex)) {
+ return mapping.getValue();
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Parses the service name from an endpoint. Can only handle endpoints of
+ * the form 'service.[region.]amazonaws.com'.
+ * or
+ * bucket.s3.[region.]awazonaws.com
+ */
+ public static String parseServiceName(URI endpoint) {
+ String host = endpoint.getHost();
+
+ if (!host.endsWith(".amazonaws.com") && !host.endsWith(".amazonaws.com.cn")) {
+ return "s3"; // cannot parse name, assume s3
+ }
+
+ String serviceAndRegion = host.substring(0, host.indexOf(".amazonaws.com"));
+
+ // Special cases for S3 endpoints with bucket names embedded.
+ if (serviceAndRegion.endsWith(".s3") || S3_ENDPOINT_PATTERN.matcher(serviceAndRegion).matches()) {
+ return "s3";
+ }
+
+ char separator = '.';
+
+ // If we don't detect a separator between service name and region, then
+ // assume that the region is not included in the hostname, and it's only
+ // the service name (ex: "http://iam.amazonaws.com").
+ if (serviceAndRegion.indexOf(separator) == -1) {
+ return serviceAndRegion;
+ }
+
+ return serviceAndRegion.substring(0, serviceAndRegion.indexOf(separator));
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/filters/AwsSignatureV4Constants.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/AwsSignatureV4Constants.java b/apis/s3/src/main/java/org/jclouds/s3/filters/AwsSignatureV4Constants.java
new file mode 100644
index 0000000..4bd42e4
--- /dev/null
+++ b/apis/s3/src/main/java/org/jclouds/s3/filters/AwsSignatureV4Constants.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+/**
+ * AWS Signature Version 4 Constants.
+ */
+public abstract class AwsSignatureV4Constants {
+
+ /**
+ * AWS authorization header key
+ */
+ public static final String AUTHORIZATION_HEADER = "Authorization";
+
+ /**
+ * AWS content sha256 header key
+ */
+ public static final String AMZ_CONTENT_SHA256_HEADER = "x-amz-content-sha256";
+
+ /**
+ * AWS date header key
+ */
+ public static final String AMZ_DATE_HEADER = "X-Amz-Date";
+
+ /**
+ * AWS security token key
+ */
+ public static final String AMZ_SECURITY_TOKEN_HEADER = "X-Amz-Security-Token";
+
+ /**
+ * For AWS Signature Version 4, you set this parameter value to "AWS4-HMAC-SHA256".
+ */
+ public static final String AMZ_ALGORITHM_PARAM = "X-Amz-Algorithm";
+ /**
+ * This string identifies AWS Signature Version 4 (AWS4) and the HMAC-SHA256 algorithm (HMAC-SHA256).
+ */
+ public static final String AMZ_ALGORITHM_HMAC_SHA256 = "AWS4-HMAC-SHA256";
+
+ /**
+ * In addition to your access key ID, this parameter also provides scope information identifying the region and
+ * service for which the signature is valid.
+ * <p>This value should match the scope that you use to calculate the signing key, as discussed in the following section.</p>
+ * <p>The general form for this parameter value is as follows:</p>
+ * <code> <your-access-key-id>/<date>/<AWS-region>/<AWS-service>/aws4_request.</code>
+ * <p>
+ * For example:
+ * <code>AKIAIOSFODNN7EXAMPLE/20130721/us-east-1/s3/aws4_request.</code><br>
+ * For Amazon S3, the AWS-service string is "s3". For a list of AWS-region strings, go to Regions and Endpoints
+ * in the Amazon Web Services General Reference
+ * </p>
+ */
+ public static final String AMZ_CREDENTIAL_PARAM = "X-Amz-Credential";
+
+ /**
+ * This header can be used in the following scenarios:
+ * <ul>
+ * <li>Provide security tokens for Amazon DevPay operations—Each request that uses Amazon DevPay requires two
+ * x-amz-security-token headers: one for the product token and one for the user token. When Amazon S3 receives
+ * an authenticated request, it compares the computed signature with the provided signature.
+ * Improperly formatted multi-value headers used to calculate a signature can cause authentication issues</li>
+ * <li>Provide security token when using temporary security credentials—When making requests using temporary
+ * security credentials you obtained from IAM you must provide a security token using this header.
+ * To learn more about temporary security credentials, go to Making Requests.</li>
+ * </ul>
+ * This header is required for requests that use Amazon DevPay and requests that are signed using temporary security credentials.
+ */
+
+ public static final String AMZ_SECURITY_TOKEN_PARAM = AMZ_SECURITY_TOKEN_HEADER;
+
+ /**
+ * The date in ISO 8601 format, for example, 20130721T201207Z. This value must match the date value used to
+ * calculate the signature.
+ */
+ public static final String AMZ_DATE_PARAM = AMZ_DATE_HEADER;
+
+ /**
+ * Provides the time period, in seconds, for which the generated presigned URL is valid.
+ * <p> For example, 86400 (24 hours). This value is an integer. The minimum value you can set is 1,
+ * and the maximum is 604800 (seven days). </p>
+ * <p> A presigned URL can be valid for a maximum of seven days because the signing key you use in signature
+ * calculation is valid for up to seven days.</p>
+ */
+ public static final String AMZ_EXPIRES_PARAM = "X-Amz-Expires";
+
+ /**
+ * Lists the headers that you used to calculate the signature.
+ * <p> The HTTP host header is required. Any x-amz-* headers that you plan to add to the request are also required
+ * for signature calculation. </p>
+ * <p> In general, for added security, you should sign all the request headers that you plan to include in your
+ * request.</p>
+ */
+ public static final String AMZ_SIGNEDHEADERS_PARAM = "X-Amz-SignedHeaders";
+
+ /**
+ * X-Amz-Signature Provides the signature to authenticate your request.
+ * <p>This signature must match the signature Amazon S3 calculates; otherwise, Amazon S3 denies the request.
+ * For example, 733255ef022bec3f2a8701cd61d4b371f3f28c9f193a1f02279211d48d5193d7</p>
+ */
+ public static final String AMZ_SIGNATURE_PARAM = "X-Amz-Signature";
+
+ /**
+ * You don't include a payload hash in the Canonical Request, because when you create a presigned URL,
+ * <p> you don't know anything about the payload. Instead, you use a constant string "UNSIGNED-PAYLOAD".</p>
+ */
+ public static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD";
+
+ /**
+ * SHA256 substitute marker used in place of x-amz-content-sha256 when
+ * employing chunked uploads
+ */
+ public static final String STREAMING_BODY_SHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
+ public static final String CHUNK_STRING_TO_SIGN_PREFIX = "AWS4-HMAC-SHA256-PAYLOAD";
+
+ public static final String CLRF = "\r\n";
+
+ public static final String CHUNK_SIGNATURE_HEADER = ";chunk-signature=";
+ public static final int SIGNATURE_LENGTH = 64;
+ public static final byte[] FINAL_CHUNK = new byte[0];
+
+ /**
+ * Content-Encoding
+ * <p>
+ * Set the value to aws-chunked.<br>
+ * Amazon S3 supports multiple content encodings, for example,<br>
+ * Content-Encoding : aws-chunked, gzip<br>
+ * That is, you can specify your custom content-encoding when using Signature Version 4 streaming API.
+ * </p>
+ */
+ public static final String CONTENT_ENCODING_HEADER_AWS_CHUNKED = "aws-chunked";
+ /**
+ * 'x-amz-decoded-content-length' is used to transmit the actual
+ */
+ public static final String AMZ_DECODED_CONTENT_LENGTH_HEADER = "x-amz-decoded-content-length";
+
+ private AwsSignatureV4Constants() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/filters/ChunkedUploadException.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/ChunkedUploadException.java b/apis/s3/src/main/java/org/jclouds/s3/filters/ChunkedUploadException.java
new file mode 100644
index 0000000..ff66337
--- /dev/null
+++ b/apis/s3/src/main/java/org/jclouds/s3/filters/ChunkedUploadException.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+public class ChunkedUploadException extends RuntimeException {
+ public ChunkedUploadException(String error, Exception e) {
+ super(error, e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/8bddbb49/apis/s3/src/main/java/org/jclouds/s3/filters/ChunkedUploadPayload.java
----------------------------------------------------------------------
diff --git a/apis/s3/src/main/java/org/jclouds/s3/filters/ChunkedUploadPayload.java b/apis/s3/src/main/java/org/jclouds/s3/filters/ChunkedUploadPayload.java
new file mode 100644
index 0000000..28e8ac6
--- /dev/null
+++ b/apis/s3/src/main/java/org/jclouds/s3/filters/ChunkedUploadPayload.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.filters;
+
+import com.google.common.io.ByteProcessor;
+import com.google.common.io.ByteStreams;
+import org.jclouds.http.HttpException;
+import org.jclouds.io.MutableContentMetadata;
+import org.jclouds.io.Payload;
+import org.jclouds.io.payloads.BaseMutableContentMetadata;
+import org.jclouds.io.payloads.BasePayload;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
+import java.util.Enumeration;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.io.ByteStreams.readBytes;
+import static org.jclouds.s3.filters.Aws4SignerBase.hash;
+import static org.jclouds.s3.filters.Aws4SignerBase.hex;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.CHUNK_SIGNATURE_HEADER;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.CHUNK_STRING_TO_SIGN_PREFIX;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.CLRF;
+import static org.jclouds.s3.filters.AwsSignatureV4Constants.FINAL_CHUNK;
+import static org.jclouds.util.Strings2.toInputStream;
+
+public class ChunkedUploadPayload extends BasePayload<Payload> {
+ private static final byte[] TRAILER = CLRF.getBytes(UTF_8);
+
+ private final Payload payload;
+ private final int chunkedBlockSize;
+ private final String timestamp;
+ private final String scope;
+ private final ByteProcessor<byte[]> hmacSHA256;
+ private String lastComputedSignature;
+
+ public ChunkedUploadPayload(Payload payload, int blockSize, String timestamp, String scope,
+ ByteProcessor<byte[]> hmacSHA256, String seedSignature) {
+ super(payload);
+ this.payload = payload;
+ this.chunkedBlockSize = blockSize;
+ this.timestamp = timestamp;
+ this.scope = scope;
+ this.hmacSHA256 = hmacSHA256;
+ this.lastComputedSignature = seedSignature;
+
+ // init content metadata
+ MutableContentMetadata contentMetadata = BaseMutableContentMetadata.fromContentMetadata(
+ payload.getContentMetadata());
+ long totalLength = Aws4SignerForChunkedUpload.calculateChunkedContentLength(
+ payload.getContentMetadata().getContentLength(),
+ chunkedBlockSize);
+ contentMetadata.setContentLength(totalLength);
+ this.setContentMetadata(contentMetadata);
+ }
+
+ /**
+ * Returns a chunk for upload consisting of the signed 'header' or chunk
+ * prefix plus the user data. The signature of the chunk incorporates the
+ * signature of the previous chunk (or, if the first chunk, the signature of
+ * the headers portion of the request).
+ *
+ * @param userDataLen The length of the user data contained in userData
+ * @param userData Contains the user data to be sent in the upload chunk
+ * @return A new buffer of data for upload containing the chunk header plus
+ * user data
+ */
+ protected byte[] constructSignedChunk(int userDataLen, byte[] userData) {
+ // to keep our computation routine signatures simple, if the userData
+ // buffer contains less data than it could, shrink it. Note the special case
+ // to handle the requirement that we send an empty chunk to complete
+ // our chunked upload.
+ byte[] dataToChunk;
+ if (userDataLen == 0) {
+ dataToChunk = FINAL_CHUNK;
+ } else {
+ if (userDataLen < userData.length) {
+ // shrink the chunkdata to fit
+ dataToChunk = new byte[userDataLen];
+ System.arraycopy(userData, 0, dataToChunk, 0, userDataLen);
+ } else {
+ dataToChunk = userData;
+ }
+ }
+
+ // string(IntHexBase(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n
+ StringBuilder chunkHeader = new StringBuilder();
+
+ // start with size of user data
+ // IntHexBase(chunk-size)
+ chunkHeader.append(Integer.toHexString(dataToChunk.length));
+
+ // chunk-signature
+
+ // nonsig-extension; we have none in these samples
+ String nonsigExtension = "";
+
+ // if this is the first chunk, we package it with the signing result
+ // of the request headers, otherwise we use the cached signature
+ // of the previous chunk
+
+ // sig-extension
+ StringBuilder buffer = new StringBuilder();
+ buffer.append(CHUNK_STRING_TO_SIGN_PREFIX);
+ buffer.append("\n");
+ buffer.append(timestamp).append("\n");
+ buffer.append(scope).append("\n");
+ buffer.append(lastComputedSignature).append("\n");
+ buffer.append(hex(hash(nonsigExtension))).append("\n");
+ buffer.append(hex(hash(dataToChunk)));
+
+ String chunkStringToSign = buffer.toString();
+
+ // compute the V4 signature for the chunk
+ String chunkSignature;
+ try {
+ chunkSignature = hex(readBytes(toInputStream(chunkStringToSign), hmacSHA256));
+ } catch (IOException e) {
+ throw new HttpException("hmac sha256 chunked signature error");
+ }
+
+ // cache the signature to include with the next chunk's signature computation
+ lastComputedSignature = chunkSignature;
+
+ // construct the actual chunk, comprised of the non-signed extensions, the
+ // 'headers' we just signed and their signature, plus a newline then copy
+ // that plus the user's data to a payload to be written to the request stream
+ chunkHeader.append(nonsigExtension + CHUNK_SIGNATURE_HEADER + chunkSignature);
+ chunkHeader.append(CLRF);
+
+ byte[] header = chunkHeader.toString().getBytes(UTF_8);
+ byte[] signedChunk = new byte[header.length + dataToChunk.length + TRAILER.length];
+ System.arraycopy(header, 0, signedChunk, 0, header.length);
+ // chunk-data
+ System.arraycopy(dataToChunk, 0, signedChunk, header.length, dataToChunk.length);
+ System.arraycopy(TRAILER, 0, signedChunk, header.length + dataToChunk.length, TRAILER.length);
+
+ // this is the total data for the chunk that will be sent to the request stream
+ return signedChunk;
+ }
+
+ @Override
+ public void release() {
+ this.payload.release();
+ }
+
+ @Override
+ public boolean isRepeatable() {
+ return this.payload.isRepeatable();
+ }
+
+ @Override
+ public InputStream openStream() throws IOException {
+ return new SequenceInputStream(new ChunkedInputStreamEnumeration(this.payload.openStream(), chunkedBlockSize));
+ }
+
+ private class ChunkedInputStreamEnumeration implements Enumeration<InputStream> {
+ private final InputStream inputStream;
+ private boolean lastChunked;
+ private byte[] buffer;
+
+ ChunkedInputStreamEnumeration(InputStream inputStream, int chunkedBlockSize) {
+ this.inputStream = new BufferedInputStream(inputStream, chunkedBlockSize);
+ buffer = new byte[chunkedBlockSize];
+ lastChunked = false;
+ }
+
+ @Override
+ public boolean hasMoreElements() {
+ return !lastChunked;
+ }
+
+ @Override
+ public InputStream nextElement() {
+ int bytesRead;
+ try {
+ bytesRead = ByteStreams.read(inputStream, buffer, 0, buffer.length);
+ } catch (IOException e) {
+ // IO EXCEPTION
+ throw new ChunkedUploadException("read from input stream error", e);
+ }
+
+ // buffer
+ byte[] chunk;
+
+ // ByteStreams.read(InputStream, byte[], int, int) returns the number of bytes read
+ // InputStream.read(byte[], int, int) returns -1 if the end of the stream has been reached.
+ if (bytesRead > 0) {
+ // process into a chunk
+ chunk = constructSignedChunk(bytesRead, buffer);
+ } else {
+ // construct last chunked block
+ chunk = constructSignedChunk(0, buffer);
+ lastChunked = true;
+ }
+ return new ByteArrayInputStream(chunk);
+ }
+ }
+
+}