You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by ga...@apache.org on 2014/07/11 07:19:56 UTC
git commit: Enforce correct MD5 for local blobstores
Repository: jclouds
Updated Branches:
refs/heads/master 1d218b170 -> f4eca0422
Enforce correct MD5 for local blobstores
Matches behavior of real blobstores.
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/f4eca042
Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/f4eca042
Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/f4eca042
Branch: refs/heads/master
Commit: f4eca0422db963a619a0215affec2cc905c78dc1
Parents: 1d218b1
Author: Andrew Gaul <ga...@apache.org>
Authored: Tue Oct 22 16:41:31 2013 -0700
Committer: Andrew Gaul <ga...@apache.org>
Committed: Thu Jul 10 22:19:42 2014 -0700
----------------------------------------------------------------------
.../internal/FilesystemStorageStrategyImpl.java | 13 ++++-
.../FilesystemBlobIntegrationTest.java | 5 --
.../jclouds/blobstore/LocalAsyncBlobStore.java | 5 ++
.../blobstore/TransientStorageStrategy.java | 59 ++++++++++----------
.../TransientBlobIntegrationTest.java | 5 --
5 files changed, 45 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds/blob/f4eca042/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
----------------------------------------------------------------------
diff --git a/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java b/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
index bd013cb..b526fbf 100644
--- a/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
+++ b/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
@@ -47,6 +47,7 @@ import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
+import com.google.common.hash.HashCode;
import com.google.common.hash.Hashing;
import com.google.common.hash.HashingInputStream;
import com.google.common.io.ByteSource;
@@ -201,9 +202,15 @@ public class FilesystemStorageStrategyImpl implements LocalStorageStrategy {
Files.createParentDirs(outputFile);
his = new HashingInputStream(Hashing.md5(), payload.openStream());
Files.asByteSink(outputFile).writeFrom(his);
- payload.getContentMetadata().setContentMD5(his.hash());
- String eTag = base16().lowerCase().encode(payload.getContentMetadata().getContentMD5());
- return eTag;
+ HashCode actualHashCode = his.hash();
+ HashCode expectedHashCode = payload.getContentMetadata().getContentMD5AsHashCode();
+ if (expectedHashCode != null && !actualHashCode.equals(expectedHashCode)) {
+ throw new IOException("MD5 hash code mismatch, actual: " + actualHashCode +
+ " expected: " + expectedHashCode);
+ }
+ payload.getContentMetadata().setContentMD5(actualHashCode);
+ // TODO: store metadata in extended attributes when moving to Java 7
+ return base16().lowerCase().encode(actualHashCode.asBytes());
} catch (IOException ex) {
if (outputFile != null) {
if (!outputFile.delete()) {
http://git-wip-us.apache.org/repos/asf/jclouds/blob/f4eca042/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java
----------------------------------------------------------------------
diff --git a/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java b/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java
index ca2face..bb20fc6 100644
--- a/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java
+++ b/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java
@@ -77,9 +77,4 @@ public class FilesystemBlobIntegrationTest extends BaseBlobIntegrationTest {
public void testPutObjectStream() throws InterruptedException, IOException, ExecutionException {
throw new SkipException("not yet implemented");
}
-
- @Override
- public void testPutIncorrectContentMD5() throws InterruptedException, IOException {
- throw new SkipException("not yet implemented");
- }
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/f4eca042/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java
----------------------------------------------------------------------
diff --git a/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java
index 7b57c62..ba987ab 100644
--- a/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java
+++ b/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java
@@ -392,6 +392,11 @@ public class LocalAsyncBlobStore extends BaseAsyncBlobStore {
try {
return immediateFuture(storageStrategy.putBlob(containerName, blob));
} catch (IOException e) {
+ if (e.getMessage().startsWith("MD5 hash code mismatch")) {
+ HttpResponseException exception = returnResponseException(400);
+ exception.initCause(e);
+ throw exception;
+ }
logger.error(e, "An error occurred storing the new blob with name [%s] to container [%s].", blobKey,
containerName);
throw Throwables.propagate(e);
http://git-wip-us.apache.org/repos/asf/jclouds/blob/f4eca042/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java
----------------------------------------------------------------------
diff --git a/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java b/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java
index 4871ddf..dfa4365 100644
--- a/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java
+++ b/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java
@@ -39,14 +39,15 @@ import org.jclouds.io.ContentMetadataCodec;
import org.jclouds.io.MutableContentMetadata;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
-import org.jclouds.io.payloads.ByteArrayPayload;
-import org.jclouds.io.payloads.DelegatingPayload;
+import org.jclouds.util.Closeables2;
import com.google.common.base.Supplier;
-import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Multimaps;
+import com.google.common.hash.HashCode;
import com.google.common.hash.Hashing;
+import com.google.common.hash.HashingInputStream;
+import com.google.common.io.ByteSource;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
@@ -123,10 +124,25 @@ public class TransientStorageStrategy implements LocalStorageStrategy {
@Override
public String putBlob(final String containerName, final Blob blob) throws IOException {
- Blob newBlob = createUpdatedCopyOfBlobInContainer(containerName, blob);
+ byte[] payload;
+ HashCode actualHashCode;
+ HashingInputStream input = new HashingInputStream(Hashing.md5(), blob.getPayload().openStream());
+ try {
+ payload = ByteStreams.toByteArray(input);
+ actualHashCode = input.hash();
+ HashCode expectedHashCode = blob.getPayload().getContentMetadata().getContentMD5AsHashCode();
+ if (expectedHashCode != null && !actualHashCode.equals(expectedHashCode)) {
+ throw new IOException("MD5 hash code mismatch, actual: " + actualHashCode +
+ " expected: " + expectedHashCode);
+ }
+ } finally {
+ Closeables2.closeQuietly(input);
+ }
+
+ Blob newBlob = createUpdatedCopyOfBlobInContainer(containerName, blob, payload, actualHashCode);
Map<String, Blob> map = containerToBlobs.get(containerName);
map.put(newBlob.getMetadata().getName(), newBlob);
- return base16().lowerCase().encode(newBlob.getPayload().getContentMetadata().getContentMD5());
+ return base16().lowerCase().encode(actualHashCode.asBytes());
}
@Override
@@ -146,37 +162,22 @@ public class TransientStorageStrategy implements LocalStorageStrategy {
return "/";
}
- private Blob createUpdatedCopyOfBlobInContainer(String containerName, Blob in) {
+ private Blob createUpdatedCopyOfBlobInContainer(String containerName, Blob in, byte[] input, HashCode contentMd5) {
+ checkNotNull(containerName, "containerName");
checkNotNull(in, "blob");
- checkNotNull(in.getPayload(), "blob.payload");
- ByteArrayPayload payload = (in.getPayload() instanceof ByteArrayPayload) ? ByteArrayPayload.class.cast(in
- .getPayload()) : null;
- if (payload == null)
- payload = (in.getPayload() instanceof DelegatingPayload) ? (DelegatingPayload.class.cast(in.getPayload())
- .getDelegate() instanceof ByteArrayPayload) ? ByteArrayPayload.class.cast(DelegatingPayload.class
- .cast(in.getPayload()).getDelegate()) : null : null;
- try {
- if (payload == null || !(payload instanceof ByteArrayPayload)) {
- MutableContentMetadata oldMd = in.getPayload().getContentMetadata();
- byte[] out = ByteStreams.toByteArray(in.getPayload());
- payload = Payloads.newByteArrayPayload(out);
- HttpUtils.copy(oldMd, payload.getContentMetadata());
- payload.getContentMetadata().setContentMD5(Hashing.md5().hashBytes(out));
- } else {
- if (payload.getContentMetadata().getContentMD5() == null) {
- payload.getContentMetadata().setContentMD5(ByteStreams.hash(payload, Hashing.md5()));
- }
- }
- } catch (IOException e) {
- Throwables.propagate(e);
- }
+ checkNotNull(input, "input");
+ checkNotNull(contentMd5, "contentMd5");
+ Payload payload = Payloads.newByteSourcePayload(ByteSource.wrap(input));
+ MutableContentMetadata oldMd = in.getPayload().getContentMetadata();
+ HttpUtils.copy(oldMd, payload.getContentMetadata());
+ payload.getContentMetadata().setContentMD5(contentMd5);
Blob blob = blobFactory.create(BlobStoreUtils.copy(in.getMetadata()));
blob.setPayload(payload);
blob.getMetadata().setContainer(containerName);
blob.getMetadata().setUri(
uriBuilder(new StringBuilder("mem://").append(containerName)).path(in.getMetadata().getName()).build());
blob.getMetadata().setLastModified(new Date());
- String eTag = base16().lowerCase().encode(payload.getContentMetadata().getContentMD5());
+ String eTag = base16().lowerCase().encode(contentMd5.asBytes());
blob.getMetadata().setETag(eTag);
// Set HTTP headers to match metadata
blob.getAllHeaders().replaceValues(HttpHeaders.LAST_MODIFIED,
http://git-wip-us.apache.org/repos/asf/jclouds/blob/f4eca042/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java
----------------------------------------------------------------------
diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java
index efd5d34..4e2d496 100644
--- a/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java
+++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java
@@ -27,9 +27,4 @@ public class TransientBlobIntegrationTest extends BaseBlobIntegrationTest {
public TransientBlobIntegrationTest() {
provider = "transient";
}
-
- @Override
- public void testPutIncorrectContentMD5() throws InterruptedException, IOException {
- throw new SkipException("not yet implemented");
- }
}