You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by ga...@apache.org on 2014/08/11 21:59:38 UTC
git commit: JCLOUDS-658: xattr for filesystem blob metadata
Repository: jclouds
Updated Branches:
refs/heads/master f236e046e -> 204a6c7f3
JCLOUDS-658: xattr for filesystem blob metadata
This allows the filesystem blobstore to mimic real blobstores more
accurately. This also doubles the speed of getBlob by storing the
Content-MD5 header instead of recalculating it.
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/204a6c7f
Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/204a6c7f
Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/204a6c7f
Branch: refs/heads/master
Commit: 204a6c7f3b3ff8689ad9df9b1084801b89dd901d
Parents: f236e04
Author: Andrew Gaul <ga...@apache.org>
Authored: Wed Jul 16 10:22:39 2014 -0700
Committer: Andrew Gaul <ga...@apache.org>
Committed: Mon Aug 11 12:56:14 2014 -0700
----------------------------------------------------------------------
.../internal/FilesystemStorageStrategyImpl.java | 93 +++++++++++++++++++-
.../FilesystemAsyncBlobStoreTest.java | 6 +-
.../FilesystemBlobIntegrationTest.java | 24 -----
.../FilesystemContainerIntegrationTest.java | 7 --
4 files changed, 94 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds/blob/204a6c7f/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
----------------------------------------------------------------------
diff --git a/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java b/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
index 86fd381..74ebef7 100644
--- a/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
+++ b/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
@@ -22,7 +22,11 @@ import static com.google.common.io.BaseEncoding.base16;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.attribute.UserDefinedFileAttributeView;
+import java.nio.charset.StandardCharsets;
import java.util.Date;
+import java.util.Map;
import java.util.Set;
import javax.annotation.Resource;
@@ -39,6 +43,7 @@ import org.jclouds.filesystem.predicates.validators.FilesystemBlobKeyValidator;
import org.jclouds.filesystem.predicates.validators.FilesystemContainerNameValidator;
import org.jclouds.filesystem.reference.FilesystemConstants;
import org.jclouds.filesystem.util.Utils;
+import org.jclouds.io.ContentMetadata;
import org.jclouds.io.Payload;
import org.jclouds.logging.Logger;
import org.jclouds.rest.annotations.ParamValidators;
@@ -46,6 +51,8 @@ import org.jclouds.rest.annotations.ParamValidators;
import com.google.common.base.Function;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.hash.HashCode;
@@ -54,9 +61,18 @@ import com.google.common.hash.HashingInputStream;
import com.google.common.io.ByteSource;
import com.google.common.io.Closeables;
import com.google.common.io.Files;
+import com.google.common.primitives.Longs;
public class FilesystemStorageStrategyImpl implements LocalStorageStrategy {
+ private static final String XATTR_CONTENT_DISPOSITION = "user.content-disposition";
+ private static final String XATTR_CONTENT_ENCODING = "user.content-encoding";
+ private static final String XATTR_CONTENT_LANGUAGE = "user.content-language";
+ private static final String XATTR_CONTENT_MD5 = "user.content-md5";
+ private static final String XATTR_CONTENT_TYPE = "user.content-type";
+ private static final String XATTR_EXPIRES = "user.expires";
+ private static final String XATTR_USER_METADATA_PREFIX = "user.user-metadata.";
+
private static final String BACK_SLASH = "\\";
@Resource
@@ -182,12 +198,47 @@ public class FilesystemStorageStrategyImpl implements LocalStorageStrategy {
File file = getFileForBlobKey(container, key);
ByteSource byteSource = Files.asByteSource(file);
try {
+ UserDefinedFileAttributeView view = java.nio.file.Files.getFileAttributeView(
+ file.toPath(), UserDefinedFileAttributeView.class);
+ Set<String> attributes = ImmutableSet.copyOf(view.list());
+
+ String contentDisposition = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_DISPOSITION);
+ String contentEncoding = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_ENCODING);
+ String contentLanguage = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_LANGUAGE);
+ String contentType = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_TYPE);
+ HashCode hashCode = null;
+ if (attributes.contains(XATTR_CONTENT_MD5)) {
+ ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_CONTENT_MD5));
+ view.read(XATTR_CONTENT_MD5, buf);
+ hashCode = HashCode.fromBytes(buf.array());
+ }
+ Date expires = null;
+ if (attributes.contains(XATTR_EXPIRES)) {
+ ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_EXPIRES));
+ view.read(XATTR_EXPIRES, buf);
+ buf.flip();
+ expires = new Date(buf.asLongBuffer().get());
+ }
+ ImmutableMap.Builder<String, String> userMetadata = ImmutableMap.builder();
+ for (String attribute : attributes) {
+ if (!attribute.startsWith(XATTR_USER_METADATA_PREFIX)) {
+ continue;
+ }
+ String value = readStringAttributeIfPresent(view, attributes, attribute);
+ userMetadata.put(attribute.substring(XATTR_USER_METADATA_PREFIX.length()), value);
+ }
+
builder.payload(byteSource)
+ .contentDisposition(contentDisposition)
+ .contentEncoding(contentEncoding)
+ .contentLanguage(contentLanguage)
.contentLength(byteSource.size())
- .contentMD5(byteSource.hash(Hashing.md5()).asBytes());
+ .contentMD5(hashCode)
+ .contentType(contentType)
+ .expires(expires)
+ .userMetadata(userMetadata.build());
} catch (IOException e) {
- logger.error("An error occurred calculating MD5 for blob %s from container ", key, container);
- Throwables.propagateIfPossible(e);
+ throw Throwables.propagate(e);
}
Blob blob = builder.build();
blob.getMetadata().setContainer(container);
@@ -201,6 +252,7 @@ public class FilesystemStorageStrategyImpl implements LocalStorageStrategy {
public String putBlob(final String containerName, final Blob blob) throws IOException {
String blobKey = blob.getMetadata().getName();
Payload payload = blob.getPayload();
+ ContentMetadata metadata = payload.getContentMetadata();
filesystemContainerNameValidator.validate(containerName);
filesystemBlobKeyValidator.validate(blobKey);
File outputFile = getFileForBlobKey(containerName, blobKey);
@@ -216,7 +268,23 @@ public class FilesystemStorageStrategyImpl implements LocalStorageStrategy {
" expected: " + expectedHashCode);
}
payload.getContentMetadata().setContentMD5(actualHashCode);
- // TODO: store metadata in extended attributes when moving to Java 7
+
+ UserDefinedFileAttributeView view = java.nio.file.Files.getFileAttributeView(
+ outputFile.toPath(), UserDefinedFileAttributeView.class);
+ view.write(XATTR_CONTENT_MD5, ByteBuffer.wrap(actualHashCode.asBytes()));
+ writeStringAttributeIfPresent(view, XATTR_CONTENT_DISPOSITION, metadata.getContentDisposition());
+ writeStringAttributeIfPresent(view, XATTR_CONTENT_ENCODING, metadata.getContentEncoding());
+ writeStringAttributeIfPresent(view, XATTR_CONTENT_LANGUAGE, metadata.getContentLanguage());
+ writeStringAttributeIfPresent(view, XATTR_CONTENT_TYPE, metadata.getContentType());
+ Date expires = metadata.getExpires();
+ if (expires != null) {
+ ByteBuffer buf = ByteBuffer.allocate(Longs.BYTES).putLong(expires.getTime());
+ buf.flip();
+ view.write(XATTR_EXPIRES, buf);
+ }
+ for (Map.Entry<String, String> entry : blob.getMetadata().getUserMetadata().entrySet()) {
+ writeStringAttributeIfPresent(view, XATTR_USER_METADATA_PREFIX + entry.getKey(), entry.getValue());
+ }
return base16().lowerCase().encode(actualHashCode.asBytes());
} catch (IOException ex) {
if (outputFile != null) {
@@ -487,4 +555,21 @@ public class FilesystemStorageStrategyImpl implements LocalStorageStrategy {
return result;
}
+ /** Read the String representation of filesystem attribute, or return null if not present. */
+ private static String readStringAttributeIfPresent(UserDefinedFileAttributeView view, Set<String> attributes,
+ String name) throws IOException {
+ if (!attributes.contains(name)) {
+ return null;
+ }
+ ByteBuffer buf = ByteBuffer.allocate(view.size(name));
+ view.read(name, buf);
+ return new String(buf.array(), StandardCharsets.UTF_8);
+ }
+
+ /** Write an filesystem attribute, if its value is non-null. */
+ private static void writeStringAttributeIfPresent(UserDefinedFileAttributeView view, String name, String value) throws IOException {
+ if (value != null) {
+ view.write(name, ByteBuffer.wrap(value.getBytes(StandardCharsets.UTF_8)));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/204a6c7f/apis/filesystem/src/test/java/org/jclouds/filesystem/FilesystemAsyncBlobStoreTest.java
----------------------------------------------------------------------
diff --git a/apis/filesystem/src/test/java/org/jclouds/filesystem/FilesystemAsyncBlobStoreTest.java b/apis/filesystem/src/test/java/org/jclouds/filesystem/FilesystemAsyncBlobStoreTest.java
index 63f808a..0c52a49 100644
--- a/apis/filesystem/src/test/java/org/jclouds/filesystem/FilesystemAsyncBlobStoreTest.java
+++ b/apis/filesystem/src/test/java/org/jclouds/filesystem/FilesystemAsyncBlobStoreTest.java
@@ -595,7 +595,11 @@ public class FilesystemAsyncBlobStoreTest {
public void testBlobMetadata_withDefaultMetadata() throws IOException {
String BLOB_KEY = TestUtils.createRandomBlobKey(null, null);
// create the blob
- TestUtils.createBlobsInContainer(CONTAINER_NAME, BLOB_KEY);
+ blobStore.createContainerInLocation(null, CONTAINER_NAME);
+ Blob blob = blobStore.blobBuilder(BLOB_KEY)
+ .payload(TestUtils.getImageForBlobPayload())
+ .build();
+ blobStore.putBlob(CONTAINER_NAME, blob);
BlobMetadata metadata = blobStore.blobMetadata(CONTAINER_NAME, BLOB_KEY);
assertNotNull(metadata, "Metadata null");
http://git-wip-us.apache.org/repos/asf/jclouds/blob/204a6c7f/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java
----------------------------------------------------------------------
diff --git a/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java b/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java
index 53bca5b..3618fd4 100644
--- a/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java
+++ b/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java
@@ -16,17 +16,13 @@
*/
package org.jclouds.filesystem.integration;
-import java.io.IOException;
import java.util.Properties;
-import org.jclouds.blobstore.domain.Blob;
-import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
import org.jclouds.filesystem.reference.FilesystemConstants;
import org.jclouds.filesystem.utils.TestUtils;
import org.testng.annotations.Test;
-import org.testng.SkipException;
@Test(groups = { "integration" }, singleThreaded = true, testName = "blobstore.FilesystemBlobIntegrationTest")
public class FilesystemBlobIntegrationTest extends BaseBlobIntegrationTest {
@@ -41,24 +37,4 @@ public class FilesystemBlobIntegrationTest extends BaseBlobIntegrationTest {
props.setProperty(FilesystemConstants.PROPERTY_BASEDIR, TestUtils.TARGET_BASE_DIR);
return props;
}
-
- @Override
- public void checkContentMetadata(Blob blob) {
- // TODO: not yet implemented
- }
-
- @Override
- protected void checkContentDisposition(Blob blob, String contentDisposition) {
- // TODO: not yet implemented
- }
-
- @Override
- protected void validateMetadata(BlobMetadata metadata) throws IOException {
- // TODO: not yet implemented
- }
-
- @Override
- public void testCreateBlobWithExpiry() throws InterruptedException {
- throw new SkipException("requires Java 7 xattr support");
- }
}
http://git-wip-us.apache.org/repos/asf/jclouds/blob/204a6c7f/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemContainerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemContainerIntegrationTest.java b/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemContainerIntegrationTest.java
index f495479..d5a02a0 100644
--- a/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemContainerIntegrationTest.java
+++ b/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemContainerIntegrationTest.java
@@ -19,7 +19,6 @@ package org.jclouds.filesystem.integration;
import static org.jclouds.blobstore.options.ListContainerOptions.Builder.maxResults;
import static org.testng.Assert.assertEquals;
-import java.io.IOException;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
@@ -36,7 +35,6 @@ import org.jclouds.filesystem.reference.FilesystemConstants;
import org.jclouds.filesystem.utils.TestUtils;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import org.testng.SkipException;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
@@ -83,11 +81,6 @@ public class FilesystemContainerIntegrationTest extends BaseContainerIntegration
}
@Override
- public void testWithDetails() throws InterruptedException, IOException {
- throw new SkipException("requires Java 7 xattr support");
- }
-
- @Override
@Test(dataProvider = "ignoreOnWindows")
public void containerExists() throws InterruptedException {
super.containerExists();