You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jclouds.apache.org by za...@apache.org on 2016/09/14 19:37:05 UTC
jclouds git commit: More fixes to parallel download resource cleanup
Repository: jclouds
Updated Branches:
refs/heads/master de68c2a1b -> 4bbca9edf
More fixes to parallel download resource cleanup
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo
Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/4bbca9ed
Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/4bbca9ed
Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/4bbca9ed
Branch: refs/heads/master
Commit: 4bbca9edf943852ce1ea5aa579fa0554f770a3ea
Parents: de68c2a
Author: Zack Shoylev <za...@rackspace.com>
Authored: Tue Sep 13 01:20:08 2016 -0500
Committer: Zack Shoylev <za...@rackspace.com>
Committed: Wed Sep 14 14:36:44 2016 -0500
----------------------------------------------------------------------
.../blobstore/RegionScopedSwiftBlobStore.java | 66 +++++++++++++++-----
...ionScopedSwiftBlobStoreParallelLiveTest.java | 3 +-
2 files changed, 51 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jclouds/blob/4bbca9ed/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStore.java
----------------------------------------------------------------------
diff --git a/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStore.java b/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStore.java
index e902c5a..66dd818 100644
--- a/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStore.java
+++ b/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStore.java
@@ -31,12 +31,14 @@ import java.io.InputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.RandomAccessFile;
+import java.lang.reflect.Method;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
@@ -677,6 +679,7 @@ public class RegionScopedSwiftBlobStore implements BlobStore {
ListeningExecutorService listeningExecutor = MoreExecutors.listeningDecorator(executor);
RandomAccessFile raf = null;
+ File tempFile = new File(destination.getName() + "." + UUID.randomUUID());
try {
long contentLength = api
.getObjectApi(regionId, container)
@@ -686,7 +689,7 @@ public class RegionScopedSwiftBlobStore implements BlobStore {
.getContentLength();
// Reserve space for performance reasons
- raf = new RandomAccessFile(destination.getAbsoluteFile(), "rw");
+ raf = new RandomAccessFile(tempFile, "rw");
raf.seek(contentLength - 1);
raf.write(0);
@@ -706,12 +709,24 @@ public class RegionScopedSwiftBlobStore implements BlobStore {
Futures.getUnchecked(Futures.allAsList(results));
+ raf.getChannel().force(true);
+ raf.getChannel().close();
+ raf.close();
+
+ if (destination.exists()) {
+ destination.delete();
+ }
+ if (!tempFile.renameTo(destination)) {
+ throw new RuntimeException("Could not move temporary downloaded file to destination " + destination);
+ }
+ tempFile = null;
} catch (IOException e) {
- Closeables2.closeQuietly(raf);
- destination.delete();
throw new RuntimeException(e);
} finally {
Closeables2.closeQuietly(raf);
+ if (tempFile != null) {
+ tempFile.delete();
+ }
}
}
@@ -745,6 +760,11 @@ public class RegionScopedSwiftBlobStore implements BlobStore {
MappedByteBuffer out = raf.getChannel().map(FileChannel.MapMode.READ_WRITE, begin, end - begin + 1);
out.put(targetArray);
out.force();
+
+ // JDK-4715154 ; TODO: Java 8 FileChannels
+ if (System.getProperty("os.name").toLowerCase().contains("windows")) {
+ closeDirectBuffer(out);
+ }
} catch (IOException e) {
lastException = e;
continue;
@@ -753,8 +773,24 @@ public class RegionScopedSwiftBlobStore implements BlobStore {
return null;
}
throw new RuntimeException("After " + retryCountLimit + " retries: " + lastException);
+ }
+
+ // JDK-4715154
+ private void closeDirectBuffer(MappedByteBuffer mbb) {
+ if ( mbb == null || !mbb.isDirect() )
+ return;
+
+ try {
+ Method cleaner = mbb.getClass().getMethod("cleaner");
+ cleaner.setAccessible(true);
+ Method clean = Class.forName("sun.misc.Cleaner").getMethod("clean");
+ clean.setAccessible(true);
+ clean.invoke(cleaner.invoke(mbb));
+ } catch (Exception e) {
+ logger.warn(e.toString());
}
}
+ }
@Beta
@Override
@@ -799,8 +835,8 @@ public class RegionScopedSwiftBlobStore implements BlobStore {
public void run() {
ListenableFuture<byte[]> result;
long from;
- for (from = 0; from < contentLength; from = from + partSize) {
- try {
+ try {
+ for (from = 0; from < contentLength; from = from + partSize) {
logger.debug(Thread.currentThread() + " writing to output");
result = results.take();
if (result == null) {
@@ -809,20 +845,16 @@ public class RegionScopedSwiftBlobStore implements BlobStore {
throw new RuntimeException("Error downloading file part to stream");
}
output.write(result.get());
- } catch (Exception e) {
- logger.debug(e.toString());
- try {
- // close pipe so client is notified of an exception
- input.close();
- } catch (IOException e1) {}
- try {
- output.close();
- } catch (IOException e1) {}
- throw new RuntimeException(e);
}
+ } catch (Exception e) {
+ logger.debug(e.toString());
+ // Close pipe so client is notified of an exception
+ Closeables2.closeQuietly(input);
+ throw new RuntimeException(e);
+ } finally {
+ // Finished writing results to stream
+ Closeables2.closeQuietly(output);
}
- // Finished writing results to stream
- Closeables2.closeQuietly(output);
}
});
http://git-wip-us.apache.org/repos/asf/jclouds/blob/4bbca9ed/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStoreParallelLiveTest.java
----------------------------------------------------------------------
diff --git a/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStoreParallelLiveTest.java b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStoreParallelLiveTest.java
index dd48612..815f15e 100644
--- a/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStoreParallelLiveTest.java
+++ b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStoreParallelLiveTest.java
@@ -57,7 +57,7 @@ import com.google.common.util.concurrent.MoreExecutors;
public class RegionScopedSwiftBlobStoreParallelLiveTest extends BaseBlobStoreIntegrationTest {
private final File BIG_FILE = new File("random.dat");
- private final long SIZE = 1000000000; //10 * 1000 * 1000;
+ private final long SIZE = 10 * 1000 * 1000;
private BlobStore blobStore;
private String ETAG;
private ListeningExecutorService executor =
@@ -100,6 +100,7 @@ public class RegionScopedSwiftBlobStoreParallelLiveTest extends BaseBlobStoreInt
public void cleanupFiles() {
// Delete local file
delete(BIG_FILE);
+ delete(new File(BIG_FILE + ".downloaded"));
// Delete uploaded file
blobStore.clearContainer(CONTAINER);