You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2012/07/31 07:10:07 UTC
[2/8] git commit: manually unmap CLS buffer patch by jbellis;
tested by Patrycjusz Matuszak for CASSANDRA-4337
manually unmap CLS buffer
patch by jbellis; tested by Patrycjusz Matuszak for CASSANDRA-4337
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/27140565
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/27140565
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/27140565
Branch: refs/heads/cassandra-1.1
Commit: 2714056562a4435da48f5a12db18c60ea6551f41
Parents: f9d1f93
Author: Jonathan Ellis <jb...@apache.org>
Authored: Sat Jun 30 19:52:31 2012 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Tue Jul 31 00:02:10 2012 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 4 -
src/java/org/apache/cassandra/db/Directories.java | 2 +-
.../cassandra/db/commitlog/CommitLogSegment.java | 1 +
.../org/apache/cassandra/io/util/FileUtils.java | 45 +++++++++++++++
.../cassandra/io/util/MmappedSegmentedFile.java | 27 +--------
6 files changed, 50 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/27140565/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 78d7267..7348a99 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
1.1.3
+ * munmap commitlog segments before rename (CASSANDRA-4337)
* (JMX) rename getRangeKeySample to sampleKeyRange to avoid returning
multi-MB results as an attribute (CASSANDRA-4452)
* flush based on data size, not throughput; overwritten columns no
http://git-wip-us.apache.org/repos/asf/cassandra/blob/27140565/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 8cb2cba..54486d0 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -41,7 +41,6 @@ import org.apache.cassandra.db.DefsTable;
import org.apache.cassandra.db.SystemTable;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.MmappedSegmentedFile;
import org.apache.cassandra.locator.DynamicEndpointSnitch;
import org.apache.cassandra.locator.EndpointSnitchInfo;
import org.apache.cassandra.locator.IEndpointSnitch;
@@ -196,9 +195,6 @@ public class DatabaseDescriptor
indexAccessMode = conf.disk_access_mode;
logger.info("DiskAccessMode is " + conf.disk_access_mode + ", indexAccessMode is " + indexAccessMode );
}
- // We could enable cleaner for index only mmap but it probably doesn't matter much
- if (conf.disk_access_mode == Config.DiskAccessMode.mmap)
- MmappedSegmentedFile.initCleaner();
logger.debug("page_cache_hinting is " + conf.populate_io_cache_on_flush);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/27140565/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index f023c8c..9c9f9b8 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -117,7 +117,7 @@ public class Directories
// Requesting GC has a chance to free space only if we're using mmap and a non SUN jvm
if (path == null
&& (DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap || DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap)
- && !MmappedSegmentedFile.isCleanerAvailable())
+ && !FileUtils.isCleanerAvailable())
{
logger.info("Forcing GC to free up disk space. Upgrade to the Oracle JVM to avoid this");
StorageService.instance.requestGC();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/27140565/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
index 0449c47..fcd4130 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
@@ -314,6 +314,7 @@ public class CommitLogSegment
try
{
+ FileUtils.clean(buffer);
logFileAccessor.close();
closed = true;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/27140565/src/java/org/apache/cassandra/io/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java
index 4ddf8b6..5ed6276 100644
--- a/src/java/org/apache/cassandra/io/util/FileUtils.java
+++ b/src/java/org/apache/cassandra/io/util/FileUtils.java
@@ -19,6 +19,9 @@
package org.apache.cassandra.io.util;
import java.io.*;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.MappedByteBuffer;
import java.text.DecimalFormat;
import java.util.Comparator;
import java.util.List;
@@ -39,6 +42,22 @@ public class FileUtils
private static final double gb_ = 1024*1024*1024d;
private static final double tb_ = 1024*1024*1024*1024d;
+ private static final Method cleanerMethod = initCleaner();
+
+ private static Method initCleaner()
+ {
+ try
+ {
+ return Class.forName("sun.nio.ch.DirectBuffer").getMethod("cleaner");
+ }
+ catch (Exception e)
+ {
+ // Perhaps a non-sun-derived JVM - contributions welcome
+ logger_.info("Cannot initialize un-mmaper. (Are you using a non-SUN JVM?) Compacted data files will not be removed promptly. Consider using a SUN JVM or using standard disk access mode");
+ return null;
+ }
+ }
+
public static void deleteWithConfirm(String file) throws IOException
{
deleteWithConfirm(new File(file));
@@ -118,6 +137,32 @@ public class FileUtils
throw e;
}
+ public static boolean isCleanerAvailable()
+ {
+ return cleanerMethod != null;
+ }
+
+ public static void clean(MappedByteBuffer buffer)
+ {
+ try
+ {
+ Object cleaner = cleanerMethod.invoke(buffer);
+ cleaner.getClass().getMethod("clean").invoke(cleaner);
+ }
+ catch (IllegalAccessException e)
+ {
+ throw new RuntimeException(e);
+ }
+ catch (InvocationTargetException e)
+ {
+ throw new RuntimeException(e);
+ }
+ catch (NoSuchMethodException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
public static class FileComparator implements Comparator<File>
{
public int compare(File f, File f2)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/27140565/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java b/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java
index a933630..5ebbe24 100644
--- a/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java
+++ b/src/java/org/apache/cassandra/io/util/MmappedSegmentedFile.java
@@ -25,7 +25,6 @@ import java.io.File;
import java.io.IOError;
import java.io.IOException;
import java.io.RandomAccessFile;
-import java.lang.reflect.Method;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
@@ -42,8 +41,6 @@ public class MmappedSegmentedFile extends SegmentedFile
// in a perfect world, MAX_SEGMENT_SIZE would be final, but we need to test with a smaller size to stay sane.
public static long MAX_SEGMENT_SIZE = Integer.MAX_VALUE;
- private static Method cleanerMethod = null;
-
/**
* Sorted array of segment offsets and MappedByteBuffers for segments. If mmap is completely disabled, or if the
* segment would be too long to mmap, the value for an offset will be null, indicating that we need to fall back
@@ -98,27 +95,9 @@ public class MmappedSegmentedFile extends SegmentedFile
}
}
- public static void initCleaner()
- {
- try
- {
- cleanerMethod = Class.forName("sun.nio.ch.DirectBuffer").getMethod("cleaner");
- }
- catch (Exception e)
- {
- // Perhaps a non-sun-derived JVM - contributions welcome
- logger.info("Cannot initialize un-mmaper. (Are you using a non-SUN JVM?) Compacted data files will not be removed promptly. Consider using a SUN JVM or using standard disk access mode");
- }
- }
-
- public static boolean isCleanerAvailable()
- {
- return cleanerMethod != null;
- }
-
public void cleanup()
{
- if (cleanerMethod == null)
+ if (!FileUtils.isCleanerAvailable())
return;
/*
@@ -132,9 +111,7 @@ public class MmappedSegmentedFile extends SegmentedFile
{
if (segment.right == null)
continue;
-
- Object cleaner = cleanerMethod.invoke(segment.right);
- cleaner.getClass().getMethod("clean").invoke(cleaner);
+ FileUtils.clean(segment.right);
}
logger.debug("All segments have been unmapped successfully");
}