You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by yu...@apache.org on 2013/09/18 21:26:47 UTC
[1/6] git commit: Fix memory leak in snapshot repair
Updated Branches:
refs/heads/cassandra-1.2 7161aec42 -> 9d7bb1e0c
refs/heads/cassandra-2.0 2648047a4 -> e9c027a3a
refs/heads/trunk 0eca1652e -> a7820f9cb
Fix memory leak in snapshot repair
patch by yukim; reviewed by jbellis for CASSANDRA-6047
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9d7bb1e0
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9d7bb1e0
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9d7bb1e0
Branch: refs/heads/cassandra-1.2
Commit: 9d7bb1e0c27fd5076afc750150ab4b86f228efb5
Parents: 7161aec
Author: Yuki Morishita <yu...@apache.org>
Authored: Wed Sep 18 14:19:45 2013 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Wed Sep 18 14:19:45 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 14 +++++++---
.../cassandra/io/sstable/SSTableReader.java | 28 +++++++++++++++-----
3 files changed, 33 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9d7bb1e0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fb9915e..c6e1169 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -20,6 +20,7 @@
* Add SSTableDeletingNotification to DataTracker (CASSANDRA-6010)
* Fix snapshots in use get deleted during snapshot repair (CASSANDRA-6011)
* Move hints and exception count to o.a.c.metrics (CASSANDRA-6017)
+ * Fix memory leak in snapshot repair (CASSANDRA-6047)
1.2.9
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9d7bb1e0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 93f3108..5c17b0a 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -714,7 +714,8 @@ public class CompactionManager implements CompactionManagerMBean
Collection<SSTableReader> sstables;
int gcBefore;
- if (cfs.snapshotExists(validator.request.sessionid))
+ boolean isSnapshotValidation = cfs.snapshotExists(validator.request.sessionid);
+ if (isSnapshotValidation)
{
// If there is a snapshot created for the session then read from there.
sstables = cfs.getSnapshotSSTableReader(validator.request.sessionid);
@@ -768,10 +769,17 @@ public class CompactionManager implements CompactionManagerMBean
}
finally
{
- SSTableReader.releaseReferences(sstables);
iter.close();
- if (cfs.snapshotExists(validator.request.sessionid))
+ if (isSnapshotValidation)
+ {
+ for (SSTableReader sstable : sstables)
+ FileUtils.closeQuietly(sstable);
cfs.clearSnapshot(validator.request.sessionid);
+ }
+ else
+ {
+ SSTableReader.releaseReferences(sstables);
+ }
metrics.finishCompaction(ci);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9d7bb1e0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index 6327273..ed221d9 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@ -62,7 +62,7 @@ import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR
* SSTableReaders are open()ed by Table.onStart; after that they are created by SSTableWriter.renameAndOpen.
* Do not re-call open() on existing SSTable files; use the references kept by ColumnFamilyStore post-start instead.
*/
-public class SSTableReader extends SSTable
+public class SSTableReader extends SSTable implements Closeable
{
private static final Logger logger = LoggerFactory.getLogger(SSTableReader.class);
@@ -345,6 +345,20 @@ public class SSTableReader extends SSTable
this.bf = bloomFilter;
}
+ /**
+ * Clean up all opened resources.
+ *
+ * @throws IOException
+ */
+ public void close() throws IOException
+ {
+ // Force finalizing mmapping if necessary
+ ifile.cleanup();
+ dfile.cleanup();
+ // close the BF so it can be opened later.
+ bf.close();
+ }
+
public void setTrackedBy(DataTracker tracker)
{
deletingTask.setTracker(tracker);
@@ -969,17 +983,17 @@ public class SSTableReader extends SSTable
}
}
+ /**
+ * Release reference to this SSTableReader.
+ * If there is no one referring to this SSTable, and is marked as compacted,
+ * all resources are cleaned up and files are deleted eventually.
+ */
public void releaseReference()
{
if (references.decrementAndGet() == 0 && isCompacted.get())
{
- // Force finalizing mmapping if necessary
- ifile.cleanup();
- dfile.cleanup();
-
+ FileUtils.closeQuietly(this);
deletingTask.schedule();
- // close the BF so it can be opened later.
- FileUtils.closeQuietly(bf);
}
assert references.get() >= 0 : "Reference counter " + references.get() + " for " + dfile.path;
}
[3/6] git commit: Fix memory leak in snapshot repair
Posted by yu...@apache.org.
Fix memory leak in snapshot repair
patch by yukim; reviewed by jbellis for CASSANDRA-6047
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9d7bb1e0
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9d7bb1e0
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9d7bb1e0
Branch: refs/heads/trunk
Commit: 9d7bb1e0c27fd5076afc750150ab4b86f228efb5
Parents: 7161aec
Author: Yuki Morishita <yu...@apache.org>
Authored: Wed Sep 18 14:19:45 2013 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Wed Sep 18 14:19:45 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 14 +++++++---
.../cassandra/io/sstable/SSTableReader.java | 28 +++++++++++++++-----
3 files changed, 33 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9d7bb1e0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fb9915e..c6e1169 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -20,6 +20,7 @@
* Add SSTableDeletingNotification to DataTracker (CASSANDRA-6010)
* Fix snapshots in use get deleted during snapshot repair (CASSANDRA-6011)
* Move hints and exception count to o.a.c.metrics (CASSANDRA-6017)
+ * Fix memory leak in snapshot repair (CASSANDRA-6047)
1.2.9
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9d7bb1e0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 93f3108..5c17b0a 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -714,7 +714,8 @@ public class CompactionManager implements CompactionManagerMBean
Collection<SSTableReader> sstables;
int gcBefore;
- if (cfs.snapshotExists(validator.request.sessionid))
+ boolean isSnapshotValidation = cfs.snapshotExists(validator.request.sessionid);
+ if (isSnapshotValidation)
{
// If there is a snapshot created for the session then read from there.
sstables = cfs.getSnapshotSSTableReader(validator.request.sessionid);
@@ -768,10 +769,17 @@ public class CompactionManager implements CompactionManagerMBean
}
finally
{
- SSTableReader.releaseReferences(sstables);
iter.close();
- if (cfs.snapshotExists(validator.request.sessionid))
+ if (isSnapshotValidation)
+ {
+ for (SSTableReader sstable : sstables)
+ FileUtils.closeQuietly(sstable);
cfs.clearSnapshot(validator.request.sessionid);
+ }
+ else
+ {
+ SSTableReader.releaseReferences(sstables);
+ }
metrics.finishCompaction(ci);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9d7bb1e0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index 6327273..ed221d9 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@ -62,7 +62,7 @@ import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR
* SSTableReaders are open()ed by Table.onStart; after that they are created by SSTableWriter.renameAndOpen.
* Do not re-call open() on existing SSTable files; use the references kept by ColumnFamilyStore post-start instead.
*/
-public class SSTableReader extends SSTable
+public class SSTableReader extends SSTable implements Closeable
{
private static final Logger logger = LoggerFactory.getLogger(SSTableReader.class);
@@ -345,6 +345,20 @@ public class SSTableReader extends SSTable
this.bf = bloomFilter;
}
+ /**
+ * Clean up all opened resources.
+ *
+ * @throws IOException
+ */
+ public void close() throws IOException
+ {
+ // Force finalizing mmapping if necessary
+ ifile.cleanup();
+ dfile.cleanup();
+ // close the BF so it can be opened later.
+ bf.close();
+ }
+
public void setTrackedBy(DataTracker tracker)
{
deletingTask.setTracker(tracker);
@@ -969,17 +983,17 @@ public class SSTableReader extends SSTable
}
}
+ /**
+ * Release reference to this SSTableReader.
+ * If there is no one referring to this SSTable, and is marked as compacted,
+ * all resources are cleaned up and files are deleted eventually.
+ */
public void releaseReference()
{
if (references.decrementAndGet() == 0 && isCompacted.get())
{
- // Force finalizing mmapping if necessary
- ifile.cleanup();
- dfile.cleanup();
-
+ FileUtils.closeQuietly(this);
deletingTask.schedule();
- // close the BF so it can be opened later.
- FileUtils.closeQuietly(bf);
}
assert references.get() >= 0 : "Reference counter " + references.get() + " for " + dfile.path;
}
[6/6] git commit: Merge branch 'cassandra-2.0' into trunk
Posted by yu...@apache.org.
Merge branch 'cassandra-2.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a7820f9c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a7820f9c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a7820f9c
Branch: refs/heads/trunk
Commit: a7820f9cb5932bbcee263b9bb56227c57148ccc4
Parents: 0eca165 e9c027a
Author: Yuki Morishita <yu...@apache.org>
Authored: Wed Sep 18 14:24:58 2013 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Wed Sep 18 14:24:58 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 14 +++++++--
.../cassandra/io/sstable/SSTableReader.java | 30 ++++++++++++++------
3 files changed, 34 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a7820f9c/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a7820f9c/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
[5/6] git commit: Merge branch 'cassandra-1.2' into cassandra-2.0
Posted by yu...@apache.org.
Merge branch 'cassandra-1.2' into cassandra-2.0
Conflicts:
src/java/org/apache/cassandra/db/compaction/CompactionManager.java
src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e9c027a3
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e9c027a3
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e9c027a3
Branch: refs/heads/cassandra-2.0
Commit: e9c027a3a74d4be211b7f95308c7b2a106dc3b9e
Parents: 2648047 9d7bb1e
Author: Yuki Morishita <yu...@apache.org>
Authored: Wed Sep 18 14:23:52 2013 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Wed Sep 18 14:23:52 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 14 +++++++--
.../cassandra/io/sstable/SSTableReader.java | 30 ++++++++++++++------
3 files changed, 34 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e9c027a3/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index fd8d852,c6e1169..26fcc2f
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -47,54 -20,14 +47,55 @@@ Merged from 1.2
* Add SSTableDeletingNotification to DataTracker (CASSANDRA-6010)
* Fix snapshots in use get deleted during snapshot repair (CASSANDRA-6011)
* Move hints and exception count to o.a.c.metrics (CASSANDRA-6017)
+ * Fix memory leak in snapshot repair (CASSANDRA-6047)
-1.2.9
+2.0.0
+ * Fix thrift validation when inserting into CQL3 tables (CASSANDRA-5138)
+ * Fix periodic memtable flushing behavior with clean memtables (CASSANDRA-5931)
+ * Fix dateOf() function for pre-2.0 timestamp columns (CASSANDRA-5928)
+ * Fix SSTable unintentionally loads BF when opened for batch (CASSANDRA-5938)
+ * Add stream session progress to JMX (CASSANDRA-4757)
+ * Fix NPE during CAS operation (CASSANDRA-5925)
+Merged from 1.2:
* Fix getBloomFilterDiskSpaceUsed for AlwaysPresentFilter (CASSANDRA-5900)
- * migrate 1.1 schema_columnfamilies.key_alias column to key_aliases
- (CASSANDRA-5800)
- * add --migrate option to sstableupgrade and sstablescrub (CASSANDRA-5831)
+ * Don't announce schema version until we've loaded the changes locally
+ (CASSANDRA-5904)
+ * Fix to support off heap bloom filters size greater than 2 GB (CASSANDRA-5903)
+ * Properly handle parsing huge map and set literals (CASSANDRA-5893)
+
+
+2.0.0-rc2
+ * enable vnodes by default (CASSANDRA-5869)
+ * fix CAS contention timeout (CASSANDRA-5830)
+ * fix HsHa to respect max frame size (CASSANDRA-4573)
+ * Fix (some) 2i on composite components omissions (CASSANDRA-5851)
+ * cqlsh: add DESCRIBE FULL SCHEMA variant (CASSANDRA-5880)
+Merged from 1.2:
+ * Correctly validate sparse composite cells in scrub (CASSANDRA-5855)
+ * Add KeyCacheHitRate metric to CF metrics (CASSANDRA-5868)
+ * cqlsh: add support for multiline comments (CASSANDRA-5798)
+ * Handle CQL3 SELECT duplicate IN restrictions on clustering columns
+ (CASSANDRA-5856)
+
+
+2.0.0-rc1
+ * improve DecimalSerializer performance (CASSANDRA-5837)
+ * fix potential spurious wakeup in AsyncOneResponse (CASSANDRA-5690)
+ * fix schema-related trigger issues (CASSANDRA-5774)
+ * Better validation when accessing CQL3 table from thrift (CASSANDRA-5138)
+ * Fix assertion error during repair (CASSANDRA-5801)
+ * Fix range tombstone bug (CASSANDRA-5805)
+ * DC-local CAS (CASSANDRA-5797)
+ * Add a native_protocol_version column to the system.local table (CASSANRDA-5819)
+ * Use index_interval from cassandra.yaml when upgraded (CASSANDRA-5822)
+ * Fix buffer underflow on socket close (CASSANDRA-5792)
+Merged from 1.2:
+ * Fix reading DeletionTime from 1.1-format sstables (CASSANDRA-5814)
+ * cqlsh: add collections support to COPY (CASSANDRA-5698)
+ * retry important messages for any IOException (CASSANDRA-5804)
+ * Allow empty IN relations in SELECT/UPDATE/DELETE statements (CASSANDRA-5626)
+ * cqlsh: fix crashing on Windows due to libedit detection (CASSANDRA-5812)
* fix bulk-loading compressed sstables (CASSANDRA-5820)
* (Hadoop) fix quoting in CqlPagingRecordReader and CqlRecordWriter
(CASSANDRA-5824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e9c027a3/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 2f9aed2,5c17b0a..f3c2011
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -755,12 -713,12 +755,13 @@@ public class CompactionManager implemen
return;
Collection<SSTableReader> sstables;
+ String snapshotName = validator.desc.sessionId.toString();
int gcBefore;
- if (cfs.snapshotExists(snapshotName))
- boolean isSnapshotValidation = cfs.snapshotExists(validator.request.sessionid);
++ boolean isSnapshotValidation = cfs.snapshotExists(snapshotName);
+ if (isSnapshotValidation)
{
// If there is a snapshot created for the session then read from there.
- sstables = cfs.getSnapshotSSTableReader(validator.request.sessionid);
+ sstables = cfs.getSnapshotSSTableReader(snapshotName);
// Computing gcbefore based on the current time wouldn't be very good because we know each replica will execute
// this at a different time (that's the whole purpose of repair with snaphsot). So instead we take the creation
@@@ -800,10 -769,17 +801,17 @@@
}
finally
{
- SSTableReader.releaseReferences(sstables);
iter.close();
- if (cfs.snapshotExists(snapshotName))
+ if (isSnapshotValidation)
+ {
+ for (SSTableReader sstable : sstables)
+ FileUtils.closeQuietly(sstable);
- cfs.clearSnapshot(validator.request.sessionid);
+ cfs.clearSnapshot(snapshotName);
+ }
+ else
+ {
+ SSTableReader.releaseReferences(sstables);
+ }
metrics.finishCompaction(ci);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e9c027a3/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index b5f7d22,ed221d9..e0c096f
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@@ -61,10 -59,10 +61,10 @@@ import org.apache.cassandra.utils.*
import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR;
/**
- * SSTableReaders are open()ed by Table.onStart; after that they are created by SSTableWriter.renameAndOpen.
+ * SSTableReaders are open()ed by Keyspace.onStart; after that they are created by SSTableWriter.renameAndOpen.
* Do not re-call open() on existing SSTable files; use the references kept by ColumnFamilyStore post-start instead.
*/
- public class SSTableReader extends SSTable
+ public class SSTableReader extends SSTable implements Closeable
{
private static final Logger logger = LoggerFactory.getLogger(SSTableReader.class);
@@@ -335,6 -345,20 +335,21 @@@
this.bf = bloomFilter;
}
+ /**
+ * Clean up all opened resources.
+ *
+ * @throws IOException
+ */
+ public void close() throws IOException
+ {
+ // Force finalizing mmapping if necessary
+ ifile.cleanup();
+ dfile.cleanup();
+ // close the BF so it can be opened later.
+ bf.close();
++ indexSummary.close();
+ }
+
public void setTrackedBy(DataTracker tracker)
{
deletingTask.setTracker(tracker);
@@@ -1032,23 -992,8 +1052,17 @@@
{
if (references.decrementAndGet() == 0 && isCompacted.get())
{
+ /**
+ * Make OS a favour and suggest (using fadvice call) that we
+ * don't want to see pages of this SSTable in memory anymore.
+ *
+ * NOTE: We can't use madvice in java because it requires address of
+ * the mapping, so instead we always open a file and run fadvice(fd, 0, 0) on it
+ */
+ dropPageCache();
+
- // Force finalizing mmapping if necessary
- ifile.cleanup();
- dfile.cleanup();
-
+ FileUtils.closeQuietly(this);
deletingTask.schedule();
- // close the BF so it can be opened later.
- FileUtils.closeQuietly(bf);
- FileUtils.closeQuietly(indexSummary);
}
assert references.get() >= 0 : "Reference counter " + references.get() + " for " + dfile.path;
}
[2/6] git commit: Fix memory leak in snapshot repair
Posted by yu...@apache.org.
Fix memory leak in snapshot repair
patch by yukim; reviewed by jbellis for CASSANDRA-6047
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9d7bb1e0
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9d7bb1e0
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9d7bb1e0
Branch: refs/heads/cassandra-2.0
Commit: 9d7bb1e0c27fd5076afc750150ab4b86f228efb5
Parents: 7161aec
Author: Yuki Morishita <yu...@apache.org>
Authored: Wed Sep 18 14:19:45 2013 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Wed Sep 18 14:19:45 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 14 +++++++---
.../cassandra/io/sstable/SSTableReader.java | 28 +++++++++++++++-----
3 files changed, 33 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9d7bb1e0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fb9915e..c6e1169 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -20,6 +20,7 @@
* Add SSTableDeletingNotification to DataTracker (CASSANDRA-6010)
* Fix snapshots in use get deleted during snapshot repair (CASSANDRA-6011)
* Move hints and exception count to o.a.c.metrics (CASSANDRA-6017)
+ * Fix memory leak in snapshot repair (CASSANDRA-6047)
1.2.9
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9d7bb1e0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 93f3108..5c17b0a 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -714,7 +714,8 @@ public class CompactionManager implements CompactionManagerMBean
Collection<SSTableReader> sstables;
int gcBefore;
- if (cfs.snapshotExists(validator.request.sessionid))
+ boolean isSnapshotValidation = cfs.snapshotExists(validator.request.sessionid);
+ if (isSnapshotValidation)
{
// If there is a snapshot created for the session then read from there.
sstables = cfs.getSnapshotSSTableReader(validator.request.sessionid);
@@ -768,10 +769,17 @@ public class CompactionManager implements CompactionManagerMBean
}
finally
{
- SSTableReader.releaseReferences(sstables);
iter.close();
- if (cfs.snapshotExists(validator.request.sessionid))
+ if (isSnapshotValidation)
+ {
+ for (SSTableReader sstable : sstables)
+ FileUtils.closeQuietly(sstable);
cfs.clearSnapshot(validator.request.sessionid);
+ }
+ else
+ {
+ SSTableReader.releaseReferences(sstables);
+ }
metrics.finishCompaction(ci);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9d7bb1e0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index 6327273..ed221d9 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@ -62,7 +62,7 @@ import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR
* SSTableReaders are open()ed by Table.onStart; after that they are created by SSTableWriter.renameAndOpen.
* Do not re-call open() on existing SSTable files; use the references kept by ColumnFamilyStore post-start instead.
*/
-public class SSTableReader extends SSTable
+public class SSTableReader extends SSTable implements Closeable
{
private static final Logger logger = LoggerFactory.getLogger(SSTableReader.class);
@@ -345,6 +345,20 @@ public class SSTableReader extends SSTable
this.bf = bloomFilter;
}
+ /**
+ * Clean up all opened resources.
+ *
+ * @throws IOException
+ */
+ public void close() throws IOException
+ {
+ // Force finalizing mmapping if necessary
+ ifile.cleanup();
+ dfile.cleanup();
+ // close the BF so it can be opened later.
+ bf.close();
+ }
+
public void setTrackedBy(DataTracker tracker)
{
deletingTask.setTracker(tracker);
@@ -969,17 +983,17 @@ public class SSTableReader extends SSTable
}
}
+ /**
+ * Release reference to this SSTableReader.
+ * If there is no one referring to this SSTable, and is marked as compacted,
+ * all resources are cleaned up and files are deleted eventually.
+ */
public void releaseReference()
{
if (references.decrementAndGet() == 0 && isCompacted.get())
{
- // Force finalizing mmapping if necessary
- ifile.cleanup();
- dfile.cleanup();
-
+ FileUtils.closeQuietly(this);
deletingTask.schedule();
- // close the BF so it can be opened later.
- FileUtils.closeQuietly(bf);
}
assert references.get() >= 0 : "Reference counter " + references.get() + " for " + dfile.path;
}
[4/6] git commit: Merge branch 'cassandra-1.2' into cassandra-2.0
Posted by yu...@apache.org.
Merge branch 'cassandra-1.2' into cassandra-2.0
Conflicts:
src/java/org/apache/cassandra/db/compaction/CompactionManager.java
src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e9c027a3
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e9c027a3
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e9c027a3
Branch: refs/heads/trunk
Commit: e9c027a3a74d4be211b7f95308c7b2a106dc3b9e
Parents: 2648047 9d7bb1e
Author: Yuki Morishita <yu...@apache.org>
Authored: Wed Sep 18 14:23:52 2013 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Wed Sep 18 14:23:52 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 14 +++++++--
.../cassandra/io/sstable/SSTableReader.java | 30 ++++++++++++++------
3 files changed, 34 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e9c027a3/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index fd8d852,c6e1169..26fcc2f
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -47,54 -20,14 +47,55 @@@ Merged from 1.2
* Add SSTableDeletingNotification to DataTracker (CASSANDRA-6010)
* Fix snapshots in use get deleted during snapshot repair (CASSANDRA-6011)
* Move hints and exception count to o.a.c.metrics (CASSANDRA-6017)
+ * Fix memory leak in snapshot repair (CASSANDRA-6047)
-1.2.9
+2.0.0
+ * Fix thrift validation when inserting into CQL3 tables (CASSANDRA-5138)
+ * Fix periodic memtable flushing behavior with clean memtables (CASSANDRA-5931)
+ * Fix dateOf() function for pre-2.0 timestamp columns (CASSANDRA-5928)
+ * Fix SSTable unintentionally loads BF when opened for batch (CASSANDRA-5938)
+ * Add stream session progress to JMX (CASSANDRA-4757)
+ * Fix NPE during CAS operation (CASSANDRA-5925)
+Merged from 1.2:
* Fix getBloomFilterDiskSpaceUsed for AlwaysPresentFilter (CASSANDRA-5900)
- * migrate 1.1 schema_columnfamilies.key_alias column to key_aliases
- (CASSANDRA-5800)
- * add --migrate option to sstableupgrade and sstablescrub (CASSANDRA-5831)
+ * Don't announce schema version until we've loaded the changes locally
+ (CASSANDRA-5904)
+ * Fix to support off heap bloom filters size greater than 2 GB (CASSANDRA-5903)
+ * Properly handle parsing huge map and set literals (CASSANDRA-5893)
+
+
+2.0.0-rc2
+ * enable vnodes by default (CASSANDRA-5869)
+ * fix CAS contention timeout (CASSANDRA-5830)
+ * fix HsHa to respect max frame size (CASSANDRA-4573)
+ * Fix (some) 2i on composite components omissions (CASSANDRA-5851)
+ * cqlsh: add DESCRIBE FULL SCHEMA variant (CASSANDRA-5880)
+Merged from 1.2:
+ * Correctly validate sparse composite cells in scrub (CASSANDRA-5855)
+ * Add KeyCacheHitRate metric to CF metrics (CASSANDRA-5868)
+ * cqlsh: add support for multiline comments (CASSANDRA-5798)
+ * Handle CQL3 SELECT duplicate IN restrictions on clustering columns
+ (CASSANDRA-5856)
+
+
+2.0.0-rc1
+ * improve DecimalSerializer performance (CASSANDRA-5837)
+ * fix potential spurious wakeup in AsyncOneResponse (CASSANDRA-5690)
+ * fix schema-related trigger issues (CASSANDRA-5774)
+ * Better validation when accessing CQL3 table from thrift (CASSANDRA-5138)
+ * Fix assertion error during repair (CASSANDRA-5801)
+ * Fix range tombstone bug (CASSANDRA-5805)
+ * DC-local CAS (CASSANDRA-5797)
+ * Add a native_protocol_version column to the system.local table (CASSANRDA-5819)
+ * Use index_interval from cassandra.yaml when upgraded (CASSANDRA-5822)
+ * Fix buffer underflow on socket close (CASSANDRA-5792)
+Merged from 1.2:
+ * Fix reading DeletionTime from 1.1-format sstables (CASSANDRA-5814)
+ * cqlsh: add collections support to COPY (CASSANDRA-5698)
+ * retry important messages for any IOException (CASSANDRA-5804)
+ * Allow empty IN relations in SELECT/UPDATE/DELETE statements (CASSANDRA-5626)
+ * cqlsh: fix crashing on Windows due to libedit detection (CASSANDRA-5812)
* fix bulk-loading compressed sstables (CASSANDRA-5820)
* (Hadoop) fix quoting in CqlPagingRecordReader and CqlRecordWriter
(CASSANDRA-5824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e9c027a3/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 2f9aed2,5c17b0a..f3c2011
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -755,12 -713,12 +755,13 @@@ public class CompactionManager implemen
return;
Collection<SSTableReader> sstables;
+ String snapshotName = validator.desc.sessionId.toString();
int gcBefore;
- if (cfs.snapshotExists(snapshotName))
- boolean isSnapshotValidation = cfs.snapshotExists(validator.request.sessionid);
++ boolean isSnapshotValidation = cfs.snapshotExists(snapshotName);
+ if (isSnapshotValidation)
{
// If there is a snapshot created for the session then read from there.
- sstables = cfs.getSnapshotSSTableReader(validator.request.sessionid);
+ sstables = cfs.getSnapshotSSTableReader(snapshotName);
// Computing gcbefore based on the current time wouldn't be very good because we know each replica will execute
// this at a different time (that's the whole purpose of repair with snaphsot). So instead we take the creation
@@@ -800,10 -769,17 +801,17 @@@
}
finally
{
- SSTableReader.releaseReferences(sstables);
iter.close();
- if (cfs.snapshotExists(snapshotName))
+ if (isSnapshotValidation)
+ {
+ for (SSTableReader sstable : sstables)
+ FileUtils.closeQuietly(sstable);
- cfs.clearSnapshot(validator.request.sessionid);
+ cfs.clearSnapshot(snapshotName);
+ }
+ else
+ {
+ SSTableReader.releaseReferences(sstables);
+ }
metrics.finishCompaction(ci);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e9c027a3/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index b5f7d22,ed221d9..e0c096f
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@@ -61,10 -59,10 +61,10 @@@ import org.apache.cassandra.utils.*
import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR;
/**
- * SSTableReaders are open()ed by Table.onStart; after that they are created by SSTableWriter.renameAndOpen.
+ * SSTableReaders are open()ed by Keyspace.onStart; after that they are created by SSTableWriter.renameAndOpen.
* Do not re-call open() on existing SSTable files; use the references kept by ColumnFamilyStore post-start instead.
*/
- public class SSTableReader extends SSTable
+ public class SSTableReader extends SSTable implements Closeable
{
private static final Logger logger = LoggerFactory.getLogger(SSTableReader.class);
@@@ -335,6 -345,20 +335,21 @@@
this.bf = bloomFilter;
}
+ /**
+ * Clean up all opened resources.
+ *
+ * @throws IOException
+ */
+ public void close() throws IOException
+ {
+ // Force finalizing mmapping if necessary
+ ifile.cleanup();
+ dfile.cleanup();
+ // close the BF so it can be opened later.
+ bf.close();
++ indexSummary.close();
+ }
+
public void setTrackedBy(DataTracker tracker)
{
deletingTask.setTracker(tracker);
@@@ -1032,23 -992,8 +1052,17 @@@
{
if (references.decrementAndGet() == 0 && isCompacted.get())
{
+ /**
+ * Make OS a favour and suggest (using fadvice call) that we
+ * don't want to see pages of this SSTable in memory anymore.
+ *
+ * NOTE: We can't use madvice in java because it requires address of
+ * the mapping, so instead we always open a file and run fadvice(fd, 0, 0) on it
+ */
+ dropPageCache();
+
- // Force finalizing mmapping if necessary
- ifile.cleanup();
- dfile.cleanup();
-
+ FileUtils.closeQuietly(this);
deletingTask.schedule();
- // close the BF so it can be opened later.
- FileUtils.closeQuietly(bf);
- FileUtils.closeQuietly(indexSummary);
}
assert references.get() >= 0 : "Reference counter " + references.get() + " for " + dfile.path;
}