You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2013/12/21 07:36:42 UTC
[1/3] git commit: Allow removing snapshots of no-longer-existing CFs
patch by Lyuben Todorov and Mikhail Stepura for CASSANDRA-6418
Updated Branches:
refs/heads/cassandra-2.0 142e13f34 -> b5c9b498f
refs/heads/trunk b34ed0db5 -> d7536612f
Allow removing snapshots of no-longer-existing CFs
patch by Lyuben Todorov and Mikhail Stepura for CASSANDRA-6418
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b5c9b498
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b5c9b498
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b5c9b498
Branch: refs/heads/cassandra-2.0
Commit: b5c9b498fec1b71bc51afb3e45f828f4014baaff
Parents: 142e13f
Author: Jonathan Ellis <jb...@apache.org>
Authored: Sat Dec 21 00:36:30 2013 -0600
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Sat Dec 21 00:36:30 2013 -0600
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 9 +++++-
.../org/apache/cassandra/db/Directories.java | 34 ++++++++++++++++++--
src/java/org/apache/cassandra/db/Keyspace.java | 6 ++--
4 files changed, 43 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b5c9b498/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 916ed7d..b80c821 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.4
+ * Allow removing snapshots of no-longer-existing CFs (CASSANDRA-6418)
* add StorageService.stopDaemon() (CASSANDRA-4268)
* add IRE for invalid CF supplied to get_count (CASSANDRA-5701)
* add client encryption support to sstableloader (CASSANDRA-6378)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b5c9b498/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index d585407..a04c9e2 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1825,9 +1825,16 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
return directories.snapshotCreationTime(snapshotName);
}
+ /**
+ * Clear all the snapshots for a given column family.
+ *
+ * @param snapshotName the user supplied snapshot name. If left empty,
+ * all the snapshots will be cleaned.
+ */
public void clearSnapshot(String snapshotName)
{
- directories.clearSnapshot(snapshotName);
+ List<File> snapshotDirs = directories.getCFDirectories();
+ Directories.clearSnapshot(snapshotName, snapshotDirs);
}
public boolean hasUnreclaimedSpace()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b5c9b498/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index b006c71..2db4d9b 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -402,11 +402,11 @@ public class Directories
return false;
}
- public void clearSnapshot(String snapshotName)
+ public static void clearSnapshot(String snapshotName, List<File> snapshotDirectories)
{
// If snapshotName is empty or null, we will delete the entire snapshot directory
String tag = snapshotName == null ? "" : snapshotName;
- for (File dir : sstableDirectories)
+ for (File dir : snapshotDirectories)
{
File snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, tag));
if (snapshotDir.exists())
@@ -430,6 +430,36 @@ public class Directories
throw new RuntimeException("Snapshot " + snapshotName + " doesn't exist");
}
+ // Recursively finds all the sub directories in the KS directory.
+ public static List<File> getKSChildDirectories(String ksName)
+ {
+ List<File> result = new ArrayList<File>();
+ for (DataDirectory dataDirectory : dataFileLocations)
+ {
+ File ksDir = new File(dataDirectory.location, ksName);
+ File[] cfDirs = ksDir.listFiles();
+ if (cfDirs == null)
+ continue;
+ for (File cfDir : cfDirs)
+ {
+ if (cfDir.isDirectory())
+ result.add(cfDir);
+ }
+ }
+ return result;
+ }
+
+ public List<File> getCFDirectories()
+ {
+ List<File> result = new ArrayList<File>();
+ for (File dataDirectory : sstableDirectories)
+ {
+ if (dataDirectory.isDirectory())
+ result.add(dataDirectory);
+ }
+ return result;
+ }
+
private static File getOrCreate(File base, String... subdirs)
{
File dir = subdirs == null || subdirs.length == 0 ? base : new File(base, join(subdirs));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b5c9b498/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index 0280ed2..36789e2 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -239,10 +239,8 @@ public class Keyspace
*/
public void clearSnapshot(String snapshotName)
{
- for (ColumnFamilyStore cfStore : columnFamilyStores.values())
- {
- cfStore.clearSnapshot(snapshotName);
- }
+ List<File> snapshotDirs = Directories.getKSChildDirectories(getName());
+ Directories.clearSnapshot(snapshotName, snapshotDirs);
}
/**
[2/3] git commit: Allow removing snapshots of no-longer-existing CFs
patch by Lyuben Todorov and Mikhail Stepura for CASSANDRA-6418
Posted by jb...@apache.org.
Allow removing snapshots of no-longer-existing CFs
patch by Lyuben Todorov and Mikhail Stepura for CASSANDRA-6418
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b5c9b498
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b5c9b498
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b5c9b498
Branch: refs/heads/trunk
Commit: b5c9b498fec1b71bc51afb3e45f828f4014baaff
Parents: 142e13f
Author: Jonathan Ellis <jb...@apache.org>
Authored: Sat Dec 21 00:36:30 2013 -0600
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Sat Dec 21 00:36:30 2013 -0600
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 9 +++++-
.../org/apache/cassandra/db/Directories.java | 34 ++++++++++++++++++--
src/java/org/apache/cassandra/db/Keyspace.java | 6 ++--
4 files changed, 43 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b5c9b498/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 916ed7d..b80c821 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.4
+ * Allow removing snapshots of no-longer-existing CFs (CASSANDRA-6418)
* add StorageService.stopDaemon() (CASSANDRA-4268)
* add IRE for invalid CF supplied to get_count (CASSANDRA-5701)
* add client encryption support to sstableloader (CASSANDRA-6378)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b5c9b498/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index d585407..a04c9e2 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1825,9 +1825,16 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
return directories.snapshotCreationTime(snapshotName);
}
+ /**
+ * Clear all the snapshots for a given column family.
+ *
+ * @param snapshotName the user supplied snapshot name. If left empty,
+ * all the snapshots will be cleaned.
+ */
public void clearSnapshot(String snapshotName)
{
- directories.clearSnapshot(snapshotName);
+ List<File> snapshotDirs = directories.getCFDirectories();
+ Directories.clearSnapshot(snapshotName, snapshotDirs);
}
public boolean hasUnreclaimedSpace()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b5c9b498/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index b006c71..2db4d9b 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -402,11 +402,11 @@ public class Directories
return false;
}
- public void clearSnapshot(String snapshotName)
+ public static void clearSnapshot(String snapshotName, List<File> snapshotDirectories)
{
// If snapshotName is empty or null, we will delete the entire snapshot directory
String tag = snapshotName == null ? "" : snapshotName;
- for (File dir : sstableDirectories)
+ for (File dir : snapshotDirectories)
{
File snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, tag));
if (snapshotDir.exists())
@@ -430,6 +430,36 @@ public class Directories
throw new RuntimeException("Snapshot " + snapshotName + " doesn't exist");
}
+ // Recursively finds all the sub directories in the KS directory.
+ public static List<File> getKSChildDirectories(String ksName)
+ {
+ List<File> result = new ArrayList<File>();
+ for (DataDirectory dataDirectory : dataFileLocations)
+ {
+ File ksDir = new File(dataDirectory.location, ksName);
+ File[] cfDirs = ksDir.listFiles();
+ if (cfDirs == null)
+ continue;
+ for (File cfDir : cfDirs)
+ {
+ if (cfDir.isDirectory())
+ result.add(cfDir);
+ }
+ }
+ return result;
+ }
+
+ public List<File> getCFDirectories()
+ {
+ List<File> result = new ArrayList<File>();
+ for (File dataDirectory : sstableDirectories)
+ {
+ if (dataDirectory.isDirectory())
+ result.add(dataDirectory);
+ }
+ return result;
+ }
+
private static File getOrCreate(File base, String... subdirs)
{
File dir = subdirs == null || subdirs.length == 0 ? base : new File(base, join(subdirs));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b5c9b498/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index 0280ed2..36789e2 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -239,10 +239,8 @@ public class Keyspace
*/
public void clearSnapshot(String snapshotName)
{
- for (ColumnFamilyStore cfStore : columnFamilyStores.values())
- {
- cfStore.clearSnapshot(snapshotName);
- }
+ List<File> snapshotDirs = Directories.getKSChildDirectories(getName());
+ Directories.clearSnapshot(snapshotName, snapshotDirs);
}
/**
[3/3] git commit: Merge branch 'cassandra-2.0' into trunk
Posted by jb...@apache.org.
Merge branch 'cassandra-2.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d7536612
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d7536612
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d7536612
Branch: refs/heads/trunk
Commit: d7536612f3d828e9132a8cdc32732ec8516ac991
Parents: b34ed0d b5c9b49
Author: Jonathan Ellis <jb...@apache.org>
Authored: Sat Dec 21 00:36:35 2013 -0600
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Sat Dec 21 00:36:35 2013 -0600
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 9 +++++-
.../org/apache/cassandra/db/Directories.java | 34 ++++++++++++++++++--
src/java/org/apache/cassandra/db/Keyspace.java | 6 ++--
4 files changed, 43 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7536612/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index c9dcc18,b80c821..8c698cd
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,27 -1,5 +1,28 @@@
+2.1
+ * Multithreaded commitlog (CASSANDRA-3578)
+ * allocate fixed index summary memory pool and resample cold index summaries
+ to use less memory (CASSANDRA-5519)
+ * Removed multithreaded compaction (CASSANDRA-6142)
+ * Parallelize fetching rows for low-cardinality indexes (CASSANDRA-1337)
+ * change logging from log4j to logback (CASSANDRA-5883)
+ * switch to LZ4 compression for internode communication (CASSANDRA-5887)
+ * Stop using Thrift-generated Index* classes internally (CASSANDRA-5971)
+ * Remove 1.2 network compatibility code (CASSANDRA-5960)
+ * Remove leveled json manifest migration code (CASSANDRA-5996)
+ * Remove CFDefinition (CASSANDRA-6253)
+ * Use AtomicIntegerFieldUpdater in RefCountedMemory (CASSANDRA-6278)
+ * User-defined types for CQL3 (CASSANDRA-5590)
+ * Use of o.a.c.metrics in nodetool (CASSANDRA-5871, 6406)
+ * Batch read from OTC's queue and cleanup (CASSANDRA-1632)
+ * Secondary index support for collections (CASSANDRA-4511)
+ * SSTable metadata(Stats.db) format change (CASSANDRA-6356)
+ * Push composites support in the storage engine (CASSANDRA-5417)
+ * Add snapshot space used to cfstats (CASSANDRA-6231)
+ * Add cardinality estimator for key count estimation (CASSANDRA-5906)
+
+
2.0.4
+ * Allow removing snapshots of no-longer-existing CFs (CASSANDRA-6418)
* add StorageService.stopDaemon() (CASSANDRA-4268)
* add IRE for invalid CF supplied to get_count (CASSANDRA-5701)
* add client encryption support to sstableloader (CASSANDRA-6378)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7536612/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7536612/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Directories.java
index 81d15d1,2db4d9b..6f00d6d
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@@ -483,38 -429,37 +483,68 @@@ public class Directorie
}
throw new RuntimeException("Snapshot " + snapshotName + " doesn't exist");
}
+
+ public long trueSnapshotsSize()
+ {
+ long result = 0L;
+ for (File dir : sstableDirectories)
+ result += getTrueAllocatedSizeIn(new File(dir, join(SNAPSHOT_SUBDIR)));
+ return result;
+ }
+
+ private String getSSTablePrefix()
+ {
+ return keyspacename + Component.separator + cfname + Component.separator;
+ }
+
+ public long getTrueAllocatedSizeIn(File input)
+ {
+ if (!input.isDirectory())
+ return 0;
+
+ TrueFilesSizeVisitor visitor = new TrueFilesSizeVisitor();
+ try
+ {
+ Files.walkFileTree(input.toPath(), visitor);
+ }
+ catch (IOException e)
+ {
+ logger.error("Could not calculate the size of {}. {}", input, e);
+ }
+
+ return visitor.getAllocatedSize();
+ }
+ // Recursively finds all the sub directories in the KS directory.
+ public static List<File> getKSChildDirectories(String ksName)
+ {
+ List<File> result = new ArrayList<File>();
+ for (DataDirectory dataDirectory : dataFileLocations)
+ {
+ File ksDir = new File(dataDirectory.location, ksName);
+ File[] cfDirs = ksDir.listFiles();
+ if (cfDirs == null)
+ continue;
+ for (File cfDir : cfDirs)
+ {
+ if (cfDir.isDirectory())
+ result.add(cfDir);
+ }
+ }
+ return result;
+ }
+
+ public List<File> getCFDirectories()
+ {
+ List<File> result = new ArrayList<File>();
+ for (File dataDirectory : sstableDirectories)
+ {
+ if (dataDirectory.isDirectory())
+ result.add(dataDirectory);
+ }
+ return result;
+ }
+
private static File getOrCreate(File base, String... subdirs)
{
File dir = subdirs == null || subdirs.length == 0 ? base : new File(base, join(subdirs));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7536612/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------