You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2016/01/22 16:43:09 UTC
cassandra git commit: Add --skip-flush option to nodetool snapshot
Repository: cassandra
Updated Branches:
refs/heads/trunk 2f984e330 -> ee9e38b7d
Add --skip-flush option to nodetool snapshot
patch by anubhavkale; reviewed by pauloricardomg for CASSANDRA-10907
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ee9e38b7
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ee9e38b7
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ee9e38b7
Branch: refs/heads/trunk
Commit: ee9e38b7dc20a3b98e5a68904be35d727ed9c09c
Parents: 2f984e3
Author: anubhavkale <an...@microsoft.com>
Authored: Tue Jan 19 12:56:29 2016 -0800
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Jan 22 16:42:55 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +-
.../apache/cassandra/db/ColumnFamilyStore.java | 21 +++-
src/java/org/apache/cassandra/db/Keyspace.java | 18 +++-
.../repair/RepairMessageVerbHandler.java | 2 +-
.../cassandra/service/StorageService.java | 106 +++++++++++++------
.../cassandra/service/StorageServiceMBean.java | 27 ++---
.../org/apache/cassandra/tools/NodeProbe.java | 15 +--
.../cassandra/tools/nodetool/Snapshot.java | 15 ++-
.../cassandra/db/ColumnFamilyStoreTest.java | 4 +-
.../service/StorageServiceServerTest.java | 13 ++-
10 files changed, 156 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3f80264..b8314e5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.4
+ * Add --skip-flush option to nodetool snapshot
* Skip values for non-queried columns (CASSANDRA-10657)
* Add support for secondary indexes on static columns (CASSANDRA-8103)
* CommitLogUpgradeTestMaker creates broken commit logs (CASSANDRA-11051)
@@ -9,7 +10,6 @@
* Add nodetool gettimeout and settimeout commands (CASSANDRA-10953)
* Add 3.0 metadata to sstablemetadata output (CASSANDRA-10838)
-
3.3
* Avoid bootstrap hanging when existing nodes have no data to stream (CASSANDRA-11010)
Merged from 3.0:
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 486808b..52d8747 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1749,16 +1749,31 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
*/
public Set<SSTableReader> snapshot(String snapshotName)
{
- return snapshot(snapshotName, null, false);
+ return snapshot(snapshotName, false);
+ }
+
+ /**
+ * Take a snap shot of this columnfamily store.
+ *
+ * @param snapshotName the name of the associated with the snapshot
+ * @param skipFlush Skip blocking flush of memtable
+ */
+ public Set<SSTableReader> snapshot(String snapshotName, boolean skipFlush)
+ {
+ return snapshot(snapshotName, null, false, skipFlush);
}
/**
* @param ephemeral If this flag is set to true, the snapshot will be cleaned up during next startup
+ * @param skipFlush Skip blocking flush of memtable
*/
- public Set<SSTableReader> snapshot(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral)
+ public Set<SSTableReader> snapshot(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral, boolean skipFlush)
{
- forceBlockingFlush();
+ if (!skipFlush)
+ {
+ forceBlockingFlush();
+ }
return snapshotWithoutFlush(snapshotName, predicate, ephemeral);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index 0ec94ea..6122479 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -219,9 +219,10 @@ public class Keyspace
*
* @param snapshotName the tag associated with the name of the snapshot. This value may not be null
* @param columnFamilyName the column family to snapshot or all on null
+ * @param skipFlush Skip blocking flush of memtable
* @throws IOException if the column family doesn't exist
*/
- public void snapshot(String snapshotName, String columnFamilyName) throws IOException
+ public void snapshot(String snapshotName, String columnFamilyName, boolean skipFlush) throws IOException
{
assert snapshotName != null;
boolean tookSnapShot = false;
@@ -230,7 +231,7 @@ public class Keyspace
if (columnFamilyName == null || cfStore.name.equals(columnFamilyName))
{
tookSnapShot = true;
- cfStore.snapshot(snapshotName);
+ cfStore.snapshot(snapshotName, skipFlush);
}
}
@@ -239,6 +240,19 @@ public class Keyspace
}
/**
+ * Take a snapshot of the specific column family, or the entire set of column families
+ * if columnFamily is null with a given timestamp
+ *
+ * @param snapshotName the tag associated with the name of the snapshot. This value may not be null
+ * @param columnFamilyName the column family to snapshot or all on null
+ * @throws IOException if the column family doesn't exist
+ */
+ public void snapshot(String snapshotName, String columnFamilyName) throws IOException
+ {
+ snapshot(snapshotName, columnFamilyName, false);
+ }
+
+ /**
* @param clientSuppliedName may be null.
* @return the name of the snapshot
*/
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
index 9ac859f..6c04729 100644
--- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
+++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
@@ -90,7 +90,7 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
!sstable.metadata.isIndex() && // exclude SSTables from 2i
new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(repairingRange);
}
- }, true); //ephemeral snapshot, if repair fails, it will be cleaned next startup
+ }, true, false); //ephemeral snapshot, if repair fails, it will be cleaned next startup
Set<SSTableReader> currentlyRepairing = ActiveRepairService.instance.currentlyRepairing(cfs.metadata.cfId, desc.parentSessionId);
if (!Sets.intersection(currentlyRepairing, snapshottedSSSTables).isEmpty())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index fd6436d..932d949 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -2775,6 +2775,47 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
}
/**
+ * Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified.
+ *
+ * @param tag
+ * the tag given to the snapshot; may not be null or empty
+ * @param options
+ * Map of options (skipFlush is the only supported option for now)
+ * @param entities
+ * list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,...
+ */
+ @Override
+ public void takeSnapshot(String tag, Map<String, String> options, String... entities) throws IOException
+ {
+ boolean skipFlush = Boolean.parseBoolean(options.getOrDefault("skipFlush", "false"));
+
+ if (entities != null && entities.length > 0 && entities[0].contains("."))
+ {
+ takeMultipleTableSnapshot(tag, skipFlush, entities);
+ }
+ else
+ {
+ takeSnapshot(tag, skipFlush, entities);
+ }
+ }
+
+ /**
+ * Takes the snapshot of a specific table. A snapshot name must be
+ * specified.
+ *
+ * @param keyspaceName
+ * the keyspace which holds the specified table
+ * @param tableName
+ * the table to snapshot
+ * @param tag
+ * the tag given to the snapshot; may not be null or empty
+ */
+ public void takeTableSnapshot(String keyspaceName, String tableName, String tag)
+ throws IOException {
+ takeMultipleTableSnapshot(tag, false, keyspaceName + "." + tableName);
+ }
+
+ /**
* Takes the snapshot for the given keyspaces. A snapshot name must be specified.
*
* @param tag the tag given to the snapshot; may not be null or empty
@@ -2782,6 +2823,32 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
*/
public void takeSnapshot(String tag, String... keyspaceNames) throws IOException
{
+ takeSnapshot(tag, false, keyspaceNames);
+ }
+
+ /**
+ * Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified.
+ *
+ * @param tag
+ * the tag given to the snapshot; may not be null or empty
+ * @param tableList
+ * list of tables from different keyspace in the form of ks1.cf1 ks2.cf2
+ */
+ public void takeMultipleTableSnapshot(String tag, String... tableList)
+ throws IOException
+ {
+ takeMultipleTableSnapshot(tag, false, tableList);
+ }
+
+ /**
+ * Takes the snapshot for the given keyspaces. A snapshot name must be specified.
+ *
+ * @param tag the tag given to the snapshot; may not be null or empty
+ * @param skipFlush Skip blocking flush of memtable
+ * @param keyspaceNames the names of the keyspaces to snapshot; empty means "all."
+ */
+ private void takeSnapshot(String tag, boolean skipFlush, String... keyspaceNames) throws IOException
+ {
if (operationMode == Mode.JOINING)
throw new IOException("Cannot snapshot until bootstrap completes");
if (tag == null || tag.equals(""))
@@ -2807,37 +2874,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
for (Keyspace keyspace : keyspaces)
- keyspace.snapshot(tag, null);
- }
-
- /**
- * Takes the snapshot of a specific table. A snapshot name must be specified.
- *
- * @param keyspaceName the keyspace which holds the specified table
- * @param tableName the table to snapshot
- * @param tag the tag given to the snapshot; may not be null or empty
- */
- public void takeTableSnapshot(String keyspaceName, String tableName, String tag) throws IOException
- {
- if (keyspaceName == null)
- throw new IOException("You must supply a keyspace name");
- if (operationMode == Mode.JOINING)
- throw new IOException("Cannot snapshot until bootstrap completes");
-
- if (tableName == null)
- throw new IOException("You must supply a table name");
- if (tableName.contains("."))
- throw new IllegalArgumentException("Cannot take a snapshot of a secondary index by itself. Run snapshot on the table that owns the index.");
-
- if (tag == null || tag.equals(""))
- throw new IOException("You must supply a snapshot name.");
-
- Keyspace keyspace = getValidKeyspace(keyspaceName);
- ColumnFamilyStore columnFamilyStore = keyspace.getColumnFamilyStore(tableName);
- if (columnFamilyStore.snapshotExists(tag))
- throw new IOException("Snapshot " + tag + " already exists.");
-
- columnFamilyStore.snapshot(tag);
+ keyspace.snapshot(tag, null, skipFlush);
}
/**
@@ -2846,11 +2883,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
*
* @param tag
* the tag given to the snapshot; may not be null or empty
+ * @param skipFlush
+ * Skip blocking flush of memtable
* @param tableList
* list of tables from different keyspace in the form of ks1.cf1 ks2.cf2
*/
- @Override
- public void takeMultipleTableSnapshot(String tag, String... tableList)
+ private void takeMultipleTableSnapshot(String tag, boolean skipFlush, String... tableList)
throws IOException
{
Map<Keyspace, List<String>> keyspaceColumnfamily = new HashMap<Keyspace, List<String>>();
@@ -2899,7 +2937,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
for (Entry<Keyspace, List<String>> entry : keyspaceColumnfamily.entrySet())
{
for (String table : entry.getValue())
- entry.getKey().snapshot(tag, table);
+ entry.getKey().snapshot(tag, table, skipFlush);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/service/StorageServiceMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
index 77bcdb7..dc12253 100644
--- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
@@ -194,31 +194,34 @@ public interface StorageServiceMBean extends NotificationEmitter
public List<InetAddress> getNaturalEndpoints(String keyspaceName, ByteBuffer key);
/**
- * Takes the snapshot for the given keyspaces. A snapshot name must be specified.
- *
- * @param tag the tag given to the snapshot; may not be null or empty
- * @param keyspaceNames the name of the keyspaces to snapshot; empty means "all."
+ * @deprecated use {@link #takeSnapshot(String tag, Map options, boolean keyspaces, String... entities)} instead.
*/
+ @Deprecated
public void takeSnapshot(String tag, String... keyspaceNames) throws IOException;
/**
- * Takes the snapshot of a specific column family. A snapshot name must be specified.
- *
- * @param keyspaceName the keyspace which holds the specified column family
- * @param tableName the table to snapshot
- * @param tag the tag given to the snapshot; may not be null or empty
+ * @deprecated use {@link #takeSnapshot(String tag, Map options, boolean keyspaces, String... entities)} instead.
*/
+ @Deprecated
public void takeTableSnapshot(String keyspaceName, String tableName, String tag) throws IOException;
/**
+ * @deprecated use {@link #takeSnapshot(String tag, Map options, boolean keyspaces, String... entities)} instead.
+ */
+ @Deprecated
+ public void takeMultipleTableSnapshot(String tag, String... tableList) throws IOException;
+
+ /**
* Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified.
*
* @param tag
* the tag given to the snapshot; may not be null or empty
- * @param tableList
- * list of tables from different keyspace in the form of ks1.cf1 ks2.cf2
+ * @param options
+ * Map of options (skipFlush is the only supported option for now)
+ * @param entities
+ * list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,...
*/
- public void takeMultipleTableSnapshot(String tag, String... tableList) throws IOException;
+ public void takeSnapshot(String tag, Map<String, String> options, String... entities) throws IOException;
/**
* Remove the snapshot with the given name from the given keyspaces.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/tools/NodeProbe.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
index 2bc516a..405f70c 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -519,9 +519,10 @@ public class NodeProbe implements AutoCloseable
*
* @param snapshotName the name of the snapshot.
* @param table the table to snapshot or all on null
+ * @param options Options (skipFlush for now)
* @param keyspaces the keyspaces to snapshot
*/
- public void takeSnapshot(String snapshotName, String table, String... keyspaces) throws IOException
+ public void takeSnapshot(String snapshotName, String table, Map<String, String> options, String... keyspaces) throws IOException
{
if (table != null)
{
@@ -529,10 +530,11 @@ public class NodeProbe implements AutoCloseable
{
throw new IOException("When specifying the table for a snapshot, you must specify one and only one keyspace");
}
- ssProxy.takeTableSnapshot(keyspaces[0], table, snapshotName);
+
+ ssProxy.takeSnapshot(snapshotName, options, keyspaces[0] + "." + table);
}
else
- ssProxy.takeSnapshot(snapshotName, keyspaces);
+ ssProxy.takeSnapshot(snapshotName, options, keyspaces);
}
/**
@@ -540,21 +542,22 @@ public class NodeProbe implements AutoCloseable
*
* @param snapshotName
* the name of the snapshot.
+ * @param options
+ * Options (skipFlush for now)
* @param tableList
* list of columnfamily from different keyspace in the form of ks1.cf1 ks2.cf2
*/
- public void takeMultipleTableSnapshot(String snapshotName, String... tableList)
+ public void takeMultipleTableSnapshot(String snapshotName, Map<String, String> options, String... tableList)
throws IOException
{
if (null != tableList && tableList.length != 0)
{
- ssProxy.takeMultipleTableSnapshot(snapshotName, tableList);
+ ssProxy.takeSnapshot(snapshotName, options, tableList);
}
else
{
throw new IOException("The column family List for a snapshot should not be empty or null");
}
-
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java b/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java
index 4f549e5..8941ec1 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java
@@ -25,7 +25,9 @@ import io.airlift.command.Option;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@@ -45,6 +47,9 @@ public class Snapshot extends NodeToolCmd
@Option(title = "ktlist", name = { "-kt", "--kt-list", "-kc", "--kc.list" }, description = "The list of Keyspace.table to take snapshot.(you must not specify only keyspace)")
private String ktList = null;
+ @Option(title = "skip-flush", name = {"-sf", "--skip-flush"}, description = "Do not flush memtables before snapshotting (snapshot will not contain unflushed data)")
+ private boolean skipFlush = false;
+
@Override
public void execute(NodeProbe probe)
{
@@ -54,6 +59,9 @@ public class Snapshot extends NodeToolCmd
sb.append("Requested creating snapshot(s) for ");
+ Map<String, String> options = new HashMap<String,String>();
+ options.put("skipFlush", Boolean.toString(skipFlush));
+
// Create a separate path for kclist to avoid breaking of already existing scripts
if (null != ktList && !ktList.isEmpty())
{
@@ -67,8 +75,9 @@ public class Snapshot extends NodeToolCmd
}
if (!snapshotName.isEmpty())
sb.append(" with snapshot name [").append(snapshotName).append("]");
+ sb.append(" and options ").append(options.toString());
System.out.println(sb.toString());
- probe.takeMultipleTableSnapshot(snapshotName, ktList.split(","));
+ probe.takeMultipleTableSnapshot(snapshotName, options, ktList.split(","));
System.out.println("Snapshot directory: " + snapshotName);
}
else
@@ -80,10 +89,10 @@ public class Snapshot extends NodeToolCmd
if (!snapshotName.isEmpty())
sb.append(" with snapshot name [").append(snapshotName).append("]");
-
+ sb.append(" and options ").append(options.toString());
System.out.println(sb.toString());
- probe.takeSnapshot(snapshotName, table, toArray(keyspaces, String.class));
+ probe.takeSnapshot(snapshotName, table, options, toArray(keyspaces, String.class));
System.out.println("Snapshot directory: " + snapshotName);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index 6840e2b..af43152 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -321,8 +321,8 @@ public class ColumnFamilyStoreTest
}
ScrubTest.fillIndexCF(cfs, false, colValues);
- cfs.snapshot("nonEphemeralSnapshot", null, false);
- cfs.snapshot("ephemeralSnapshot", null, true);
+ cfs.snapshot("nonEphemeralSnapshot", null, false, false);
+ cfs.snapshot("ephemeralSnapshot", null, true, false);
Map<String, Pair<Long, Long>> snapshotDetails = cfs.getSnapshotDetails();
assertEquals(2, snapshotDetails.size());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee9e38b7/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
index 392b6f4..e438a6b 100644
--- a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
+++ b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
@@ -94,10 +94,10 @@ public class StorageServiceServerTest
}
@Test
- public void testSnapshot() throws IOException
+ public void testSnapshotWithFlush() throws IOException
{
// no need to insert extra data, even an "empty" database will have a little information in the system keyspace
- StorageService.instance.takeSnapshot("snapshot");
+ StorageService.instance.takeSnapshot(UUID.randomUUID().toString());
}
private void checkTempFilePresence(File f, boolean exist)
@@ -173,7 +173,14 @@ public class StorageServiceServerTest
public void testTableSnapshot() throws IOException
{
// no need to insert extra data, even an "empty" database will have a little information in the system keyspace
- StorageService.instance.takeTableSnapshot(SchemaKeyspace.NAME, SchemaKeyspace.KEYSPACES, "cf_snapshot");
+ StorageService.instance.takeTableSnapshot(SchemaKeyspace.NAME, SchemaKeyspace.KEYSPACES, UUID.randomUUID().toString());
+ }
+
+ @Test
+ public void testSnapshot() throws IOException
+ {
+ // no need to insert extra data, even an "empty" database will have a little information in the system keyspace
+ StorageService.instance.takeSnapshot(UUID.randomUUID().toString(), SchemaKeyspace.NAME);
}
@Test