You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2015/08/11 10:25:51 UTC
[1/2] cassandra git commit: Add tool to find why expired sstables are
not getting dropped
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 76e8eb853 -> 012b987e2
Add tool to find why expired sstables are not getting dropped
Patch by marcuse; reviewed by stefania for CASSANDRA-10015
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/028e7cb5
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/028e7cb5
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/028e7cb5
Branch: refs/heads/cassandra-2.1
Commit: 028e7cb5afb633cfb5197b7d29224b67b083b670
Parents: de84a5c
Author: Marcus Eriksson <ma...@apache.org>
Authored: Fri Aug 7 16:09:18 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 11 09:31:30 2015 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/tools/SSTableExpiredBlockers.java | 135 +++++++++++++++++++
.../cassandra/db/compaction/TTLExpiryTest.java | 29 ++++
tools/bin/sstableexpiredblockers | 49 +++++++
4 files changed, 214 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/028e7cb5/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fe060af..7d84538 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.17
+ * Add tool to find why expired sstables are not getting dropped (CASSANDRA-10015)
* Remove erroneous pending HH tasks from tpstats/jmx (CASSANDRA-9129)
* Don't cast expected bf size to an int (CASSANDRA-9959)
* Log when messages are dropped due to cross_node_timeout (CASSANDRA-9793)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/028e7cb5/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
new file mode 100644
index 0000000..b5fa779
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableReader;
+
+/**
+ * During compaction we can drop entire sstables if they only contain expired tombstones and if it is guaranteed
+ * to not cover anything in other sstables. An expired sstable can be blocked from getting dropped if its newest
+ * timestamp is newer than the oldest data in another sstable.
+ *
+ * This class outputs all sstables that are blocking other sstables from getting dropped so that a user can
+ * figure out why certain sstables are still on disk.
+ */
+public class SSTableExpiredBlockers
+{
+ public static void main(String[] args) throws IOException
+ {
+ PrintStream out = System.out;
+ if (args.length < 2)
+ {
+ out.println("Usage: sstableexpiredblockers <keyspace> <table>");
+ System.exit(1);
+ }
+ String keyspace = args[args.length - 2];
+ String columnfamily = args[args.length - 1];
+ DatabaseDescriptor.loadSchemas();
+
+ CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnfamily);
+ if (metadata == null)
+ throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
+ keyspace,
+ columnfamily));
+
+ Keyspace.openWithoutSSTables(keyspace);
+ Directories directories = Directories.create(keyspace, columnfamily);
+ Set<SSTableReader> sstables = new HashSet<>();
+
+ for (Map.Entry<Descriptor, Set<Component>> sstable : directories.sstableLister().skipTemporary(true).list().entrySet())
+ {
+ if (sstable.getKey() != null)
+ {
+ try
+ {
+ SSTableReader reader = SSTableReader.open(sstable.getKey());
+ sstables.add(reader);
+ }
+ catch (Throwable t)
+ {
+ out.println("Couldn't open sstable: " + sstable.getKey().filenameFor(Component.DATA));
+ Throwables.propagate(t);
+ }
+ }
+ }
+ if (sstables.isEmpty())
+ {
+ out.println("No sstables for " + keyspace + "." + columnfamily);
+ System.exit(1);
+ }
+
+ int gcBefore = (int)(System.currentTimeMillis()/1000) - metadata.getGcGraceSeconds();
+ Multimap<SSTableReader, SSTableReader> blockers = checkForExpiredSSTableBlockers(sstables, gcBefore);
+ for (SSTableReader blocker : blockers.keySet())
+ {
+ out.println(String.format("%s blocks %d expired sstables from getting dropped: %s%n",
+ formatForExpiryTracing(Collections.singleton(blocker)),
+ blockers.get(blocker).size(),
+ formatForExpiryTracing(blockers.get(blocker))));
+ }
+
+ System.exit(0);
+ }
+
+ public static Multimap<SSTableReader, SSTableReader> checkForExpiredSSTableBlockers(Iterable<SSTableReader> sstables, int gcBefore)
+ {
+ Multimap<SSTableReader, SSTableReader> blockers = ArrayListMultimap.create();
+ for (SSTableReader sstable : sstables)
+ {
+ if (sstable.getSSTableMetadata().maxLocalDeletionTime < gcBefore)
+ {
+ for (SSTableReader potentialBlocker : sstables)
+ {
+ if (!potentialBlocker.equals(sstable) &&
+ potentialBlocker.getMinTimestamp() <= sstable.getMaxTimestamp() &&
+ potentialBlocker.getSSTableMetadata().maxLocalDeletionTime > gcBefore)
+ blockers.put(potentialBlocker, sstable);
+ }
+ }
+ }
+ return blockers;
+ }
+
+ private static String formatForExpiryTracing(Iterable<SSTableReader> sstables)
+ {
+ StringBuilder sb = new StringBuilder();
+
+ for (SSTableReader sstable : sstables)
+ sb.append(String.format("[%s (minTS = %d, maxTS = %d, maxLDT = %d)]", sstable, sstable.getMinTimestamp(), sstable.getMaxTimestamp(), sstable.getSSTableMetadata().maxLocalDeletionTime)).append(", ");
+
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/028e7cb5/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
index 3fad0ec..5a83c76 100644
--- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
@@ -26,6 +26,7 @@ import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ExecutionException;
+import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -41,6 +42,7 @@ import org.apache.cassandra.db.RowMutation;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.sstable.SSTableScanner;
+import org.apache.cassandra.tools.SSTableExpiredBlockers;
import org.apache.cassandra.utils.ByteBufferUtil;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -197,4 +199,31 @@ public class TTLExpiryTest extends SchemaLoader
cfs.clearUnsafe();
}
+ @Test
+ public void testCheckForExpiredSSTableBlockers() throws InterruptedException
+ {
+ String KEYSPACE1 = "Keyspace1";
+ ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+ cfs.metadata.gcGraceSeconds(0);
+
+ RowMutation rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
+ rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis());
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+ SSTableReader blockingSSTable = cfs.getSSTables().iterator().next();
+ for (int i = 0; i < 10; i++)
+ {
+ rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
+ rm.delete("Standard1", System.currentTimeMillis());
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+ }
+ Multimap<SSTableReader, SSTableReader> blockers = SSTableExpiredBlockers.checkForExpiredSSTableBlockers(cfs.getSSTables(), (int) (System.currentTimeMillis() / 1000) + 100);
+ assertEquals(1, blockers.keySet().size());
+ assertTrue(blockers.keySet().contains(blockingSSTable));
+ assertEquals(10, blockers.get(blockingSSTable).size());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/028e7cb5/tools/bin/sstableexpiredblockers
----------------------------------------------------------------------
diff --git a/tools/bin/sstableexpiredblockers b/tools/bin/sstableexpiredblockers
new file mode 100755
index 0000000..58cefce
--- /dev/null
+++ b/tools/bin/sstableexpiredblockers
@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+ for include in "`dirname $0`/cassandra.in.sh" \
+ "$HOME/.cassandra.in.sh" \
+ /usr/share/cassandra/cassandra.in.sh \
+ /usr/local/share/cassandra/cassandra.in.sh \
+ /opt/cassandra/cassandra.in.sh; do
+ if [ -r $include ]; then
+ . $include
+ break
+ fi
+ done
+elif [ -r $CASSANDRA_INCLUDE ]; then
+ . $CASSANDRA_INCLUDE
+fi
+
+
+# Use JAVA_HOME if set, otherwise look for java in PATH
+if [ -x $JAVA_HOME/bin/java ]; then
+ JAVA=$JAVA_HOME/bin/java
+else
+ JAVA=`which java`
+fi
+
+if [ -z "$CLASSPATH" ]; then
+ echo "You must set the CLASSPATH var" >&2
+ exit 1
+fi
+
+"$JAVA" -cp "$CLASSPATH" -Dstorage-config=$CASSANDRA_CONF \
+ -Dlog4j.configuration=log4j-tools.properties \
+ org.apache.cassandra.tools.SSTableExpiredBlockers "$@"
[2/2] cassandra git commit: Merge branch 'cassandra-2.0' into
cassandra-2.1
Posted by ma...@apache.org.
Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
CHANGES.txt
test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/012b987e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/012b987e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/012b987e
Branch: refs/heads/cassandra-2.1
Commit: 012b987e267bdfb39b44c8afd4f4ccbb48a1ddf4
Parents: 76e8eb8 028e7cb
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Aug 11 09:46:33 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 11 10:11:15 2015 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/tools/SSTableExpiredBlockers.java | 136 +++++++++++++++++++
.../cassandra/db/compaction/TTLExpiryTest.java | 29 ++++
tools/bin/sstableexpiredblockers | 54 ++++++++
tools/bin/sstableexpiredblockers.bat | 23 ++++
5 files changed, 243 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/012b987e/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 462de44,7d84538..f7fb63c
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,26 -1,5 +1,27 @@@
-2.0.17
+2.1.9
+ * Add new JMX methods to change local compaction strategy (CASSANDRA-9965)
+ * Write hints for paxos commits (CASSANDRA-7342)
+ * (cqlsh) Fix timestamps before 1970 on Windows, always
+ use UTC for timestamp display (CASSANDRA-10000)
+ * (cqlsh) Avoid overwriting new config file with old config
+ when both exist (CASSANDRA-9777)
+ * Release snapshot selfRef when doing snapshot repair (CASSANDRA-9998)
+ * Cannot replace token does not exist - DN node removed as Fat Client (CASSANDRA-9871)
+ * Fix handling of enable/disable autocompaction (CASSANDRA-9899)
+ * Commit log segment recycling is disabled by default (CASSANDRA-9896)
+ * Add consistency level to tracing ouput (CASSANDRA-9827)
+ * Fix MarshalException when upgrading superColumn family (CASSANDRA-9582)
+ * Fix broken logging for "empty" flushes in Memtable (CASSANDRA-9837)
+ * Handle corrupt files on startup (CASSANDRA-9686)
+ * Fix clientutil jar and tests (CASSANDRA-9760)
+ * (cqlsh) Allow the SSL protocol version to be specified through the
+ config file or environment variables (CASSANDRA-9544)
+ * Remove repair snapshot leftover on startup (CASSANDRA-7357)
+ * Use random nodes for batch log when only 2 racks (CASSANDRA-8735)
+ * Ensure atomicity inside thrift and stream session (CASSANDRA-7757)
+ * Fix nodetool info error when the node is not joined (CASSANDRA-9031)
+Merged from 2.0:
+ * Add tool to find why expired sstables are not getting dropped (CASSANDRA-10015)
* Remove erroneous pending HH tasks from tpstats/jmx (CASSANDRA-9129)
* Don't cast expected bf size to an int (CASSANDRA-9959)
* Log when messages are dropped due to cross_node_timeout (CASSANDRA-9793)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/012b987e/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
index 0000000,b5fa779..b4f7063
mode 000000,100644..100644
--- a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
@@@ -1,0 -1,135 +1,136 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.cassandra.tools;
+
+ import java.io.IOException;
+ import java.io.PrintStream;
+ import java.util.Collections;
+ import java.util.HashSet;
+ import java.util.Map;
+ import java.util.Set;
+
+ import com.google.common.base.Throwables;
+ import com.google.common.collect.ArrayListMultimap;
+ import com.google.common.collect.Multimap;
+
+ import org.apache.cassandra.config.CFMetaData;
+ import org.apache.cassandra.config.DatabaseDescriptor;
+ import org.apache.cassandra.config.Schema;
++import org.apache.cassandra.db.ColumnFamilyStore;
+ import org.apache.cassandra.db.Directories;
+ import org.apache.cassandra.db.Keyspace;
+ import org.apache.cassandra.io.sstable.Component;
+ import org.apache.cassandra.io.sstable.Descriptor;
+ import org.apache.cassandra.io.sstable.SSTableReader;
+
+ /**
+ * During compaction we can drop entire sstables if they only contain expired tombstones and if it is guaranteed
+ * to not cover anything in other sstables. An expired sstable can be blocked from getting dropped if its newest
+ * timestamp is newer than the oldest data in another sstable.
+ *
+ * This class outputs all sstables that are blocking other sstables from getting dropped so that a user can
+ * figure out why certain sstables are still on disk.
+ */
+ public class SSTableExpiredBlockers
+ {
+ public static void main(String[] args) throws IOException
+ {
+ PrintStream out = System.out;
+ if (args.length < 2)
+ {
+ out.println("Usage: sstableexpiredblockers <keyspace> <table>");
+ System.exit(1);
+ }
+ String keyspace = args[args.length - 2];
+ String columnfamily = args[args.length - 1];
+ DatabaseDescriptor.loadSchemas();
+
+ CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnfamily);
+ if (metadata == null)
+ throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
+ keyspace,
+ columnfamily));
+
- Keyspace.openWithoutSSTables(keyspace);
- Directories directories = Directories.create(keyspace, columnfamily);
++ Keyspace ks = Keyspace.openWithoutSSTables(keyspace);
++ ColumnFamilyStore cfs = ks.getColumnFamilyStore(columnfamily);
++ Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
+ Set<SSTableReader> sstables = new HashSet<>();
-
- for (Map.Entry<Descriptor, Set<Component>> sstable : directories.sstableLister().skipTemporary(true).list().entrySet())
++ for (Map.Entry<Descriptor, Set<Component>> sstable : lister.list().entrySet())
+ {
+ if (sstable.getKey() != null)
+ {
+ try
+ {
+ SSTableReader reader = SSTableReader.open(sstable.getKey());
+ sstables.add(reader);
+ }
+ catch (Throwable t)
+ {
+ out.println("Couldn't open sstable: " + sstable.getKey().filenameFor(Component.DATA));
+ Throwables.propagate(t);
+ }
+ }
+ }
+ if (sstables.isEmpty())
+ {
+ out.println("No sstables for " + keyspace + "." + columnfamily);
+ System.exit(1);
+ }
+
+ int gcBefore = (int)(System.currentTimeMillis()/1000) - metadata.getGcGraceSeconds();
+ Multimap<SSTableReader, SSTableReader> blockers = checkForExpiredSSTableBlockers(sstables, gcBefore);
+ for (SSTableReader blocker : blockers.keySet())
+ {
+ out.println(String.format("%s blocks %d expired sstables from getting dropped: %s%n",
+ formatForExpiryTracing(Collections.singleton(blocker)),
+ blockers.get(blocker).size(),
+ formatForExpiryTracing(blockers.get(blocker))));
+ }
+
+ System.exit(0);
+ }
+
+ public static Multimap<SSTableReader, SSTableReader> checkForExpiredSSTableBlockers(Iterable<SSTableReader> sstables, int gcBefore)
+ {
+ Multimap<SSTableReader, SSTableReader> blockers = ArrayListMultimap.create();
+ for (SSTableReader sstable : sstables)
+ {
+ if (sstable.getSSTableMetadata().maxLocalDeletionTime < gcBefore)
+ {
+ for (SSTableReader potentialBlocker : sstables)
+ {
+ if (!potentialBlocker.equals(sstable) &&
+ potentialBlocker.getMinTimestamp() <= sstable.getMaxTimestamp() &&
+ potentialBlocker.getSSTableMetadata().maxLocalDeletionTime > gcBefore)
+ blockers.put(potentialBlocker, sstable);
+ }
+ }
+ }
+ return blockers;
+ }
+
+ private static String formatForExpiryTracing(Iterable<SSTableReader> sstables)
+ {
+ StringBuilder sb = new StringBuilder();
+
+ for (SSTableReader sstable : sstables)
+ sb.append(String.format("[%s (minTS = %d, maxTS = %d, maxLDT = %d)]", sstable, sstable.getMinTimestamp(), sstable.getMaxTimestamp(), sstable.getSSTableMetadata().maxLocalDeletionTime)).append(", ");
+
+ return sb.toString();
+ }
+ }
http://git-wip-us.apache.org/repos/asf/cassandra/blob/012b987e/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
index 678601b,5a83c76..a5b376e
--- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
@@@ -20,6 -20,13 +20,7 @@@ package org.apache.cassandra.db.compact
*
*/
-
-
-import java.util.Collections;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-
+ import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import org.junit.Test;
import org.junit.runner.RunWith;
@@@ -27,15 -34,16 +28,16 @@@
import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DataRange;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.RowMutation;
+import org.apache.cassandra.db.*;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableScanner;
+ import org.apache.cassandra.tools.SSTableExpiredBlockers;
import org.apache.cassandra.utils.ByteBufferUtil;
+
+import java.util.Collections;
+import java.util.Set;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@@ -189,4 -152,78 +191,31 @@@ public class TTLExpiryTest extends Sche
assertEquals(noTTLKey, iter.getKey());
}
}
+
+ @Test
- public void testAggressiveFullyExpired()
- {
- String KEYSPACE1 = "Keyspace1";
- ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
- cfs.disableAutoCompaction();
- cfs.metadata.gcGraceSeconds(0);
-
- DecoratedKey ttlKey = Util.dk("ttl");
- RowMutation rm = new RowMutation("Keyspace1", ttlKey.key);
- rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 1, 1);
- rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 3, 1);
- rm.applyUnsafe();
- cfs.forceBlockingFlush();
-
- rm = new RowMutation(KEYSPACE1, ttlKey.key);
- rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2, 1);
- rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 5, 1);
- rm.applyUnsafe();
- cfs.forceBlockingFlush();
-
- rm = new RowMutation(KEYSPACE1, ttlKey.key);
- rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 4, 1);
- rm.add("Standard1", ByteBufferUtil.bytes("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 7, 1);
- rm.applyUnsafe();
- cfs.forceBlockingFlush();
-
- rm = new RowMutation(KEYSPACE1, ttlKey.key);
- rm.add("Standard1", ByteBufferUtil.bytes("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 6, 3);
- rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8, 1);
- rm.applyUnsafe();
- cfs.forceBlockingFlush();
-
- Set<SSTableReader> sstables = Sets.newHashSet(cfs.getSSTables());
- int now = (int)(System.currentTimeMillis() / 1000);
- int gcBefore = now + 2;
- Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables(
- cfs,
- sstables,
- Collections.EMPTY_SET,
- gcBefore);
- assertEquals(2, expired.size());
-
- cfs.clearUnsafe();
- }
-
- @Test
+ public void testCheckForExpiredSSTableBlockers() throws InterruptedException
+ {
+ String KEYSPACE1 = "Keyspace1";
+ ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
+ cfs.truncateBlocking();
+ cfs.disableAutoCompaction();
+ cfs.metadata.gcGraceSeconds(0);
+
- RowMutation rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
- rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis());
++ Mutation rm = new Mutation(KEYSPACE1, Util.dk("test").getKey());
++ rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis());
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+ SSTableReader blockingSSTable = cfs.getSSTables().iterator().next();
+ for (int i = 0; i < 10; i++)
+ {
- rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
++ rm = new Mutation(KEYSPACE1, Util.dk("test").getKey());
+ rm.delete("Standard1", System.currentTimeMillis());
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+ }
+ Multimap<SSTableReader, SSTableReader> blockers = SSTableExpiredBlockers.checkForExpiredSSTableBlockers(cfs.getSSTables(), (int) (System.currentTimeMillis() / 1000) + 100);
+ assertEquals(1, blockers.keySet().size());
+ assertTrue(blockers.keySet().contains(blockingSSTable));
+ assertEquals(10, blockers.get(blockingSSTable).size());
+ }
-
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/012b987e/tools/bin/sstableexpiredblockers
----------------------------------------------------------------------
diff --cc tools/bin/sstableexpiredblockers
index 0000000,58cefce..0027208
mode 000000,100755..100755
--- a/tools/bin/sstableexpiredblockers
+++ b/tools/bin/sstableexpiredblockers
@@@ -1,0 -1,49 +1,54 @@@
+ #!/bin/sh
+
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+ if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
- for include in "`dirname $0`/cassandra.in.sh" \
- "$HOME/.cassandra.in.sh" \
- /usr/share/cassandra/cassandra.in.sh \
++ for include in /usr/share/cassandra/cassandra.in.sh \
+ /usr/local/share/cassandra/cassandra.in.sh \
- /opt/cassandra/cassandra.in.sh; do
- if [ -r $include ]; then
- . $include
++ /opt/cassandra/cassandra.in.sh \
++ ~/.cassandra.in.sh \
++ "`dirname "$0"`/cassandra.in.sh"; do
++ if [ -r "$include" ]; then
++ . "$include"
+ break
+ fi
+ done
-elif [ -r $CASSANDRA_INCLUDE ]; then
- . $CASSANDRA_INCLUDE
++elif [ -r "$CASSANDRA_INCLUDE" ]; then
++ . "$CASSANDRA_INCLUDE"
+ fi
+
-
+ # Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x $JAVA_HOME/bin/java ]; then
- JAVA=$JAVA_HOME/bin/java
++if [ -x "$JAVA_HOME/bin/java" ]; then
++ JAVA="$JAVA_HOME/bin/java"
+ else
- JAVA=`which java`
++ JAVA="`which java`"
+ fi
+
+ if [ -z "$CLASSPATH" ]; then
+ echo "You must set the CLASSPATH var" >&2
+ exit 1
+ fi
+
-"$JAVA" -cp "$CLASSPATH" -Dstorage-config=$CASSANDRA_CONF \
- -Dlog4j.configuration=log4j-tools.properties \
++if [ "x$MAX_HEAP_SIZE" = "x" ]; then
++ MAX_HEAP_SIZE="256M"
++fi
++
++"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" -Xmx$MAX_HEAP_SIZE \
++ -Dcassandra.storagedir="$cassandra_storagedir" \
++ -Dlogback.configurationFile=logback-tools.xml \
+ org.apache.cassandra.tools.SSTableExpiredBlockers "$@"
++
http://git-wip-us.apache.org/repos/asf/cassandra/blob/012b987e/tools/bin/sstableexpiredblockers.bat
----------------------------------------------------------------------
diff --cc tools/bin/sstableexpiredblockers.bat
index 0000000,0000000..7af1105
new file mode 100644
--- /dev/null
+++ b/tools/bin/sstableexpiredblockers.bat
@@@ -1,0 -1,0 +1,23 @@@
++@REM Licensed to the Apache Software Foundation (ASF) under one or more
++@REM contributor license agreements. See the NOTICE file distributed with
++@REM this work for additional information regarding copyright ownership.
++@REM The ASF licenses this file to You under the Apache License, Version 2.0
++@REM (the "License"); you may not use this file except in compliance with
++@REM the License. You may obtain a copy of the License at
++@REM
++@REM http://www.apache.org/licenses/LICENSE-2.0
++@REM
++@REM Unless required by applicable law or agreed to in writing, software
++@REM distributed under the License is distributed on an "AS IS" BASIS,
++@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++@REM See the License for the specific language governing permissions and
++@REM limitations under the License.
++
++@echo off
++
++if "%OS%" == "Windows_NT" setlocal
++
++pushd "%~dp0"
++call cassandra.in.bat
++
++"%JAVA_HOME%\bin\java" -cp %CLASSPATH% org.apache.cassandra.tools.SSTableExpiredBlockers %*