You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by al...@apache.org on 2015/05/12 20:06:39 UTC
[1/3] cassandra git commit: Fix counting of tombstones for
TombstoneOverwhelmingException
Repository: cassandra
Updated Branches:
refs/heads/trunk 5ff69f2c9 -> 21a915cd7
Fix counting of tombstones for TombstoneOverwhelmingException
patch by Aleksey Yeschenko; reviewed by Tyler Hobbs for CASSANDRA-9299
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/bed42c21
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/bed42c21
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/bed42c21
Branch: refs/heads/trunk
Commit: bed42c2104ebaac83da4292703c08a5c963e062c
Parents: a7cae32
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Tue May 12 20:52:13 2015 +0300
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Tue May 12 20:52:13 2015 +0300
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 2 +-
.../cassandra/db/filter/ColumnCounter.java | 33 ++--
.../cassandra/db/filter/SliceQueryFilter.java | 19 ++-
.../SliceQueryFilterWithTombstonesTest.java | 150 +++++++++++++++++++
5 files changed, 182 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bed42c21/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 685b945..d7d01cf 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.15:
+ * Fix counting of tombstones for TombstoneOverwhelmingException (CASSANDRA-9299)
* Fix ReconnectableSnitch reconnecting to peers during upgrade (CASSANDRA-6702)
* Include keyspace and table name in error log for collections over the size
limit (CASSANDRA-9286)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bed42c21/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index d8640e8..5ea1287 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1396,7 +1396,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
if (filter.filter instanceof SliceQueryFilter)
{
// Log the number of tombstones scanned on single key queries
- metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastIgnored());
+ metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastTombstones());
metric.liveScannedHistogram.update(((SliceQueryFilter) filter.filter).lastLive());
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bed42c21/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/ColumnCounter.java b/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
index 814d8ed..2d0df1f 100644
--- a/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
+++ b/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
@@ -31,7 +31,7 @@ import org.apache.cassandra.utils.ByteBufferUtil;
public class ColumnCounter
{
protected int live;
- protected int ignored;
+ protected int tombstones;
protected final long timestamp;
public ColumnCounter(long timestamp)
@@ -41,15 +41,15 @@ public class ColumnCounter
public void count(Column column, DeletionInfo.InOrderTester tester)
{
- if (!isLive(column, tester, timestamp))
- ignored++;
- else
- live++;
- }
+ // The cell is shadowed by a higher-level deletion, and won't be retained.
+ // For the purposes of this counter, we don't care if it's a tombstone or not.
+ if (tester.isDeleted(column))
+ return;
- protected static boolean isLive(Column column, DeletionInfo.InOrderTester tester, long timestamp)
- {
- return column.isLive(timestamp) && (!tester.isDeleted(column));
+ if (column.isLive(timestamp))
+ live++;
+ else
+ tombstones++;
}
public int live()
@@ -57,9 +57,9 @@ public class ColumnCounter
return live;
}
- public int ignored()
+ public int tombstones()
{
- return ignored;
+ return tombstones;
}
public ColumnCounter countAll(ColumnFamily container)
@@ -101,9 +101,12 @@ public class ColumnCounter
public void count(Column column, DeletionInfo.InOrderTester tester)
{
- if (!isLive(column, tester, timestamp))
+ if (tester.isDeleted(column))
+ return;
+
+ if (!column.isLive(timestamp))
{
- ignored++;
+ tombstones++;
return;
}
@@ -119,11 +122,11 @@ public class ColumnCounter
if (previous == null)
{
// Only the first group can be static
- previousGroupIsStatic = type.isStaticName(column.name());
+ previousGroupIsStatic = CompositeType.isStaticName(column.name());
}
else
{
- boolean isSameGroup = previousGroupIsStatic == type.isStaticName(column.name());
+ boolean isSameGroup = previousGroupIsStatic == CompositeType.isStaticName(column.name());
if (isSameGroup)
{
for (int i = 0; i < toGroup; i++)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bed42c21/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index ad1a92b..6e6ab6b 100644
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@ -200,7 +200,7 @@ public class SliceQueryFilter implements IDiskAtomFilter
if (columnCounter.live() > count)
break;
- if (respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneFailureThreshold())
+ if (respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneFailureThreshold())
{
Tracing.trace("Scanned over {} tombstones; query aborted (see tombstone_failure_threshold)", DatabaseDescriptor.getTombstoneFailureThreshold());
logger.error("Scanned over {} tombstones in {}.{}; query aborted (see tombstone_failure_threshold)",
@@ -211,8 +211,8 @@ public class SliceQueryFilter implements IDiskAtomFilter
container.addIfRelevant(column, tester, gcBefore);
}
- Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
- if (respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneWarnThreshold())
+ Tracing.trace("Read {} live and {} tombstone cells", columnCounter.live(), columnCounter.tombstones());
+ if (respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneWarnThreshold())
{
StringBuilder sb = new StringBuilder();
AbstractType<?> type = container.metadata().comparator;
@@ -228,8 +228,13 @@ public class SliceQueryFilter implements IDiskAtomFilter
sb.append(']');
}
- logger.warn("Read {} live and {} tombstoned cells in {}.{} (see tombstone_warn_threshold). {} columns was requested, slices={}",
- columnCounter.live(), columnCounter.ignored(), container.metadata().ksName, container.metadata().cfName, count, sb);
+ logger.warn("Read {} live and {} tombstone cells in {}.{} (see tombstone_warn_threshold). {} columns was requested, slices={}",
+ columnCounter.live(),
+ columnCounter.tombstones(),
+ container.metadata().ksName,
+ container.metadata().cfName,
+ count,
+ sb);
}
}
@@ -305,9 +310,9 @@ public class SliceQueryFilter implements IDiskAtomFilter
return columnCounter == null ? 0 : Math.min(columnCounter.live(), count);
}
- public int lastIgnored()
+ public int lastTombstones()
{
- return columnCounter == null ? 0 : columnCounter.ignored();
+ return columnCounter == null ? 0 : columnCounter.tombstones();
}
public int lastLive()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bed42c21/test/unit/org/apache/cassandra/cql3/SliceQueryFilterWithTombstonesTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/SliceQueryFilterWithTombstonesTest.java b/test/unit/org/apache/cassandra/cql3/SliceQueryFilterWithTombstonesTest.java
new file mode 100644
index 0000000..4440782
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/SliceQueryFilterWithTombstonesTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.filter.TombstoneOverwhelmingException;
+import org.apache.cassandra.gms.Gossiper;
+
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
+
+import static org.apache.cassandra.cql3.QueryProcessor.process;
+import static org.apache.cassandra.cql3.QueryProcessor.processInternal;
+
+/**
+ * Test that TombstoneOverwhelmingException gets thrown when it should be and doesn't when it shouldn't be.
+ */
+public class SliceQueryFilterWithTombstonesTest
+{
+ static final String KEYSPACE = "tombstone_overwhelming_exception_test";
+ static final String TABLE = "overwhelmed";
+
+ static final int ORIGINAL_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold();
+ static final int THRESHOLD = 100;
+
+ @BeforeClass
+ public static void setUp() throws Throwable
+ {
+ SchemaLoader.loadSchema();
+
+ process(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}",
+ KEYSPACE),
+ ConsistencyLevel.ONE);
+
+ process(String.format("CREATE TABLE IF NOT EXISTS %s.%s (a text, b text, c text, PRIMARY KEY (a, b));",
+ KEYSPACE,
+ TABLE),
+ ConsistencyLevel.ONE);
+
+ DatabaseDescriptor.setTombstoneFailureThreshold(THRESHOLD);
+ }
+
+ @AfterClass
+ public static void tearDown()
+ {
+ Gossiper.instance.stop();
+
+ DatabaseDescriptor.setTombstoneFailureThreshold(ORIGINAL_THRESHOLD);
+ }
+
+ private static UntypedResultSet execute(String query)
+ {
+ return processInternal(String.format(query, KEYSPACE, TABLE));
+ }
+
+ @Test
+ public void testBelowThresholdSelect()
+ {
+ // insert exactly the amount of tombstones that shouldn't trigger an exception
+ for (int i = 0; i < THRESHOLD; i++)
+ execute("INSERT INTO %s.%s (a, b, c) VALUES ('key1', 'column" + i + "', null);");
+
+ try
+ {
+ execute("SELECT * FROM %s.%s WHERE a = 'key1';");
+ }
+ catch (Throwable e)
+ {
+ fail("SELECT with tombstones below the threshold should not have failed, but has: " + e);
+ }
+ }
+
+ @Test
+ public void testBeyondThresholdSelect()
+ {
+ // insert exactly the amount of tombstones that *SHOULD* trigger an exception
+ for (int i = 0; i < THRESHOLD + 1; i++)
+ execute("INSERT INTO %s.%s (a, b, c) VALUES ('key2', 'column" + i + "', null);");
+
+ try
+ {
+ execute("SELECT * FROM %s.%s WHERE a = 'key2';");
+ fail("SELECT with tombstones beyond the threshold should have failed, but hasn't");
+ }
+ catch (Throwable e)
+ {
+ assertTrue(e instanceof TombstoneOverwhelmingException);
+ }
+ }
+
+ @Test
+ public void testAllShadowedSelect()
+ {
+ // insert exactly the amount of tombstones that *SHOULD* normally trigger an exception
+ for (int i = 0; i < THRESHOLD + 1; i++)
+ execute("INSERT INTO %s.%s (a, b, c) VALUES ('key3', 'column" + i + "', null);");
+
+ // delete all with a partition level tombstone
+ execute("DELETE FROM %s.%s WHERE a = 'key3'");
+
+ try
+ {
+ execute("SELECT * FROM %s.%s WHERE a = 'key3';");
+ }
+ catch (Throwable e)
+ {
+ fail("SELECT with tombstones shadowed by a partition tombstone should not have failed, but has: " + e);
+ }
+ }
+
+ @Test
+ public void testLiveShadowedCellsSelect()
+ {
+ for (int i = 0; i < THRESHOLD + 1; i++)
+ execute("INSERT INTO %s.%s (a, b, c) VALUES ('key4', 'column" + i + "', 'column');");
+
+ // delete all with a partition level tombstone
+ execute("DELETE FROM %s.%s WHERE a = 'key4'");
+
+ try
+ {
+ execute("SELECT * FROM %s.%s WHERE a = 'key4';");
+ }
+ catch (Throwable e)
+ {
+ fail("SELECT with regular cells shadowed by a partition tombstone should not have failed, but has: " + e);
+ }
+ }
+}
[3/3] cassandra git commit: Merge branch 'cassandra-2.1' into trunk
Posted by al...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/21a915cd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/21a915cd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/21a915cd
Branch: refs/heads/trunk
Commit: 21a915cd7148d03e9bbe32d68395d7b04dcfc55a
Parents: 5ff69f2 9beeba3
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Tue May 12 21:06:54 2015 +0300
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Tue May 12 21:06:54 2015 +0300
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/ArrayBackedSortedColumns.java | 14 +-
.../apache/cassandra/db/AtomicBTreeColumns.java | 5 +
.../org/apache/cassandra/db/ColumnFamily.java | 5 +
.../apache/cassandra/db/ColumnFamilyStore.java | 2 +-
.../cassandra/db/filter/ColumnCounter.java | 48 ++++--
.../cassandra/db/filter/SliceQueryFilter.java | 34 ++--
.../SliceQueryFilterWithTombstonesTest.java | 166 +++++++++++++++++++
8 files changed, 236 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java
index f7d7ec1,c53832b..1beb982
--- a/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java
+++ b/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java
@@@ -283,10 -282,10 +283,16 @@@ public class ArrayBackedSortedColumns e
public void maybeAppendColumn(Cell cell, DeletionInfo.InOrderTester tester, int gcBefore)
{
if (cell.getLocalDeletionTime() >= gcBefore && !tester.isDeleted(cell))
-- {
-- internalAdd(cell);
-- sortedSize++;
-- }
++ appendColumn(cell);
++ }
++
++ /**
++ * Adds a cell, assuming that it sorts *strictly after* the current-last cell in the array.
++ */
++ public void appendColumn(Cell cell)
++ {
++ internalAdd(cell);
++ sortedSize++;
}
public void addColumn(Cell cell)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/AtomicBTreeColumns.java
index 0b1a58c,47f0b85..9ef0c14
--- a/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java
+++ b/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java
@@@ -319,6 -310,6 +319,11 @@@ public class AtomicBTreeColumns extend
throw new UnsupportedOperationException();
}
++ public void appendColumn(Cell cell)
++ {
++ throw new UnsupportedOperationException();
++ }
++
public void addAll(ColumnFamily cf)
{
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/src/java/org/apache/cassandra/db/ColumnFamily.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamily.java
index 88ab9e4,c9a008f..9caf20b
--- a/src/java/org/apache/cassandra/db/ColumnFamily.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamily.java
@@@ -205,6 -203,6 +205,11 @@@ public abstract class ColumnFamily impl
public abstract void maybeAppendColumn(Cell cell, DeletionInfo.InOrderTester tester, int gcBefore);
/**
++ * Appends a cell. Requires that the cell to add is sorted strictly after the last cell in the container.
++ */
++ public abstract void appendColumn(Cell cell);
++
++ /**
* Adds all the columns of a given column map to this column map.
* This is equivalent to:
* <code>
http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/filter/ColumnCounter.java
index 86cfc40,8be26e1..43555bc
--- a/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
+++ b/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
@@@ -37,17 -37,17 +37,23 @@@ public class ColumnCounte
this.timestamp = timestamp;
}
-- public void count(Cell cell, DeletionInfo.InOrderTester tester)
++ /**
++ * @return true if the cell counted as a live cell or a valid tombstone; false if it got immediately discarded for
++ * being shadowed by a range- or a partition tombstone
++ */
++ public boolean count(Cell cell, DeletionInfo.InOrderTester tester)
{
- if (!isLive(cell, tester, timestamp))
- ignored++;
- else
+ // The cell is shadowed by a higher-level deletion, and won't be retained.
+ // For the purposes of this counter, we don't care if it's a tombstone or not.
+ if (tester.isDeleted(cell))
- return;
++ return false;
+
+ if (cell.isLive(timestamp))
live++;
- }
+ else
+ tombstones++;
+
- protected static boolean isLive(Cell cell, DeletionInfo.InOrderTester tester, long timestamp)
- {
- return cell.isLive(timestamp) && !tester.isDeleted(cell);
++ return true;
}
public int live()
@@@ -96,18 -96,21 +102,22 @@@
assert toGroup == 0 || type != null;
}
-- public void count(Cell cell, DeletionInfo.InOrderTester tester)
++ @Override
++ public boolean count(Cell cell, DeletionInfo.InOrderTester tester)
{
- if (!isLive(cell, tester, timestamp))
+ if (tester.isDeleted(cell))
- return;
++ return false;
+
+ if (!cell.isLive(timestamp))
{
- ignored++;
- return;
+ tombstones++;
- return;
++ return true;
}
if (toGroup == 0)
{
live = 1;
-- return;
++ return true;
}
CellName current = cell.name();
@@@ -129,7 -132,7 +139,7 @@@
}
if (isSameGroup)
-- return;
++ return true;
// We want to count the static group as 1 (CQL) row only if it's the only
// group in the partition. So, since we have already counted it at this point,
@@@ -137,12 -140,12 +147,14 @@@
if (previous.isStatic())
{
previous = current;
-- return;
++ return true;
}
}
live++;
previous = current;
++
++ return true;
}
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index 396fc06,1195d4c..d914f51
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@@ -202,65 -203,60 +202,68 @@@ public class SliceQueryFilter implement
while (reducedColumns.hasNext())
{
Cell cell = reducedColumns.next();
+
if (logger.isTraceEnabled())
- logger.trace(String.format("collecting %s of %s: %s",
- columnCounter.live(), count, cell.getString(container.getComparator())));
+ logger.trace("collecting {} of {}: {}", columnCounter.live(), count, cell.getString(container.getComparator()));
- columnCounter.count(cell, tester);
+ // An expired tombstone will be immediately discarded in memory, and needn't be counted.
- if (cell.getLocalDeletionTime() < gcBefore)
++ // Neither should be any cell shadowed by a range- or a partition tombstone.
++ if (cell.getLocalDeletionTime() < gcBefore || !columnCounter.count(cell, tester))
+ continue;
- columnCounter.count(cell, tester);
-
if (columnCounter.live() > count)
break;
- if (respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneFailureThreshold())
+ if (respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneFailureThreshold())
{
- Tracing.trace("Scanned over {} tombstones; query aborted (see tombstone_failure_threshold)",
- DatabaseDescriptor.getTombstoneFailureThreshold());
- logger.error("Scanned over {} tombstones in {}.{}; query aborted (see tombstone_failure_threshold)",
- DatabaseDescriptor.getTombstoneFailureThreshold(),
- container.metadata().ksName,
- container.metadata().cfName);
- throw new TombstoneOverwhelmingException();
+ Tracing.trace("Scanned over {} tombstones; query aborted (see tombstone_failure_threshold); slices={}",
+ DatabaseDescriptor.getTombstoneFailureThreshold(), getSlicesInfo(container));
+
- throw new TombstoneOverwhelmingException(columnCounter.ignored(),
++ throw new TombstoneOverwhelmingException(columnCounter.tombstones(),
+ count,
+ container.metadata().ksName,
+ container.metadata().cfName,
+ container.getComparator().getString(cell.name()),
+ getSlicesInfo(container));
}
-- container.maybeAppendColumn(cell, tester, gcBefore);
++ container.appendColumn(cell);
}
- boolean warnTombstones = logger.isWarnEnabled() && respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneWarnThreshold();
- Tracing.trace("Read {} live and {} tombstone cells", columnCounter.live(), columnCounter.tombstones());
- if (logger.isWarnEnabled() && respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneWarnThreshold())
++ boolean warnTombstones = logger.isWarnEnabled() && respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneWarnThreshold();
+ if (warnTombstones)
{
- String msg = String.format("Read %d live and %d tombstoned cells in %s.%s for key: %1.512s (see tombstone_warn_threshold). %d columns were requested, slices=%1.512s",
- StringBuilder sb = new StringBuilder();
- CellNameType type = container.metadata().comparator;
-
- for (ColumnSlice sl : slices)
- {
- assert sl != null;
-
- sb.append('[');
- sb.append(type.getString(sl.start));
- sb.append('-');
- sb.append(type.getString(sl.finish));
- sb.append(']');
- }
-
+ String msg = String.format("Read %d live and %d tombstone cells in %s.%s for key: %1.512s (see tombstone_warn_threshold). %d columns were requested, slices=%1.512s",
columnCounter.live(),
- columnCounter.ignored(),
+ columnCounter.tombstones(),
container.metadata().ksName,
container.metadata().cfName,
container.metadata().getKeyValidator().getString(key.getKey()),
count,
- sb);
+ getSlicesInfo(container));
logger.warn(msg);
}
- Tracing.trace("Read {} live and {} tombstoned cells{}",
++ Tracing.trace("Read {} live and {} tombstone cells{}",
+ columnCounter.live(),
- columnCounter.ignored(),
++ columnCounter.tombstones(),
+ warnTombstones ? " (see tombstone_warn_threshold)" : "");
+ }
+
+ private String getSlicesInfo(ColumnFamily container)
+ {
+ StringBuilder sb = new StringBuilder();
+ CellNameType type = container.metadata().comparator;
+ for (ColumnSlice sl : slices)
+ {
+ assert sl != null;
+
+ sb.append('[');
+ sb.append(type.getString(sl.start));
+ sb.append('-');
+ sb.append(type.getString(sl.finish));
+ sb.append(']');
+ }
+ return sb.toString();
}
protected boolean respectTombstoneThresholds()
[2/3] cassandra git commit: Merge branch 'cassandra-2.0' into
cassandra-2.1
Posted by al...@apache.org.
Merge branch 'cassandra-2.0' into cassandra-2.1
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9beeba32
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9beeba32
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9beeba32
Branch: refs/heads/trunk
Commit: 9beeba3202716d26906b76feeff3bee3e16522d5
Parents: 2e7b088 bed42c2
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Tue May 12 21:01:39 2015 +0300
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Tue May 12 21:01:39 2015 +0300
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 2 +-
.../cassandra/db/filter/ColumnCounter.java | 29 ++--
.../cassandra/db/filter/SliceQueryFilter.java | 34 ++--
.../SliceQueryFilterWithTombstonesTest.java | 166 +++++++++++++++++++
5 files changed, 204 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9beeba32/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index aa5f235,d7d01cf..8794509
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,24 -1,5 +1,25 @@@
-2.0.15:
+2.1.6
+ * Fix commitlog getCompletedTasks to not increment (CASSANDRA-9339)
+ * Fix for harmless exceptions logged as ERROR (CASSANDRA-8564)
+ * Delete processed sstables in sstablesplit/sstableupgrade (CASSANDRA-8606)
+ * Improve sstable exclusion from partition tombstones (CASSANDRA-9298)
+ * Validate the indexed column rather than the cell's contents for 2i (CASSANDRA-9057)
+ * Add support for top-k custom 2i queries (CASSANDRA-8717)
+ * Fix error when dropping table during compaction (CASSANDRA-9251)
+ * cassandra-stress supports validation operations over user profiles (CASSANDRA-8773)
+ * Add support for rate limiting log messages (CASSANDRA-9029)
+ * Log the partition key with tombstone warnings (CASSANDRA-8561)
+ * Reduce runWithCompactionsDisabled poll interval to 1ms (CASSANDRA-9271)
+ * Fix PITR commitlog replay (CASSANDRA-9195)
+ * GCInspector logs very different times (CASSANDRA-9124)
+ * Fix deleting from an empty list (CASSANDRA-9198)
+ * Update tuple and collection types that use a user-defined type when that UDT
+ is modified (CASSANDRA-9148, CASSANDRA-9192)
+ * Use higher timeout for prepair and snapshot in repair (CASSANDRA-9261)
+ * Fix anticompaction blocking ANTI_ENTROPY stage (CASSANDRA-9151)
+ * Repair waits for anticompaction to finish (CASSANDRA-9097)
+Merged from 2.0:
+ * Fix counting of tombstones for TombstoneOverwhelmingException (CASSANDRA-9299)
* Fix ReconnectableSnitch reconnecting to peers during upgrade (CASSANDRA-6702)
* Include keyspace and table name in error log for collections over the size
limit (CASSANDRA-9286)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9beeba32/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9beeba32/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/filter/ColumnCounter.java
index 86cfc40,2d0df1f..8be26e1
--- a/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
+++ b/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
@@@ -37,17 -39,17 +37,17 @@@ public class ColumnCounte
this.timestamp = timestamp;
}
- public void count(Column column, DeletionInfo.InOrderTester tester)
+ public void count(Cell cell, DeletionInfo.InOrderTester tester)
{
- if (!isLive(cell, tester, timestamp))
- ignored++;
- else
- live++;
- }
+ // The cell is shadowed by a higher-level deletion, and won't be retained.
+ // For the purposes of this counter, we don't care if it's a tombstone or not.
- if (tester.isDeleted(column))
++ if (tester.isDeleted(cell))
+ return;
- protected static boolean isLive(Cell cell, DeletionInfo.InOrderTester tester, long timestamp)
- {
- return cell.isLive(timestamp) && !tester.isDeleted(cell);
- if (column.isLive(timestamp))
++ if (cell.isLive(timestamp))
+ live++;
+ else
+ tombstones++;
}
public int live()
@@@ -96,11 -99,14 +96,14 @@@
assert toGroup == 0 || type != null;
}
- public void count(Column column, DeletionInfo.InOrderTester tester)
+ public void count(Cell cell, DeletionInfo.InOrderTester tester)
{
- if (!isLive(cell, tester, timestamp))
- if (tester.isDeleted(column))
++ if (tester.isDeleted(cell))
+ return;
+
- if (!column.isLive(timestamp))
++ if (!cell.isLive(timestamp))
{
- ignored++;
+ tombstones++;
return;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9beeba32/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index 38947bf,6e6ab6b..1195d4c
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@@ -202,36 -190,36 +202,43 @@@ public class SliceQueryFilter implement
while (reducedColumns.hasNext())
{
- Column column = reducedColumns.next();
+ Cell cell = reducedColumns.next();
++
if (logger.isTraceEnabled())
-- logger.trace(String.format("collecting %s of %s: %s",
- columnCounter.live(), count, cell.getString(container.getComparator())));
- columnCounter.live(), count, column.getString(container.getComparator())));
++ logger.trace("collecting {} of {}: {}", columnCounter.live(), count, cell.getString(container.getComparator()));
++
++ // An expired tombstone will be immediately discarded in memory, and needn't be counted.
++ if (cell.getLocalDeletionTime() < gcBefore)
++ continue;
- columnCounter.count(column, tester);
+ columnCounter.count(cell, tester);
if (columnCounter.live() > count)
break;
- if (respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneFailureThreshold())
+ if (respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneFailureThreshold())
{
-- Tracing.trace("Scanned over {} tombstones; query aborted (see tombstone_failure_threshold)", DatabaseDescriptor.getTombstoneFailureThreshold());
++ Tracing.trace("Scanned over {} tombstones; query aborted (see tombstone_failure_threshold)",
++ DatabaseDescriptor.getTombstoneFailureThreshold());
logger.error("Scanned over {} tombstones in {}.{}; query aborted (see tombstone_failure_threshold)",
-- DatabaseDescriptor.getTombstoneFailureThreshold(), container.metadata().ksName, container.metadata().cfName);
++ DatabaseDescriptor.getTombstoneFailureThreshold(),
++ container.metadata().ksName,
++ container.metadata().cfName);
throw new TombstoneOverwhelmingException();
}
- container.addIfRelevant(column, tester, gcBefore);
+ container.maybeAppendColumn(cell, tester, gcBefore);
}
- Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
- if (logger.isWarnEnabled() && respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneWarnThreshold())
+ Tracing.trace("Read {} live and {} tombstone cells", columnCounter.live(), columnCounter.tombstones());
- if (respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneWarnThreshold())
++ if (logger.isWarnEnabled() && respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneWarnThreshold())
{
StringBuilder sb = new StringBuilder();
- AbstractType<?> type = container.metadata().comparator;
+ CellNameType type = container.metadata().comparator;
+
for (ColumnSlice sl : slices)
{
- if (sl == null)
- continue;
+ assert sl != null;
sb.append('[');
sb.append(type.getString(sl.start));
@@@ -240,15 -228,13 +247,15 @@@
sb.append(']');
}
- String msg = String.format("Read %d live and %d tombstoned cells in %s.%s for key: %1.512s (see tombstone_warn_threshold). %d columns were requested, slices=%1.512s",
- logger.warn("Read {} live and {} tombstone cells in {}.{} (see tombstone_warn_threshold). {} columns was requested, slices={}",
- columnCounter.live(),
- columnCounter.tombstones(),
- container.metadata().ksName,
- container.metadata().cfName,
- count,
- sb);
++ String msg = String.format("Read %d live and %d tombstone cells in %s.%s for key: %1.512s (see tombstone_warn_threshold). %d columns were requested, slices=%1.512s",
+ columnCounter.live(),
- columnCounter.ignored(),
++ columnCounter.tombstones(),
+ container.metadata().ksName,
+ container.metadata().cfName,
+ container.metadata().getKeyValidator().getString(key.getKey()),
+ count,
+ sb);
+ logger.warn(msg);
}
}
@@@ -435,11 -376,11 +442,10 @@@
ColumnSlice[] slices;
slices = new ColumnSlice[in.readInt()];
for (int i = 0; i < slices.length; i++)
- slices[i] = ColumnSlice.serializer.deserialize(in, version);
+ slices[i] = type.sliceSerializer().deserialize(in, version);
boolean reversed = in.readBoolean();
int count = in.readInt();
-- int compositesToGroup = -1;
-- compositesToGroup = in.readInt();
++ int compositesToGroup = in.readInt();
return new SliceQueryFilter(slices, reversed, count, compositesToGroup);
}
@@@ -459,43 -400,4 +465,43 @@@
return size;
}
}
+
+ public Iterator<RangeTombstone> getRangeTombstoneIterator(final ColumnFamily source)
+ {
+ final DeletionInfo delInfo = source.deletionInfo();
+ if (!delInfo.hasRanges() || slices.length == 0)
- return Iterators.<RangeTombstone>emptyIterator();
++ return Iterators.emptyIterator();
+
+ return new AbstractIterator<RangeTombstone>()
+ {
+ private int sliceIdx = 0;
+ private Iterator<RangeTombstone> sliceIter = currentRangeIter();
+
+ protected RangeTombstone computeNext()
+ {
+ while (true)
+ {
+ if (sliceIter.hasNext())
+ return sliceIter.next();
+
+ if (!nextSlice())
+ return endOfData();
+
+ sliceIter = currentRangeIter();
+ }
+ }
+
+ private Iterator<RangeTombstone> currentRangeIter()
+ {
+ ColumnSlice slice = slices[reversed ? (slices.length - 1 - sliceIdx) : sliceIdx];
+ return reversed ? delInfo.rangeIterator(slice.finish, slice.start)
+ : delInfo.rangeIterator(slice.start, slice.finish);
+ }
+
+ private boolean nextSlice()
+ {
+ return ++sliceIdx < slices.length;
+ }
+ };
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9beeba32/test/unit/org/apache/cassandra/cql3/SliceQueryFilterWithTombstonesTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/SliceQueryFilterWithTombstonesTest.java
index 0000000,4440782..0cb9819
mode 000000,100644..100644
--- a/test/unit/org/apache/cassandra/cql3/SliceQueryFilterWithTombstonesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/SliceQueryFilterWithTombstonesTest.java
@@@ -1,0 -1,150 +1,166 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.cassandra.cql3;
+
++import java.util.concurrent.TimeUnit;
++
+ import org.junit.AfterClass;
+ import org.junit.BeforeClass;
+ import org.junit.Test;
+
-import org.apache.cassandra.SchemaLoader;
+ import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ConsistencyLevel;
+ import org.apache.cassandra.db.filter.TombstoneOverwhelmingException;
-import org.apache.cassandra.gms.Gossiper;
+
+ import static junit.framework.Assert.assertTrue;
+ import static junit.framework.Assert.fail;
+
-import static org.apache.cassandra.cql3.QueryProcessor.process;
-import static org.apache.cassandra.cql3.QueryProcessor.processInternal;
-
+ /**
+ * Test that TombstoneOverwhelmingException gets thrown when it should be and doesn't when it shouldn't be.
+ */
-public class SliceQueryFilterWithTombstonesTest
++public class SliceQueryFilterWithTombstonesTest extends CQLTester
+ {
- static final String KEYSPACE = "tombstone_overwhelming_exception_test";
- static final String TABLE = "overwhelmed";
-
+ static final int ORIGINAL_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold();
+ static final int THRESHOLD = 100;
+
+ @BeforeClass
+ public static void setUp() throws Throwable
+ {
- SchemaLoader.loadSchema();
-
- process(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}",
- KEYSPACE),
- ConsistencyLevel.ONE);
-
- process(String.format("CREATE TABLE IF NOT EXISTS %s.%s (a text, b text, c text, PRIMARY KEY (a, b));",
- KEYSPACE,
- TABLE),
- ConsistencyLevel.ONE);
-
+ DatabaseDescriptor.setTombstoneFailureThreshold(THRESHOLD);
+ }
+
+ @AfterClass
+ public static void tearDown()
+ {
- Gossiper.instance.stop();
-
+ DatabaseDescriptor.setTombstoneFailureThreshold(ORIGINAL_THRESHOLD);
+ }
+
- private static UntypedResultSet execute(String query)
- {
- return processInternal(String.format(query, KEYSPACE, TABLE));
- }
-
+ @Test
- public void testBelowThresholdSelect()
++ public void testBelowThresholdSelect() throws Throwable
+ {
++ createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
++
+ // insert exactly the amount of tombstones that shouldn't trigger an exception
+ for (int i = 0; i < THRESHOLD; i++)
- execute("INSERT INTO %s.%s (a, b, c) VALUES ('key1', 'column" + i + "', null);");
++ execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+
+ try
+ {
- execute("SELECT * FROM %s.%s WHERE a = 'key1';");
++ execute("SELECT * FROM %s WHERE a = 'key';");
+ }
+ catch (Throwable e)
+ {
+ fail("SELECT with tombstones below the threshold should not have failed, but has: " + e);
+ }
+ }
+
+ @Test
- public void testBeyondThresholdSelect()
++ public void testBeyondThresholdSelect() throws Throwable
+ {
++ createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
++
+ // insert exactly the amount of tombstones that *SHOULD* trigger an exception
+ for (int i = 0; i < THRESHOLD + 1; i++)
- execute("INSERT INTO %s.%s (a, b, c) VALUES ('key2', 'column" + i + "', null);");
++ execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+
+ try
+ {
- execute("SELECT * FROM %s.%s WHERE a = 'key2';");
++ execute("SELECT * FROM %s WHERE a = 'key';");
+ fail("SELECT with tombstones beyond the threshold should have failed, but hasn't");
+ }
+ catch (Throwable e)
+ {
+ assertTrue(e instanceof TombstoneOverwhelmingException);
+ }
+ }
+
+ @Test
- public void testAllShadowedSelect()
++ public void testAllShadowedSelect() throws Throwable
+ {
++ createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
++
+ // insert exactly the amount of tombstones that *SHOULD* normally trigger an exception
+ for (int i = 0; i < THRESHOLD + 1; i++)
- execute("INSERT INTO %s.%s (a, b, c) VALUES ('key3', 'column" + i + "', null);");
++ execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+
+ // delete all with a partition level tombstone
- execute("DELETE FROM %s.%s WHERE a = 'key3'");
++ execute("DELETE FROM %s WHERE a = 'key'");
+
+ try
+ {
- execute("SELECT * FROM %s.%s WHERE a = 'key3';");
++ execute("SELECT * FROM %s WHERE a = 'key';");
+ }
+ catch (Throwable e)
+ {
+ fail("SELECT with tombstones shadowed by a partition tombstone should not have failed, but has: " + e);
+ }
+ }
+
+ @Test
- public void testLiveShadowedCellsSelect()
++ public void testLiveShadowedCellsSelect() throws Throwable
+ {
++ createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
++
+ for (int i = 0; i < THRESHOLD + 1; i++)
- execute("INSERT INTO %s.%s (a, b, c) VALUES ('key4', 'column" + i + "', 'column');");
++ execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', 'column');");
+
+ // delete all with a partition level tombstone
- execute("DELETE FROM %s.%s WHERE a = 'key4'");
++ execute("DELETE FROM %s WHERE a = 'key'");
+
+ try
+ {
- execute("SELECT * FROM %s.%s WHERE a = 'key4';");
++ execute("SELECT * FROM %s WHERE a = 'key';");
+ }
+ catch (Throwable e)
+ {
+ fail("SELECT with regular cells shadowed by a partition tombstone should not have failed, but has: " + e);
+ }
+ }
++
++ @Test
++ public void testExpiredTombstones() throws Throwable
++ {
++ createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds = 1;");
++
++ for (int i = 0; i < THRESHOLD + 1; i++)
++ execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
++
++ // not yet past gc grace - must throw a TOE
++ try
++ {
++ execute("SELECT * FROM %s WHERE a = 'key';");
++ fail("SELECT with tombstones beyond the threshold should have failed, but hasn't");
++ }
++ catch (Throwable e)
++ {
++ assertTrue(e instanceof TombstoneOverwhelmingException);
++ }
++
++ // sleep past gc grace
++ TimeUnit.SECONDS.sleep(2);
++
++ // past gc grace - must not throw a TOE now
++ try
++ {
++ execute("SELECT * FROM %s WHERE a = 'key';");
++ }
++ catch (Throwable e)
++ {
++ fail("SELECT with expired tombstones beyond the threshold should not have failed, but has: " + e);
++ }
++ }
+ }