You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2016/06/13 13:28:52 UTC
[01/10] cassandra git commit: Create interval tree over canonical
sstables to avoid missing sstables during streaming
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 68319f7c3 -> 72acbcd00
refs/heads/cassandra-2.2 593bbf57d -> 05f8a008f
refs/heads/cassandra-3.0 3d211e9fb -> 73a8341fe
refs/heads/trunk db8df9153 -> 719e7d662
Create interval tree over canonical sstables to avoid missing sstables during streaming
patch by marcuse; reviewed by benedict for CASSANDRA-11886
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/72acbcd0
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/72acbcd0
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/72acbcd0
Branch: refs/heads/cassandra-2.1
Commit: 72acbcd00fe7c46e54cd267f42868531e99e39df
Parents: 68319f7
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed May 25 08:38:14 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Jun 13 14:31:47 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 4 ++
.../org/apache/cassandra/db/DataTracker.java | 8 ++-
.../cassandra/streaming/StreamSession.java | 21 +++---
.../io/sstable/SSTableRewriterTest.java | 72 ++++++++++++++++++++
5 files changed, 93 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index af641e1..ebcc90c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.15
+ * Create interval tree over canonical sstables to avoid missing sstables during streaming (CASSANDRA-11886)
* cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
* Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 166ce7e..559ba0b 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -1531,6 +1531,10 @@ public class DatabaseDescriptor
{
return conf.sstable_preemptive_open_interval_in_mb;
}
+ public static void setSSTablePreempiveOpenIntervalInMB(int mb)
+ {
+ conf.sstable_preemptive_open_interval_in_mb = mb;
+ }
public static boolean getTrickleFsync()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/src/java/org/apache/cassandra/db/DataTracker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/DataTracker.java b/src/java/org/apache/cassandra/db/DataTracker.java
index c731a35..927e717 100644
--- a/src/java/org/apache/cassandra/db/DataTracker.java
+++ b/src/java/org/apache/cassandra/db/DataTracker.java
@@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.sstable.IndexSummary;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
@@ -810,9 +811,14 @@ public class DataTracker
public List<SSTableReader> sstablesInBounds(AbstractBounds<RowPosition> rowBounds)
{
+ return sstablesInBounds(rowBounds, intervalTree, liveMemtables.get(0).cfs.partitioner);
+ }
+
+ public static List<SSTableReader> sstablesInBounds(AbstractBounds<RowPosition> rowBounds, SSTableIntervalTree intervalTree, IPartitioner partitioner)
+ {
if (intervalTree.isEmpty())
return Collections.emptyList();
- RowPosition stopInTree = rowBounds.right.isMinimum(liveMemtables.get(0).cfs.partitioner) ? intervalTree.max() : rowBounds.right;
+ RowPosition stopInTree = rowBounds.right.isMinimum(partitioner) ? intervalTree.max() : rowBounds.right;
return intervalTree.search(Interval.<RowPosition, SSTableReader>create(rowBounds.left, stopInTree));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/src/java/org/apache/cassandra/streaming/StreamSession.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java
index 4eb8557..273631c 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.collect.*;
import org.slf4j.Logger;
@@ -38,6 +39,8 @@ import org.apache.cassandra.db.DataTracker;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowPosition;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.*;
@@ -295,7 +298,8 @@ public class StreamSession implements IEndpointStateChangeSubscriber
return stores;
}
- private List<SSTableStreamingSections> getSSTableSectionsForRanges(Collection<Range<Token>> ranges, Collection<ColumnFamilyStore> stores, long overriddenRepairedAt, final boolean isIncremental)
+ @VisibleForTesting
+ public static List<SSTableStreamingSections> getSSTableSectionsForRanges(Collection<Range<Token>> ranges, Collection<ColumnFamilyStore> stores, long overriddenRepairedAt, final boolean isIncremental)
{
Refs<SSTableReader> refs = new Refs<>();
try
@@ -303,30 +307,23 @@ public class StreamSession implements IEndpointStateChangeSubscriber
for (ColumnFamilyStore cfStore : stores)
{
final List<AbstractBounds<RowPosition>> rowBoundsList = new ArrayList<>(ranges.size());
+ final IPartitioner partitioner = cfStore.partitioner;
for (Range<Token> range : ranges)
rowBoundsList.add(range.toRowBounds());
refs.addAll(cfStore.selectAndReference(new Function<DataTracker.View, List<SSTableReader>>()
{
public List<SSTableReader> apply(DataTracker.View view)
{
- Map<SSTableReader, SSTableReader> permittedInstances = new HashMap<>();
- for (SSTableReader reader : ColumnFamilyStore.CANONICAL_SSTABLES.apply(view))
- permittedInstances.put(reader, reader);
-
+ DataTracker.SSTableIntervalTree intervalTree = DataTracker.buildIntervalTree(ColumnFamilyStore.CANONICAL_SSTABLES.apply(view));
Set<SSTableReader> sstables = Sets.newHashSet();
for (AbstractBounds<RowPosition> rowBounds : rowBoundsList)
{
- // sstableInBounds may contain early opened sstables
- for (SSTableReader sstable : view.sstablesInBounds(rowBounds))
+ for (SSTableReader sstable : DataTracker.View.sstablesInBounds(rowBounds, intervalTree, partitioner))
{
- if (isIncremental && sstable.isRepaired())
- continue;
- sstable = permittedInstances.get(sstable);
- if (sstable != null)
+ if (!isIncremental || !sstable.isRepaired())
sstables.add(sstable);
}
}
-
return ImmutableList.copyOf(sstables);
}
}).refs);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index a735657..1fb28f5 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -21,13 +21,16 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Sets;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
@@ -39,7 +42,11 @@ import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.metrics.StorageMetrics;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
@@ -767,6 +774,71 @@ public class SSTableRewriterTest extends SchemaLoader
truncate(cfs);
}
+ @Test
+ public void testSSTableSectionsForRanges() throws IOException, InterruptedException, ExecutionException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+ truncate(cfs);
+
+ cfs.addSSTable(writeFile(cfs, 1000));
+
+ Collection<SSTableReader> allSSTables = cfs.getSSTables();
+ assertEquals(1, allSSTables.size());
+ final Token firstToken = allSSTables.iterator().next().first.getToken();
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(1);
+
+ List<StreamSession.SSTableStreamingSections> sectionsBeforeRewrite = StreamSession.getSSTableSectionsForRanges(
+ Collections.singleton(new Range<Token>(firstToken, firstToken)),
+ Collections.singleton(cfs), 0L, false);
+ assertEquals(1, sectionsBeforeRewrite.size());
+ for (StreamSession.SSTableStreamingSections section : sectionsBeforeRewrite)
+ section.ref.release();
+ final AtomicInteger checkCount = new AtomicInteger();
+ // needed since we get notified when compaction is done as well - we can't get sections for ranges for obsoleted sstables
+ INotificationConsumer consumer = new INotificationConsumer()
+ {
+ public void handleNotification(INotification notification, Object sender)
+ {
+ if (notification instanceof SSTableListChangedNotification)
+ {
+ Collection<SSTableReader> added = ((SSTableListChangedNotification) notification).added;
+ Collection<SSTableReader> removed = ((SSTableListChangedNotification) notification).removed;
+ // note that we need to check if added.equals(removed) because once the compaction is done the old sstable will have
+ // selfRef().globalCount() == 0 and we cant get the SectionsForRanges then. During incremental opening we always add and remove the same
+ // sstable (note that the sstables are x.equal(y) but not x == y since the new one will be a new instance with a moved starting point
+ // In this case we must avoid trying to call getSSTableSectionsForRanges since we are in the notification
+ // method and trying to reference an sstable with globalcount == 0 puts it into a loop, and this blocks the tracker from removing the
+ // unreferenced sstable.
+ if (added.isEmpty() || !added.iterator().next().getColumnFamilyName().equals(cfs.getColumnFamilyName()) || !added.equals(removed))
+ return;
+
+ // at no point must the rewrite process hide
+ // sections returned by getSSTableSectionsForRanges
+ Set<Range<Token>> range = Collections.singleton(new Range<Token>(firstToken, firstToken));
+ List<StreamSession.SSTableStreamingSections> sections = StreamSession.getSSTableSectionsForRanges(range, Collections.singleton(cfs), 0L, false);
+ assertEquals(1, sections.size());
+ for (StreamSession.SSTableStreamingSections section : sections)
+ section.ref.release();
+ checkCount.incrementAndGet();
+ }
+ }
+ };
+ cfs.getDataTracker().subscribe(consumer);
+ try
+ {
+ cfs.forceMajorCompaction();
+ // reset
+ }
+ finally
+ {
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(50);
+ cfs.getDataTracker().unsubscribe(consumer);
+ }
+ assertTrue(checkCount.get() >= 2);
+ truncate(cfs);
+ }
+
/**
* emulates anticompaction - writing from one source sstable to two new sstables
*
[04/10] cassandra git commit: Create interval tree over canonical
sstables to avoid missing sstables during streaming
Posted by ma...@apache.org.
Create interval tree over canonical sstables to avoid missing sstables during streaming
patch by marcuse; reviewed by benedict for CASSANDRA-11886
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/72acbcd0
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/72acbcd0
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/72acbcd0
Branch: refs/heads/trunk
Commit: 72acbcd00fe7c46e54cd267f42868531e99e39df
Parents: 68319f7
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed May 25 08:38:14 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Jun 13 14:31:47 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 4 ++
.../org/apache/cassandra/db/DataTracker.java | 8 ++-
.../cassandra/streaming/StreamSession.java | 21 +++---
.../io/sstable/SSTableRewriterTest.java | 72 ++++++++++++++++++++
5 files changed, 93 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index af641e1..ebcc90c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.15
+ * Create interval tree over canonical sstables to avoid missing sstables during streaming (CASSANDRA-11886)
* cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
* Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 166ce7e..559ba0b 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -1531,6 +1531,10 @@ public class DatabaseDescriptor
{
return conf.sstable_preemptive_open_interval_in_mb;
}
+ public static void setSSTablePreempiveOpenIntervalInMB(int mb)
+ {
+ conf.sstable_preemptive_open_interval_in_mb = mb;
+ }
public static boolean getTrickleFsync()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/src/java/org/apache/cassandra/db/DataTracker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/DataTracker.java b/src/java/org/apache/cassandra/db/DataTracker.java
index c731a35..927e717 100644
--- a/src/java/org/apache/cassandra/db/DataTracker.java
+++ b/src/java/org/apache/cassandra/db/DataTracker.java
@@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.sstable.IndexSummary;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
@@ -810,9 +811,14 @@ public class DataTracker
public List<SSTableReader> sstablesInBounds(AbstractBounds<RowPosition> rowBounds)
{
+ return sstablesInBounds(rowBounds, intervalTree, liveMemtables.get(0).cfs.partitioner);
+ }
+
+ public static List<SSTableReader> sstablesInBounds(AbstractBounds<RowPosition> rowBounds, SSTableIntervalTree intervalTree, IPartitioner partitioner)
+ {
if (intervalTree.isEmpty())
return Collections.emptyList();
- RowPosition stopInTree = rowBounds.right.isMinimum(liveMemtables.get(0).cfs.partitioner) ? intervalTree.max() : rowBounds.right;
+ RowPosition stopInTree = rowBounds.right.isMinimum(partitioner) ? intervalTree.max() : rowBounds.right;
return intervalTree.search(Interval.<RowPosition, SSTableReader>create(rowBounds.left, stopInTree));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/src/java/org/apache/cassandra/streaming/StreamSession.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java
index 4eb8557..273631c 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.collect.*;
import org.slf4j.Logger;
@@ -38,6 +39,8 @@ import org.apache.cassandra.db.DataTracker;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowPosition;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.*;
@@ -295,7 +298,8 @@ public class StreamSession implements IEndpointStateChangeSubscriber
return stores;
}
- private List<SSTableStreamingSections> getSSTableSectionsForRanges(Collection<Range<Token>> ranges, Collection<ColumnFamilyStore> stores, long overriddenRepairedAt, final boolean isIncremental)
+ @VisibleForTesting
+ public static List<SSTableStreamingSections> getSSTableSectionsForRanges(Collection<Range<Token>> ranges, Collection<ColumnFamilyStore> stores, long overriddenRepairedAt, final boolean isIncremental)
{
Refs<SSTableReader> refs = new Refs<>();
try
@@ -303,30 +307,23 @@ public class StreamSession implements IEndpointStateChangeSubscriber
for (ColumnFamilyStore cfStore : stores)
{
final List<AbstractBounds<RowPosition>> rowBoundsList = new ArrayList<>(ranges.size());
+ final IPartitioner partitioner = cfStore.partitioner;
for (Range<Token> range : ranges)
rowBoundsList.add(range.toRowBounds());
refs.addAll(cfStore.selectAndReference(new Function<DataTracker.View, List<SSTableReader>>()
{
public List<SSTableReader> apply(DataTracker.View view)
{
- Map<SSTableReader, SSTableReader> permittedInstances = new HashMap<>();
- for (SSTableReader reader : ColumnFamilyStore.CANONICAL_SSTABLES.apply(view))
- permittedInstances.put(reader, reader);
-
+ DataTracker.SSTableIntervalTree intervalTree = DataTracker.buildIntervalTree(ColumnFamilyStore.CANONICAL_SSTABLES.apply(view));
Set<SSTableReader> sstables = Sets.newHashSet();
for (AbstractBounds<RowPosition> rowBounds : rowBoundsList)
{
- // sstableInBounds may contain early opened sstables
- for (SSTableReader sstable : view.sstablesInBounds(rowBounds))
+ for (SSTableReader sstable : DataTracker.View.sstablesInBounds(rowBounds, intervalTree, partitioner))
{
- if (isIncremental && sstable.isRepaired())
- continue;
- sstable = permittedInstances.get(sstable);
- if (sstable != null)
+ if (!isIncremental || !sstable.isRepaired())
sstables.add(sstable);
}
}
-
return ImmutableList.copyOf(sstables);
}
}).refs);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index a735657..1fb28f5 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -21,13 +21,16 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Sets;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
@@ -39,7 +42,11 @@ import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.metrics.StorageMetrics;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
@@ -767,6 +774,71 @@ public class SSTableRewriterTest extends SchemaLoader
truncate(cfs);
}
+ @Test
+ public void testSSTableSectionsForRanges() throws IOException, InterruptedException, ExecutionException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+ truncate(cfs);
+
+ cfs.addSSTable(writeFile(cfs, 1000));
+
+ Collection<SSTableReader> allSSTables = cfs.getSSTables();
+ assertEquals(1, allSSTables.size());
+ final Token firstToken = allSSTables.iterator().next().first.getToken();
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(1);
+
+ List<StreamSession.SSTableStreamingSections> sectionsBeforeRewrite = StreamSession.getSSTableSectionsForRanges(
+ Collections.singleton(new Range<Token>(firstToken, firstToken)),
+ Collections.singleton(cfs), 0L, false);
+ assertEquals(1, sectionsBeforeRewrite.size());
+ for (StreamSession.SSTableStreamingSections section : sectionsBeforeRewrite)
+ section.ref.release();
+ final AtomicInteger checkCount = new AtomicInteger();
+ // needed since we get notified when compaction is done as well - we can't get sections for ranges for obsoleted sstables
+ INotificationConsumer consumer = new INotificationConsumer()
+ {
+ public void handleNotification(INotification notification, Object sender)
+ {
+ if (notification instanceof SSTableListChangedNotification)
+ {
+ Collection<SSTableReader> added = ((SSTableListChangedNotification) notification).added;
+ Collection<SSTableReader> removed = ((SSTableListChangedNotification) notification).removed;
+ // note that we need to check if added.equals(removed) because once the compaction is done the old sstable will have
+ // selfRef().globalCount() == 0 and we cant get the SectionsForRanges then. During incremental opening we always add and remove the same
+ // sstable (note that the sstables are x.equal(y) but not x == y since the new one will be a new instance with a moved starting point
+ // In this case we must avoid trying to call getSSTableSectionsForRanges since we are in the notification
+ // method and trying to reference an sstable with globalcount == 0 puts it into a loop, and this blocks the tracker from removing the
+ // unreferenced sstable.
+ if (added.isEmpty() || !added.iterator().next().getColumnFamilyName().equals(cfs.getColumnFamilyName()) || !added.equals(removed))
+ return;
+
+ // at no point must the rewrite process hide
+ // sections returned by getSSTableSectionsForRanges
+ Set<Range<Token>> range = Collections.singleton(new Range<Token>(firstToken, firstToken));
+ List<StreamSession.SSTableStreamingSections> sections = StreamSession.getSSTableSectionsForRanges(range, Collections.singleton(cfs), 0L, false);
+ assertEquals(1, sections.size());
+ for (StreamSession.SSTableStreamingSections section : sections)
+ section.ref.release();
+ checkCount.incrementAndGet();
+ }
+ }
+ };
+ cfs.getDataTracker().subscribe(consumer);
+ try
+ {
+ cfs.forceMajorCompaction();
+ // reset
+ }
+ finally
+ {
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(50);
+ cfs.getDataTracker().unsubscribe(consumer);
+ }
+ assertTrue(checkCount.get() >= 2);
+ truncate(cfs);
+ }
+
/**
* emulates anticompaction - writing from one source sstable to two new sstables
*
[02/10] cassandra git commit: Create interval tree over canonical
sstables to avoid missing sstables during streaming
Posted by ma...@apache.org.
Create interval tree over canonical sstables to avoid missing sstables during streaming
patch by marcuse; reviewed by benedict for CASSANDRA-11886
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/72acbcd0
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/72acbcd0
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/72acbcd0
Branch: refs/heads/cassandra-2.2
Commit: 72acbcd00fe7c46e54cd267f42868531e99e39df
Parents: 68319f7
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed May 25 08:38:14 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Jun 13 14:31:47 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 4 ++
.../org/apache/cassandra/db/DataTracker.java | 8 ++-
.../cassandra/streaming/StreamSession.java | 21 +++---
.../io/sstable/SSTableRewriterTest.java | 72 ++++++++++++++++++++
5 files changed, 93 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index af641e1..ebcc90c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.15
+ * Create interval tree over canonical sstables to avoid missing sstables during streaming (CASSANDRA-11886)
* cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
* Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 166ce7e..559ba0b 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -1531,6 +1531,10 @@ public class DatabaseDescriptor
{
return conf.sstable_preemptive_open_interval_in_mb;
}
+ public static void setSSTablePreempiveOpenIntervalInMB(int mb)
+ {
+ conf.sstable_preemptive_open_interval_in_mb = mb;
+ }
public static boolean getTrickleFsync()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/src/java/org/apache/cassandra/db/DataTracker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/DataTracker.java b/src/java/org/apache/cassandra/db/DataTracker.java
index c731a35..927e717 100644
--- a/src/java/org/apache/cassandra/db/DataTracker.java
+++ b/src/java/org/apache/cassandra/db/DataTracker.java
@@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.sstable.IndexSummary;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
@@ -810,9 +811,14 @@ public class DataTracker
public List<SSTableReader> sstablesInBounds(AbstractBounds<RowPosition> rowBounds)
{
+ return sstablesInBounds(rowBounds, intervalTree, liveMemtables.get(0).cfs.partitioner);
+ }
+
+ public static List<SSTableReader> sstablesInBounds(AbstractBounds<RowPosition> rowBounds, SSTableIntervalTree intervalTree, IPartitioner partitioner)
+ {
if (intervalTree.isEmpty())
return Collections.emptyList();
- RowPosition stopInTree = rowBounds.right.isMinimum(liveMemtables.get(0).cfs.partitioner) ? intervalTree.max() : rowBounds.right;
+ RowPosition stopInTree = rowBounds.right.isMinimum(partitioner) ? intervalTree.max() : rowBounds.right;
return intervalTree.search(Interval.<RowPosition, SSTableReader>create(rowBounds.left, stopInTree));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/src/java/org/apache/cassandra/streaming/StreamSession.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java
index 4eb8557..273631c 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.collect.*;
import org.slf4j.Logger;
@@ -38,6 +39,8 @@ import org.apache.cassandra.db.DataTracker;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowPosition;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.*;
@@ -295,7 +298,8 @@ public class StreamSession implements IEndpointStateChangeSubscriber
return stores;
}
- private List<SSTableStreamingSections> getSSTableSectionsForRanges(Collection<Range<Token>> ranges, Collection<ColumnFamilyStore> stores, long overriddenRepairedAt, final boolean isIncremental)
+ @VisibleForTesting
+ public static List<SSTableStreamingSections> getSSTableSectionsForRanges(Collection<Range<Token>> ranges, Collection<ColumnFamilyStore> stores, long overriddenRepairedAt, final boolean isIncremental)
{
Refs<SSTableReader> refs = new Refs<>();
try
@@ -303,30 +307,23 @@ public class StreamSession implements IEndpointStateChangeSubscriber
for (ColumnFamilyStore cfStore : stores)
{
final List<AbstractBounds<RowPosition>> rowBoundsList = new ArrayList<>(ranges.size());
+ final IPartitioner partitioner = cfStore.partitioner;
for (Range<Token> range : ranges)
rowBoundsList.add(range.toRowBounds());
refs.addAll(cfStore.selectAndReference(new Function<DataTracker.View, List<SSTableReader>>()
{
public List<SSTableReader> apply(DataTracker.View view)
{
- Map<SSTableReader, SSTableReader> permittedInstances = new HashMap<>();
- for (SSTableReader reader : ColumnFamilyStore.CANONICAL_SSTABLES.apply(view))
- permittedInstances.put(reader, reader);
-
+ DataTracker.SSTableIntervalTree intervalTree = DataTracker.buildIntervalTree(ColumnFamilyStore.CANONICAL_SSTABLES.apply(view));
Set<SSTableReader> sstables = Sets.newHashSet();
for (AbstractBounds<RowPosition> rowBounds : rowBoundsList)
{
- // sstableInBounds may contain early opened sstables
- for (SSTableReader sstable : view.sstablesInBounds(rowBounds))
+ for (SSTableReader sstable : DataTracker.View.sstablesInBounds(rowBounds, intervalTree, partitioner))
{
- if (isIncremental && sstable.isRepaired())
- continue;
- sstable = permittedInstances.get(sstable);
- if (sstable != null)
+ if (!isIncremental || !sstable.isRepaired())
sstables.add(sstable);
}
}
-
return ImmutableList.copyOf(sstables);
}
}).refs);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index a735657..1fb28f5 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -21,13 +21,16 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Sets;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
@@ -39,7 +42,11 @@ import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.metrics.StorageMetrics;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
@@ -767,6 +774,71 @@ public class SSTableRewriterTest extends SchemaLoader
truncate(cfs);
}
+ @Test
+ public void testSSTableSectionsForRanges() throws IOException, InterruptedException, ExecutionException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+ truncate(cfs);
+
+ cfs.addSSTable(writeFile(cfs, 1000));
+
+ Collection<SSTableReader> allSSTables = cfs.getSSTables();
+ assertEquals(1, allSSTables.size());
+ final Token firstToken = allSSTables.iterator().next().first.getToken();
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(1);
+
+ List<StreamSession.SSTableStreamingSections> sectionsBeforeRewrite = StreamSession.getSSTableSectionsForRanges(
+ Collections.singleton(new Range<Token>(firstToken, firstToken)),
+ Collections.singleton(cfs), 0L, false);
+ assertEquals(1, sectionsBeforeRewrite.size());
+ for (StreamSession.SSTableStreamingSections section : sectionsBeforeRewrite)
+ section.ref.release();
+ final AtomicInteger checkCount = new AtomicInteger();
+ // needed since we get notified when compaction is done as well - we can't get sections for ranges for obsoleted sstables
+ INotificationConsumer consumer = new INotificationConsumer()
+ {
+ public void handleNotification(INotification notification, Object sender)
+ {
+ if (notification instanceof SSTableListChangedNotification)
+ {
+ Collection<SSTableReader> added = ((SSTableListChangedNotification) notification).added;
+ Collection<SSTableReader> removed = ((SSTableListChangedNotification) notification).removed;
+ // note that we need to check if added.equals(removed) because once the compaction is done the old sstable will have
+ // selfRef().globalCount() == 0 and we cant get the SectionsForRanges then. During incremental opening we always add and remove the same
+ // sstable (note that the sstables are x.equal(y) but not x == y since the new one will be a new instance with a moved starting point
+ // In this case we must avoid trying to call getSSTableSectionsForRanges since we are in the notification
+ // method and trying to reference an sstable with globalcount == 0 puts it into a loop, and this blocks the tracker from removing the
+ // unreferenced sstable.
+ if (added.isEmpty() || !added.iterator().next().getColumnFamilyName().equals(cfs.getColumnFamilyName()) || !added.equals(removed))
+ return;
+
+ // at no point must the rewrite process hide
+ // sections returned by getSSTableSectionsForRanges
+ Set<Range<Token>> range = Collections.singleton(new Range<Token>(firstToken, firstToken));
+ List<StreamSession.SSTableStreamingSections> sections = StreamSession.getSSTableSectionsForRanges(range, Collections.singleton(cfs), 0L, false);
+ assertEquals(1, sections.size());
+ for (StreamSession.SSTableStreamingSections section : sections)
+ section.ref.release();
+ checkCount.incrementAndGet();
+ }
+ }
+ };
+ cfs.getDataTracker().subscribe(consumer);
+ try
+ {
+ cfs.forceMajorCompaction();
+ // reset
+ }
+ finally
+ {
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(50);
+ cfs.getDataTracker().unsubscribe(consumer);
+ }
+ assertTrue(checkCount.get() >= 2);
+ truncate(cfs);
+ }
+
/**
* emulates anticompaction - writing from one source sstable to two new sstables
*
[03/10] cassandra git commit: Create interval tree over canonical
sstables to avoid missing sstables during streaming
Posted by ma...@apache.org.
Create interval tree over canonical sstables to avoid missing sstables during streaming
patch by marcuse; reviewed by benedict for CASSANDRA-11886
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/72acbcd0
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/72acbcd0
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/72acbcd0
Branch: refs/heads/cassandra-3.0
Commit: 72acbcd00fe7c46e54cd267f42868531e99e39df
Parents: 68319f7
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed May 25 08:38:14 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Jun 13 14:31:47 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 4 ++
.../org/apache/cassandra/db/DataTracker.java | 8 ++-
.../cassandra/streaming/StreamSession.java | 21 +++---
.../io/sstable/SSTableRewriterTest.java | 72 ++++++++++++++++++++
5 files changed, 93 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index af641e1..ebcc90c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.15
+ * Create interval tree over canonical sstables to avoid missing sstables during streaming (CASSANDRA-11886)
* cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
* Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 166ce7e..559ba0b 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -1531,6 +1531,10 @@ public class DatabaseDescriptor
{
return conf.sstable_preemptive_open_interval_in_mb;
}
+ public static void setSSTablePreempiveOpenIntervalInMB(int mb)
+ {
+ conf.sstable_preemptive_open_interval_in_mb = mb;
+ }
public static boolean getTrickleFsync()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/src/java/org/apache/cassandra/db/DataTracker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/DataTracker.java b/src/java/org/apache/cassandra/db/DataTracker.java
index c731a35..927e717 100644
--- a/src/java/org/apache/cassandra/db/DataTracker.java
+++ b/src/java/org/apache/cassandra/db/DataTracker.java
@@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.sstable.IndexSummary;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
@@ -810,9 +811,14 @@ public class DataTracker
public List<SSTableReader> sstablesInBounds(AbstractBounds<RowPosition> rowBounds)
{
+ return sstablesInBounds(rowBounds, intervalTree, liveMemtables.get(0).cfs.partitioner);
+ }
+
+ public static List<SSTableReader> sstablesInBounds(AbstractBounds<RowPosition> rowBounds, SSTableIntervalTree intervalTree, IPartitioner partitioner)
+ {
if (intervalTree.isEmpty())
return Collections.emptyList();
- RowPosition stopInTree = rowBounds.right.isMinimum(liveMemtables.get(0).cfs.partitioner) ? intervalTree.max() : rowBounds.right;
+ RowPosition stopInTree = rowBounds.right.isMinimum(partitioner) ? intervalTree.max() : rowBounds.right;
return intervalTree.search(Interval.<RowPosition, SSTableReader>create(rowBounds.left, stopInTree));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/src/java/org/apache/cassandra/streaming/StreamSession.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java
index 4eb8557..273631c 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.collect.*;
import org.slf4j.Logger;
@@ -38,6 +39,8 @@ import org.apache.cassandra.db.DataTracker;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowPosition;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.*;
@@ -295,7 +298,8 @@ public class StreamSession implements IEndpointStateChangeSubscriber
return stores;
}
- private List<SSTableStreamingSections> getSSTableSectionsForRanges(Collection<Range<Token>> ranges, Collection<ColumnFamilyStore> stores, long overriddenRepairedAt, final boolean isIncremental)
+ @VisibleForTesting
+ public static List<SSTableStreamingSections> getSSTableSectionsForRanges(Collection<Range<Token>> ranges, Collection<ColumnFamilyStore> stores, long overriddenRepairedAt, final boolean isIncremental)
{
Refs<SSTableReader> refs = new Refs<>();
try
@@ -303,30 +307,23 @@ public class StreamSession implements IEndpointStateChangeSubscriber
for (ColumnFamilyStore cfStore : stores)
{
final List<AbstractBounds<RowPosition>> rowBoundsList = new ArrayList<>(ranges.size());
+ final IPartitioner partitioner = cfStore.partitioner;
for (Range<Token> range : ranges)
rowBoundsList.add(range.toRowBounds());
refs.addAll(cfStore.selectAndReference(new Function<DataTracker.View, List<SSTableReader>>()
{
public List<SSTableReader> apply(DataTracker.View view)
{
- Map<SSTableReader, SSTableReader> permittedInstances = new HashMap<>();
- for (SSTableReader reader : ColumnFamilyStore.CANONICAL_SSTABLES.apply(view))
- permittedInstances.put(reader, reader);
-
+ DataTracker.SSTableIntervalTree intervalTree = DataTracker.buildIntervalTree(ColumnFamilyStore.CANONICAL_SSTABLES.apply(view));
Set<SSTableReader> sstables = Sets.newHashSet();
for (AbstractBounds<RowPosition> rowBounds : rowBoundsList)
{
- // sstableInBounds may contain early opened sstables
- for (SSTableReader sstable : view.sstablesInBounds(rowBounds))
+ for (SSTableReader sstable : DataTracker.View.sstablesInBounds(rowBounds, intervalTree, partitioner))
{
- if (isIncremental && sstable.isRepaired())
- continue;
- sstable = permittedInstances.get(sstable);
- if (sstable != null)
+ if (!isIncremental || !sstable.isRepaired())
sstables.add(sstable);
}
}
-
return ImmutableList.copyOf(sstables);
}
}).refs);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/72acbcd0/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index a735657..1fb28f5 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -21,13 +21,16 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Sets;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
@@ -39,7 +42,11 @@ import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.metrics.StorageMetrics;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
@@ -767,6 +774,71 @@ public class SSTableRewriterTest extends SchemaLoader
truncate(cfs);
}
+ @Test
+ public void testSSTableSectionsForRanges() throws IOException, InterruptedException, ExecutionException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+ truncate(cfs);
+
+ cfs.addSSTable(writeFile(cfs, 1000));
+
+ Collection<SSTableReader> allSSTables = cfs.getSSTables();
+ assertEquals(1, allSSTables.size());
+ final Token firstToken = allSSTables.iterator().next().first.getToken();
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(1);
+
+ List<StreamSession.SSTableStreamingSections> sectionsBeforeRewrite = StreamSession.getSSTableSectionsForRanges(
+ Collections.singleton(new Range<Token>(firstToken, firstToken)),
+ Collections.singleton(cfs), 0L, false);
+ assertEquals(1, sectionsBeforeRewrite.size());
+ for (StreamSession.SSTableStreamingSections section : sectionsBeforeRewrite)
+ section.ref.release();
+ final AtomicInteger checkCount = new AtomicInteger();
+ // needed since we get notified when compaction is done as well - we can't get sections for ranges for obsoleted sstables
+ INotificationConsumer consumer = new INotificationConsumer()
+ {
+ public void handleNotification(INotification notification, Object sender)
+ {
+ if (notification instanceof SSTableListChangedNotification)
+ {
+ Collection<SSTableReader> added = ((SSTableListChangedNotification) notification).added;
+ Collection<SSTableReader> removed = ((SSTableListChangedNotification) notification).removed;
+ // note that we need to check if added.equals(removed) because once the compaction is done the old sstable will have
+ // selfRef().globalCount() == 0 and we cant get the SectionsForRanges then. During incremental opening we always add and remove the same
+ // sstable (note that the sstables are x.equal(y) but not x == y since the new one will be a new instance with a moved starting point
+ // In this case we must avoid trying to call getSSTableSectionsForRanges since we are in the notification
+ // method and trying to reference an sstable with globalcount == 0 puts it into a loop, and this blocks the tracker from removing the
+ // unreferenced sstable.
+ if (added.isEmpty() || !added.iterator().next().getColumnFamilyName().equals(cfs.getColumnFamilyName()) || !added.equals(removed))
+ return;
+
+ // at no point must the rewrite process hide
+ // sections returned by getSSTableSectionsForRanges
+ Set<Range<Token>> range = Collections.singleton(new Range<Token>(firstToken, firstToken));
+ List<StreamSession.SSTableStreamingSections> sections = StreamSession.getSSTableSectionsForRanges(range, Collections.singleton(cfs), 0L, false);
+ assertEquals(1, sections.size());
+ for (StreamSession.SSTableStreamingSections section : sections)
+ section.ref.release();
+ checkCount.incrementAndGet();
+ }
+ }
+ };
+ cfs.getDataTracker().subscribe(consumer);
+ try
+ {
+ cfs.forceMajorCompaction();
+ // reset
+ }
+ finally
+ {
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(50);
+ cfs.getDataTracker().unsubscribe(consumer);
+ }
+ assertTrue(checkCount.get() >= 2);
+ truncate(cfs);
+ }
+
/**
* emulates anticompaction - writing from one source sstable to two new sstables
*
[10/10] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/719e7d66
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/719e7d66
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/719e7d66
Branch: refs/heads/trunk
Commit: 719e7d6624c2755295dd89670cff1aebf3c73795
Parents: db8df91 73a8341
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Jun 13 15:16:33 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Jun 13 15:16:33 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 4 ++
.../org/apache/cassandra/db/lifecycle/View.java | 11 +++
.../cassandra/streaming/StreamSession.java | 18 +++--
.../io/sstable/SSTableRewriterTest.java | 75 +++++++++++++++++---
5 files changed, 88 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/719e7d66/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/719e7d66/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/719e7d66/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/719e7d66/src/java/org/apache/cassandra/streaming/StreamSession.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/719e7d66/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
[09/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/73a8341f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/73a8341f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/73a8341f
Branch: refs/heads/trunk
Commit: 73a8341fef25de7236bc591e84cddc637c0b7b2f
Parents: 3d211e9 05f8a00
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Jun 13 15:14:28 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Jun 13 15:14:28 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 4 ++
.../org/apache/cassandra/db/lifecycle/View.java | 11 +++
.../cassandra/streaming/StreamSession.java | 18 +++--
.../io/sstable/SSTableRewriterTest.java | 75 +++++++++++++++++---
5 files changed, 88 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/73a8341f/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 47aef7e,491f72a..8a04077
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,28 -1,5 +1,29 @@@
-2.2.7
+3.0.8
+ * Add TimeWindowCompactionStrategy (CASSANDRA-9666)
+Merged from 2.2:
* StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
+Merged from 2.1:
++ * Create interval tree over canonical sstables to avoid missing sstables during streaming (CASSANDRA-11886)
+ * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
+
+
+3.0.7
+ * Fix legacy serialization of Thrift-generated non-compound range tombstones
+ when communicating with 2.x nodes (CASSANDRA-11930)
+ * Fix Directories instantiations where CFS.initialDirectories should be used (CASSANDRA-11849)
+ * Avoid referencing DatabaseDescriptor in AbstractType (CASSANDRA-11912)
+ * Fix sstables not being protected from removal during index build (CASSANDRA-11905)
+ * cqlsh: Suppress stack trace from Read/WriteFailures (CASSANDRA-11032)
+ * Remove unneeded code to repair index summaries that have
+ been improperly down-sampled (CASSANDRA-11127)
+ * Avoid WriteTimeoutExceptions during commit log replay due to materialized
+ view lock contention (CASSANDRA-11891)
+ * Prevent OOM failures on SSTable corruption, improve tests for corruption detection (CASSANDRA-9530)
+ * Use CFS.initialDirectories when clearing snapshots (CASSANDRA-11705)
+ * Allow compaction strategies to disable early open (CASSANDRA-11754)
+ * Refactor Materialized View code (CASSANDRA-11475)
+ * Update Java Driver (CASSANDRA-11615)
+Merged from 2.2:
* Persist local metadata earlier in startup sequence (CASSANDRA-11742)
* Run CommitLog tests with different compression settings (CASSANDRA-9039)
* cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/73a8341f/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/73a8341f/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/View.java
index 17062b4,e303801..99903fc
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@@ -179,52 -131,23 +179,63 @@@ public class Vie
}
/**
- * Returns the sstables that have any partition between {@code left} and {@code right}, when both bounds are taken inclusively.
- * The interval formed by {@code left} and {@code right} shouldn't wrap.
- */
- public List<SSTableReader> sstablesInBounds(RowPosition left, RowPosition right)
+ * Returns the sstables that have any partition between {@code left} and {@code right}, when both bounds are taken inclusively.
+ * The interval formed by {@code left} and {@code right} shouldn't wrap.
+ */
+ public Iterable<SSTableReader> sstablesInBounds(SSTableSet sstableSet, PartitionPosition left, PartitionPosition right)
{
- return sstablesInBounds(left, right, intervalTree);
+ assert !AbstractBounds.strictlyWrapsAround(left, right);
+
+ if (intervalTree.isEmpty())
+ return Collections.emptyList();
+
+ PartitionPosition stopInTree = right.isMinimum() ? intervalTree.max() : right;
+ return select(sstableSet, intervalTree.search(Interval.create(left, stopInTree)));
}
- public static List<SSTableReader> sstablesInBounds(RowPosition left, RowPosition right, SSTableIntervalTree intervalTree)
++ public static List<SSTableReader> sstablesInBounds(PartitionPosition left, PartitionPosition right, SSTableIntervalTree intervalTree)
+ {
+ assert !AbstractBounds.strictlyWrapsAround(left, right);
+
+ if (intervalTree.isEmpty())
+ return Collections.emptyList();
+
- RowPosition stopInTree = right.isMinimum() ? intervalTree.max() : right;
- return intervalTree.search(Interval.<RowPosition, SSTableReader>create(left, stopInTree));
++ PartitionPosition stopInTree = right.isMinimum() ? intervalTree.max() : right;
++ return intervalTree.search(Interval.create(left, stopInTree));
++ }
++
+ public static Function<View, Iterable<SSTableReader>> select(SSTableSet sstableSet)
+ {
+ return (view) -> view.sstables(sstableSet);
+ }
+
+ public static Function<View, Iterable<SSTableReader>> select(SSTableSet sstableSet, Predicate<SSTableReader> filter)
+ {
+ return (view) -> view.sstables(sstableSet, filter);
+ }
+
+ /**
+ * @return a ViewFragment containing the sstables and memtables that may need to be merged
+ * for the given @param key, according to the interval tree
+ */
+ public static Function<View, Iterable<SSTableReader>> select(SSTableSet sstableSet, DecoratedKey key)
+ {
+ assert sstableSet == SSTableSet.LIVE;
+ return (view) -> view.intervalTree.search(key);
+ }
+
+ /**
+ * @return a ViewFragment containing the sstables and memtables that may need to be merged
+ * for rows within @param rowBounds, inclusive, according to the interval tree.
+ */
+ public static Function<View, Iterable<SSTableReader>> select(SSTableSet sstableSet, AbstractBounds<PartitionPosition> rowBounds)
+ {
+ // Note that View.sstablesInBounds always includes it's bound while rowBounds may not. This is ok however
+ // because the fact we restrict the sstables returned by this function is an optimization in the first
+ // place and the returned sstables will (almost) never cover *exactly* rowBounds anyway. It's also
+ // *very* unlikely that a sstable is included *just* because we consider one of the bound inclusively
+ // instead of exclusively, so the performance impact is negligible in practice.
+ return (view) -> view.sstablesInBounds(sstableSet, rowBounds.left, rowBounds.right);
}
// METHODS TO CONSTRUCT FUNCTIONS FOR MODIFYING A VIEW:
http://git-wip-us.apache.org/repos/asf/cassandra/blob/73a8341f/src/java/org/apache/cassandra/streaming/StreamSession.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/streaming/StreamSession.java
index 2ed6ad1,f4c900e..d5c060e
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@@ -25,10 -25,12 +25,13 @@@ import java.util.*
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
+ import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
import com.google.common.collect.*;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+ import org.apache.cassandra.db.lifecycle.SSTableIntervalTree;
+ import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -320,30 -319,33 +324,24 @@@ public class StreamSession implements I
{
for (ColumnFamilyStore cfStore : stores)
{
- final List<Range<RowPosition>> keyRanges = new ArrayList<>(ranges.size());
+ final List<Range<PartitionPosition>> keyRanges = new ArrayList<>(ranges.size());
for (Range<Token> range : ranges)
keyRanges.add(Range.makeRowRange(range));
- refs.addAll(cfStore.selectAndReference(new Function<View, List<SSTableReader>>()
- {
- public List<SSTableReader> apply(View view)
+ refs.addAll(cfStore.selectAndReference(view -> {
+ Set<SSTableReader> sstables = Sets.newHashSet();
++ SSTableIntervalTree intervalTree = SSTableIntervalTree.build(view.sstables(SSTableSet.CANONICAL));
+ for (Range<PartitionPosition> keyRange : keyRanges)
{
- // keyRange excludes its start, while sstableInBounds is inclusive (of both start and end).
- // This is fine however, because keyRange has been created from a token range through Range.makeRowRange (see above).
- // And that later method uses the Token.maxKeyBound() method to creates the range, which return a "fake" key that
- // sort after all keys having the token. That "fake" key cannot however be equal to any real key, so that even
- // including keyRange.left will still exclude any key having the token of the original token range, and so we're
- // still actually selecting what we wanted.
- for (SSTableReader sstable : view.sstablesInBounds(SSTableSet.CANONICAL, keyRange.left, keyRange.right))
- SSTableIntervalTree intervalTree = SSTableIntervalTree.build(ColumnFamilyStore.CANONICAL_SSTABLES.apply(view));
- Set<SSTableReader> sstables = Sets.newHashSet();
- for (Range<RowPosition> keyRange : keyRanges)
++ for (SSTableReader sstable : View.sstablesInBounds(keyRange.left, keyRange.right, intervalTree))
{
- // sstableInBounds may contain early opened sstables
- // keyRange excludes its start, while sstableInBounds is inclusive (of both start and end).
- // This is fine however, because keyRange has been created from a token range through Range.makeRowRange (see above).
- // And that later method uses the Token.maxKeyBound() method to creates the range, which return a "fake" key that
- // sort after all keys having the token. That "fake" key cannot however be equal to any real key, so that even
- // including keyRange.left will still exclude any key having the token of the original token range, and so we're
- // still actually selecting what we wanted.
- for (SSTableReader sstable : View.sstablesInBounds(keyRange.left, keyRange.right, intervalTree))
- {
- if (!isIncremental || !sstable.isRepaired())
- sstables.add(sstable);
- }
+ if (!isIncremental || !sstable.isRepaired())
+ sstables.add(sstable);
}
-
- logger.debug("ViewFilter for {}/{} sstables", sstables.size(), view.sstables.size());
- return ImmutableList.copyOf(sstables);
}
+
+ if (logger.isDebugEnabled())
+ logger.debug("ViewFilter for {}/{} sstables", sstables.size(), Iterables.size(view.sstables(SSTableSet.CANONICAL)));
+ return sstables;
}).refs);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/73a8341f/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index 008df06,f50953a..18bc760
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@@ -20,48 -19,49 +20,42 @@@ package org.apache.cassandra.io.sstable
import java.io.File;
import java.io.IOException;
--import java.nio.ByteBuffer;
import java.util.*;
- import java.util.concurrent.ThreadLocalRandom;
+ import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
--import org.junit.After;
--import org.junit.AfterClass;
--import org.junit.BeforeClass;
import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Test;
--import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.Config;
+import org.apache.cassandra.UpdateBuilder;
- import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.KSMetaData;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.compaction.AbstractCompactedRow;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.SerializationHeader;
++import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionController;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.compaction.LazilyCompactedRow;
+import org.apache.cassandra.db.compaction.CompactionIterator;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.db.compaction.SSTableSplitter;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
- import org.apache.cassandra.db.lifecycle.View;
- import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
- import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.metrics.StorageMetrics;
- import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.notifications.INotification;
-import org.apache.cassandra.notifications.INotificationConsumer;
-import org.apache.cassandra.notifications.SSTableListChangedNotification;
-import org.apache.cassandra.service.StorageService;
+ import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
[07/10] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/05f8a008
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/05f8a008
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/05f8a008
Branch: refs/heads/trunk
Commit: 05f8a008f696d9624ec85176fa0e2a1ce06a1ad5
Parents: 593bbf5 72acbcd
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Jun 13 14:34:01 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Jun 13 15:00:08 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 4 ++
.../org/apache/cassandra/db/lifecycle/View.java | 5 ++
.../cassandra/streaming/StreamSession.java | 22 +++----
.../io/sstable/SSTableRewriterTest.java | 66 ++++++++++++++++++++
5 files changed, 86 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index d639d43,ebcc90c..491f72a
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,31 -1,7 +1,32 @@@
-2.1.15
+2.2.7
+ * StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
+ * Persist local metadata earlier in startup sequence (CASSANDRA-11742)
+ * Run CommitLog tests with different compression settings (CASSANDRA-9039)
+ * cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
+ * Avoid showing estimated key as -1 in tablestats (CASSANDRA-11587)
+ * Fix possible race condition in CommitLog.recover (CASSANDRA-11743)
+ * Enable client encryption in sstableloader with cli options (CASSANDRA-11708)
+ * Possible memory leak in NIODataInputStream (CASSANDRA-11867)
+ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+ * Add seconds to cqlsh tracing session duration (CASSANDRA-11753)
+ * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
+ * cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
+ * Exit JVM if JMX server fails to startup (CASSANDRA-11540)
+ * Produce a heap dump when exiting on OOM (CASSANDRA-9861)
+ * Avoid read repairing purgeable tombstones on range slices (CASSANDRA-11427)
+ * Restore ability to filter on clustering columns when using a 2i (CASSANDRA-11510)
+ * JSON datetime formatting needs timezone (CASSANDRA-11137)
+ * Fix is_dense recalculation for Thrift-updated tables (CASSANDRA-11502)
+ * Remove unnescessary file existence check during anticompaction (CASSANDRA-11660)
+ * Add missing files to debian packages (CASSANDRA-11642)
+ * Avoid calling Iterables::concat in loops during ModificationStatement::getFunctions (CASSANDRA-11621)
+ * cqlsh: COPY FROM should use regular inserts for single statement batches and
+ report errors correctly if workers processes crash on initialization (CASSANDRA-11474)
+ * Always close cluster with connection in CqlRecordWriter (CASSANDRA-11553)
+ * Fix slice queries on ordered COMPACT tables (CASSANDRA-10988)
+Merged from 2.1:
+ * Create interval tree over canonical sstables to avoid missing sstables during streaming (CASSANDRA-11886)
* cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
- * Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
* Backport CASSANDRA-11578 (CASSANDRA-11750)
* Clear out parent repair session if repair coordinator dies (CASSANDRA-11824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index e24917c,559ba0b..d3a5028
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@@ -1588,8 -1529,12 +1588,12 @@@ public class DatabaseDescripto
public static int getSSTablePreempiveOpenIntervalInMB()
{
- return conf.sstable_preemptive_open_interval_in_mb;
+ return FBUtilities.isWindows() ? -1 : conf.sstable_preemptive_open_interval_in_mb;
}
+ public static void setSSTablePreempiveOpenIntervalInMB(int mb)
+ {
+ conf.sstable_preemptive_open_interval_in_mb = mb;
+ }
public static boolean getTrickleFsync()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/View.java
index fba1627,0000000..e303801
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@@ -1,281 -1,0 +1,286 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.lifecycle;
+
+import java.util.*;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.*;
+
+import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.RowPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.Interval;
+
+import static com.google.common.base.Predicates.equalTo;
+import static com.google.common.base.Predicates.not;
+import static com.google.common.collect.ImmutableList.copyOf;
+import static com.google.common.collect.ImmutableList.of;
+import static com.google.common.collect.Iterables.all;
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.filter;
+import static java.util.Collections.singleton;
+import static org.apache.cassandra.db.lifecycle.Helpers.emptySet;
+import static org.apache.cassandra.db.lifecycle.Helpers.replace;
+
+/**
+ * An immutable structure holding the current memtable, the memtables pending
+ * flush, the sstables for a column family, and the sstables that are active
+ * in compaction (a subset of the sstables).
+ *
+ * Modifications to instances are all performed via a Function produced by the static methods in this class.
+ * These are composed as necessary and provided to the Tracker.apply() methods, which atomically reject or
+ * accept and apply the changes to the View.
+ *
+ */
+public class View
+{
+ /**
+ * ordinarily a list of size 1, but when preparing to flush will contain both the memtable we will flush
+ * and the new replacement memtable, until all outstanding write operations on the old table complete.
+ * The last item in the list is always the "current" memtable.
+ */
+ public final List<Memtable> liveMemtables;
+ /**
+ * contains all memtables that are no longer referenced for writing and are queued for / in the process of being
+ * flushed. In chronologically ascending order.
+ */
+ public final List<Memtable> flushingMemtables;
+ public final Set<SSTableReader> compacting;
+ public final Set<SSTableReader> sstables;
+ public final Set<SSTableReader> premature;
+ // we use a Map here so that we can easily perform identity checks as well as equality checks.
+ // When marking compacting, we now indicate if we expect the sstables to be present (by default we do),
+ // and we then check that not only are they all present in the live set, but that the exact instance present is
+ // the one we made our decision to compact against.
+ public final Map<SSTableReader, SSTableReader> sstablesMap;
+
+ public final SSTableIntervalTree intervalTree;
+
+ View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
+ {
+ assert liveMemtables != null;
+ assert flushingMemtables != null;
+ assert sstables != null;
+ assert compacting != null;
+ assert intervalTree != null;
+
+ this.liveMemtables = liveMemtables;
+ this.flushingMemtables = flushingMemtables;
+
+ this.sstablesMap = sstables;
+ this.sstables = sstablesMap.keySet();
+ this.compacting = compacting;
+ this.premature = premature;
+ this.intervalTree = intervalTree;
+ }
+
+ public Memtable getCurrentMemtable()
+ {
+ return liveMemtables.get(liveMemtables.size() - 1);
+ }
+
+ /**
+ * @return the active memtable and all the memtables that are pending flush.
+ */
+ public Iterable<Memtable> getAllMemtables()
+ {
+ return concat(flushingMemtables, liveMemtables);
+ }
+
+ public Sets.SetView<SSTableReader> nonCompactingSStables()
+ {
+ return Sets.difference(sstables, compacting);
+ }
+
+ public Iterable<SSTableReader> getUncompacting(Iterable<SSTableReader> candidates)
+ {
+ return filter(candidates, new Predicate<SSTableReader>()
+ {
+ public boolean apply(SSTableReader sstable)
+ {
+ return !compacting.contains(sstable);
+ }
+ });
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("View(pending_count=%d, sstables=%s, compacting=%s)", liveMemtables.size() + flushingMemtables.size() - 1, sstables, compacting);
+ }
+
+ /**
+ * Returns the sstables that have any partition between {@code left} and {@code right}, when both bounds are taken inclusively.
+ * The interval formed by {@code left} and {@code right} shouldn't wrap.
+ */
+ public List<SSTableReader> sstablesInBounds(RowPosition left, RowPosition right)
+ {
++ return sstablesInBounds(left, right, intervalTree);
++ }
++
++ public static List<SSTableReader> sstablesInBounds(RowPosition left, RowPosition right, SSTableIntervalTree intervalTree)
++ {
+ assert !AbstractBounds.strictlyWrapsAround(left, right);
+
+ if (intervalTree.isEmpty())
+ return Collections.emptyList();
+
+ RowPosition stopInTree = right.isMinimum() ? intervalTree.max() : right;
+ return intervalTree.search(Interval.<RowPosition, SSTableReader>create(left, stopInTree));
+ }
+
+ // METHODS TO CONSTRUCT FUNCTIONS FOR MODIFYING A VIEW:
+
+ // return a function to un/mark the provided readers compacting in a view
+ static Function<View, View> updateCompacting(final Set<SSTableReader> unmark, final Iterable<SSTableReader> mark)
+ {
+ if (unmark.isEmpty() && Iterables.isEmpty(mark))
+ return Functions.identity();
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ assert all(mark, Helpers.idIn(view.sstablesMap));
+ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap,
+ replace(view.compacting, unmark, mark),
+ view.premature, view.intervalTree);
+ }
+ };
+ }
+
+ // construct a predicate to reject views that do not permit us to mark these readers compacting;
+ // i.e. one of them is either already compacting, has been compacted, or has been replaced
+ static Predicate<View> permitCompacting(final Iterable<SSTableReader> readers)
+ {
+ return new Predicate<View>()
+ {
+ public boolean apply(View view)
+ {
+ for (SSTableReader reader : readers)
+ if (view.compacting.contains(reader) || view.sstablesMap.get(reader) != reader || reader.isMarkedCompacted() || view.premature.contains(reader))
+ return false;
+ return true;
+ }
+ };
+ }
+
+ // construct a function to change the liveset in a Snapshot
+ static Function<View, View> updateLiveSet(final Set<SSTableReader> remove, final Iterable<SSTableReader> add)
+ {
+ if (remove.isEmpty() && Iterables.isEmpty(add))
+ return Functions.identity();
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, remove, add);
+ return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting, view.premature,
+ SSTableIntervalTree.build(sstableMap.keySet()));
+ }
+ };
+ }
+
+ // called prior to initiating flush: add newMemtable to liveMemtables, making it the latest memtable
+ static Function<View, View> switchMemtable(final Memtable newMemtable)
+ {
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ List<Memtable> newLive = ImmutableList.<Memtable>builder().addAll(view.liveMemtables).add(newMemtable).build();
+ assert newLive.size() == view.liveMemtables.size() + 1;
+ return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
+ }
+ };
+ }
+
+ // called before flush: move toFlush from liveMemtables to flushingMemtables
+ static Function<View, View> markFlushing(final Memtable toFlush)
+ {
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ List<Memtable> live = view.liveMemtables, flushing = view.flushingMemtables;
+ List<Memtable> newLive = copyOf(filter(live, not(equalTo(toFlush))));
+ List<Memtable> newFlushing = copyOf(concat(filter(flushing, lessThan(toFlush)),
+ of(toFlush),
+ filter(flushing, not(lessThan(toFlush)))));
+ assert newLive.size() == live.size() - 1;
+ assert newFlushing.size() == flushing.size() + 1;
+ return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
+ }
+ };
+ }
+
+ // called after flush: removes memtable from flushingMemtables, and inserts flushed into the live sstable set
+ static Function<View, View> replaceFlushed(final Memtable memtable, final SSTableReader flushed)
+ {
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ List<Memtable> flushingMemtables = copyOf(filter(view.flushingMemtables, not(equalTo(memtable))));
+ assert flushingMemtables.size() == view.flushingMemtables.size() - 1;
+
+ if (flushed == null)
+ return new View(view.liveMemtables, flushingMemtables, view.sstablesMap,
+ view.compacting, view.premature, view.intervalTree);
+
+ Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), singleton(flushed));
+ Set<SSTableReader> compacting = replace(view.compacting, emptySet(), singleton(flushed));
+ Set<SSTableReader> premature = replace(view.premature, emptySet(), singleton(flushed));
+ return new View(view.liveMemtables, flushingMemtables, sstableMap, compacting, premature,
+ SSTableIntervalTree.build(sstableMap.keySet()));
+ }
+ };
+ }
+
+ static Function<View, View> permitCompactionOfFlushed(final SSTableReader reader)
+ {
+ return new Function<View, View>()
+ {
+
+ @Nullable
+ public View apply(View view)
+ {
+ Set<SSTableReader> premature = ImmutableSet.copyOf(filter(view.premature, not(equalTo(reader))));
+ Set<SSTableReader> compacting = ImmutableSet.copyOf(filter(view.compacting, not(equalTo(reader))));
+ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compacting, premature, view.intervalTree);
+ }
+ };
+ }
+
+
+ private static <T extends Comparable<T>> Predicate<T> lessThan(final T lessThan)
+ {
+ return new Predicate<T>()
+ {
+ public boolean apply(T t)
+ {
+ return t.compareTo(lessThan) < 0;
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/src/java/org/apache/cassandra/streaming/StreamSession.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/streaming/StreamSession.java
index f14b94c,273631c..f4c900e
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@@ -25,18 -25,22 +25,22 @@@ import java.util.*
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
-import javax.annotation.Nullable;
-
+ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.collect.*;
+
++import org.apache.cassandra.db.lifecycle.SSTableIntervalTree;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DataTracker;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowPosition;
+ import org.apache.cassandra.dht.AbstractBounds;
+ import org.apache.cassandra.dht.Bounds;
-import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.*;
@@@ -279,7 -270,7 +283,7 @@@ public class StreamSession implements I
flushSSTables(stores);
List<Range<Token>> normalizedRanges = Range.normalize(ranges);
- List<SSTableStreamingSections> sections = getSSTableSectionsForRanges(normalizedRanges, stores, repairedAt);
- List<SSTableStreamingSections> sections = getSSTableSectionsForRanges(normalizedRanges, stores, repairedAt, repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE);
++ List<SSTableStreamingSections> sections = getSSTableSectionsForRanges(normalizedRanges, stores, repairedAt, isIncremental);
try
{
addTransferFiles(sections);
@@@ -314,33 -306,21 +319,26 @@@
{
for (ColumnFamilyStore cfStore : stores)
{
- final List<AbstractBounds<RowPosition>> rowBoundsList = new ArrayList<>(ranges.size());
- final IPartitioner partitioner = cfStore.partitioner;
+ final List<Range<RowPosition>> keyRanges = new ArrayList<>(ranges.size());
for (Range<Token> range : ranges)
- rowBoundsList.add(range.toRowBounds());
- refs.addAll(cfStore.selectAndReference(new Function<DataTracker.View, List<SSTableReader>>()
+ keyRanges.add(Range.makeRowRange(range));
+ refs.addAll(cfStore.selectAndReference(new Function<View, List<SSTableReader>>()
{
- public List<SSTableReader> apply(DataTracker.View view)
+ public List<SSTableReader> apply(View view)
{
- Map<SSTableReader, SSTableReader> permittedInstances = new HashMap<>();
- for (SSTableReader reader : ColumnFamilyStore.CANONICAL_SSTABLES.apply(view))
- permittedInstances.put(reader, reader);
-
- DataTracker.SSTableIntervalTree intervalTree = DataTracker.buildIntervalTree(ColumnFamilyStore.CANONICAL_SSTABLES.apply(view));
++ SSTableIntervalTree intervalTree = SSTableIntervalTree.build(ColumnFamilyStore.CANONICAL_SSTABLES.apply(view));
Set<SSTableReader> sstables = Sets.newHashSet();
- for (AbstractBounds<RowPosition> rowBounds : rowBoundsList)
+ for (Range<RowPosition> keyRange : keyRanges)
{
- for (SSTableReader sstable : DataTracker.View.sstablesInBounds(rowBounds, intervalTree, partitioner))
+ // keyRange excludes its start, while sstableInBounds is inclusive (of both start and end).
+ // This is fine however, because keyRange has been created from a token range through Range.makeRowRange (see above).
+ // And that later method uses the Token.maxKeyBound() method to creates the range, which return a "fake" key that
+ // sort after all keys having the token. That "fake" key cannot however be equal to any real key, so that even
+ // including keyRange.left will still exclude any key having the token of the original token range, and so we're
+ // still actually selecting what we wanted.
- for (SSTableReader sstable : view.sstablesInBounds(keyRange.left, keyRange.right))
++ for (SSTableReader sstable : View.sstablesInBounds(keyRange.left, keyRange.right, intervalTree))
{
- // sstableInBounds may contain early opened sstables
- if (isIncremental && sstable.isRepaired())
- continue;
- sstable = permittedInstances.get(sstable);
- if (sstable != null)
+ if (!isIncremental || !sstable.isRepaired())
sstables.add(sstable);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index c2cc6e3,1fb28f5..f50953a
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@@ -21,15 -21,11 +21,18 @@@ import java.io.File
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
+ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
++import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
@@@ -41,25 -35,24 +44,29 @@@ import org.apache.cassandra.db.*
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.LazilyCompactedRow;
import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.db.compaction.SSTableSplitter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.metrics.StorageMetrics;
+ import org.apache.cassandra.notifications.INotification;
+ import org.apache.cassandra.notifications.INotificationConsumer;
+ import org.apache.cassandra.notifications.SSTableListChangedNotification;
import org.apache.cassandra.service.StorageService;
+ import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
public class SSTableRewriterTest extends SchemaLoader
{
@@@ -807,10 -769,76 +814,69 @@@
}
}
}
- writer.abort();
- cfs.getDataTracker().unmarkCompacting(sstables);
- truncate(cfs);
+ truncateCF();
+ validateCFS(cfs);
}
+ @Test
+ public void testSSTableSectionsForRanges() throws IOException, InterruptedException, ExecutionException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+ truncate(cfs);
+
+ cfs.addSSTable(writeFile(cfs, 1000));
+
+ Collection<SSTableReader> allSSTables = cfs.getSSTables();
+ assertEquals(1, allSSTables.size());
+ final Token firstToken = allSSTables.iterator().next().first.getToken();
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(1);
+
+ List<StreamSession.SSTableStreamingSections> sectionsBeforeRewrite = StreamSession.getSSTableSectionsForRanges(
+ Collections.singleton(new Range<Token>(firstToken, firstToken)),
+ Collections.singleton(cfs), 0L, false);
+ assertEquals(1, sectionsBeforeRewrite.size());
+ for (StreamSession.SSTableStreamingSections section : sectionsBeforeRewrite)
+ section.ref.release();
+ final AtomicInteger checkCount = new AtomicInteger();
+ // needed since we get notified when compaction is done as well - we can't get sections for ranges for obsoleted sstables
- INotificationConsumer consumer = new INotificationConsumer()
++ final AtomicBoolean done = new AtomicBoolean(false);
++ final AtomicBoolean failed = new AtomicBoolean(false);
++ Runnable r = new Runnable()
++ {
++ public void run()
++ {
++ while (!done.get())
+ {
- public void handleNotification(INotification notification, Object sender)
- {
- if (notification instanceof SSTableListChangedNotification)
- {
- Collection<SSTableReader> added = ((SSTableListChangedNotification) notification).added;
- Collection<SSTableReader> removed = ((SSTableListChangedNotification) notification).removed;
- // note that we need to check if added.equals(removed) because once the compaction is done the old sstable will have
- // selfRef().globalCount() == 0 and we cant get the SectionsForRanges then. During incremental opening we always add and remove the same
- // sstable (note that the sstables are x.equal(y) but not x == y since the new one will be a new instance with a moved starting point
- // In this case we must avoid trying to call getSSTableSectionsForRanges since we are in the notification
- // method and trying to reference an sstable with globalcount == 0 puts it into a loop, and this blocks the tracker from removing the
- // unreferenced sstable.
- if (added.isEmpty() || !added.iterator().next().getColumnFamilyName().equals(cfs.getColumnFamilyName()) || !added.equals(removed))
- return;
-
- // at no point must the rewrite process hide
- // sections returned by getSSTableSectionsForRanges
- Set<Range<Token>> range = Collections.singleton(new Range<Token>(firstToken, firstToken));
- List<StreamSession.SSTableStreamingSections> sections = StreamSession.getSSTableSectionsForRanges(range, Collections.singleton(cfs), 0L, false);
- assertEquals(1, sections.size());
- for (StreamSession.SSTableStreamingSections section : sections)
- section.ref.release();
- checkCount.incrementAndGet();
- }
- }
- };
- cfs.getDataTracker().subscribe(consumer);
++ Set<Range<Token>> range = Collections.singleton(new Range<Token>(firstToken, firstToken));
++ List<StreamSession.SSTableStreamingSections> sections = StreamSession.getSSTableSectionsForRanges(range, Collections.singleton(cfs), 0L, false);
++ if (sections.size() != 1)
++ failed.set(true);
++ for (StreamSession.SSTableStreamingSections section : sections)
++ section.ref.release();
++ checkCount.incrementAndGet();
++ Uninterruptibles.sleepUninterruptibly(5, TimeUnit.MILLISECONDS);
++ }
++ }
++ };
++ Thread t = new Thread(r);
+ try
+ {
++ t.start();
+ cfs.forceMajorCompaction();
+ // reset
+ }
+ finally
+ {
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(50);
- cfs.getDataTracker().unsubscribe(consumer);
++ done.set(true);
++ t.join(20);
+ }
++ assertFalse(failed.get());
+ assertTrue(checkCount.get() >= 2);
+ truncate(cfs);
+ }
+
/**
* emulates anticompaction - writing from one source sstable to two new sstables
*
[05/10] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/05f8a008
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/05f8a008
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/05f8a008
Branch: refs/heads/cassandra-3.0
Commit: 05f8a008f696d9624ec85176fa0e2a1ce06a1ad5
Parents: 593bbf5 72acbcd
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Jun 13 14:34:01 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Jun 13 15:00:08 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 4 ++
.../org/apache/cassandra/db/lifecycle/View.java | 5 ++
.../cassandra/streaming/StreamSession.java | 22 +++----
.../io/sstable/SSTableRewriterTest.java | 66 ++++++++++++++++++++
5 files changed, 86 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index d639d43,ebcc90c..491f72a
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,31 -1,7 +1,32 @@@
-2.1.15
+2.2.7
+ * StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
+ * Persist local metadata earlier in startup sequence (CASSANDRA-11742)
+ * Run CommitLog tests with different compression settings (CASSANDRA-9039)
+ * cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
+ * Avoid showing estimated key as -1 in tablestats (CASSANDRA-11587)
+ * Fix possible race condition in CommitLog.recover (CASSANDRA-11743)
+ * Enable client encryption in sstableloader with cli options (CASSANDRA-11708)
+ * Possible memory leak in NIODataInputStream (CASSANDRA-11867)
+ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+ * Add seconds to cqlsh tracing session duration (CASSANDRA-11753)
+ * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
+ * cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
+ * Exit JVM if JMX server fails to startup (CASSANDRA-11540)
+ * Produce a heap dump when exiting on OOM (CASSANDRA-9861)
+ * Avoid read repairing purgeable tombstones on range slices (CASSANDRA-11427)
+ * Restore ability to filter on clustering columns when using a 2i (CASSANDRA-11510)
+ * JSON datetime formatting needs timezone (CASSANDRA-11137)
+ * Fix is_dense recalculation for Thrift-updated tables (CASSANDRA-11502)
+ * Remove unnescessary file existence check during anticompaction (CASSANDRA-11660)
+ * Add missing files to debian packages (CASSANDRA-11642)
+ * Avoid calling Iterables::concat in loops during ModificationStatement::getFunctions (CASSANDRA-11621)
+ * cqlsh: COPY FROM should use regular inserts for single statement batches and
+ report errors correctly if workers processes crash on initialization (CASSANDRA-11474)
+ * Always close cluster with connection in CqlRecordWriter (CASSANDRA-11553)
+ * Fix slice queries on ordered COMPACT tables (CASSANDRA-10988)
+Merged from 2.1:
+ * Create interval tree over canonical sstables to avoid missing sstables during streaming (CASSANDRA-11886)
* cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
- * Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
* Backport CASSANDRA-11578 (CASSANDRA-11750)
* Clear out parent repair session if repair coordinator dies (CASSANDRA-11824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index e24917c,559ba0b..d3a5028
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@@ -1588,8 -1529,12 +1588,12 @@@ public class DatabaseDescripto
public static int getSSTablePreempiveOpenIntervalInMB()
{
- return conf.sstable_preemptive_open_interval_in_mb;
+ return FBUtilities.isWindows() ? -1 : conf.sstable_preemptive_open_interval_in_mb;
}
+ public static void setSSTablePreempiveOpenIntervalInMB(int mb)
+ {
+ conf.sstable_preemptive_open_interval_in_mb = mb;
+ }
public static boolean getTrickleFsync()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/View.java
index fba1627,0000000..e303801
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@@ -1,281 -1,0 +1,286 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.lifecycle;
+
+import java.util.*;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.*;
+
+import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.RowPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.Interval;
+
+import static com.google.common.base.Predicates.equalTo;
+import static com.google.common.base.Predicates.not;
+import static com.google.common.collect.ImmutableList.copyOf;
+import static com.google.common.collect.ImmutableList.of;
+import static com.google.common.collect.Iterables.all;
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.filter;
+import static java.util.Collections.singleton;
+import static org.apache.cassandra.db.lifecycle.Helpers.emptySet;
+import static org.apache.cassandra.db.lifecycle.Helpers.replace;
+
+/**
+ * An immutable structure holding the current memtable, the memtables pending
+ * flush, the sstables for a column family, and the sstables that are active
+ * in compaction (a subset of the sstables).
+ *
+ * Modifications to instances are all performed via a Function produced by the static methods in this class.
+ * These are composed as necessary and provided to the Tracker.apply() methods, which atomically reject or
+ * accept and apply the changes to the View.
+ *
+ */
+public class View
+{
+ /**
+ * ordinarily a list of size 1, but when preparing to flush will contain both the memtable we will flush
+ * and the new replacement memtable, until all outstanding write operations on the old table complete.
+ * The last item in the list is always the "current" memtable.
+ */
+ public final List<Memtable> liveMemtables;
+ /**
+ * contains all memtables that are no longer referenced for writing and are queued for / in the process of being
+ * flushed. In chronologically ascending order.
+ */
+ public final List<Memtable> flushingMemtables;
+ public final Set<SSTableReader> compacting;
+ public final Set<SSTableReader> sstables;
+ public final Set<SSTableReader> premature;
+ // we use a Map here so that we can easily perform identity checks as well as equality checks.
+ // When marking compacting, we now indicate if we expect the sstables to be present (by default we do),
+ // and we then check that not only are they all present in the live set, but that the exact instance present is
+ // the one we made our decision to compact against.
+ public final Map<SSTableReader, SSTableReader> sstablesMap;
+
+ public final SSTableIntervalTree intervalTree;
+
+ View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
+ {
+ assert liveMemtables != null;
+ assert flushingMemtables != null;
+ assert sstables != null;
+ assert compacting != null;
+ assert intervalTree != null;
+
+ this.liveMemtables = liveMemtables;
+ this.flushingMemtables = flushingMemtables;
+
+ this.sstablesMap = sstables;
+ this.sstables = sstablesMap.keySet();
+ this.compacting = compacting;
+ this.premature = premature;
+ this.intervalTree = intervalTree;
+ }
+
+ public Memtable getCurrentMemtable()
+ {
+ return liveMemtables.get(liveMemtables.size() - 1);
+ }
+
+ /**
+ * @return the active memtable and all the memtables that are pending flush.
+ */
+ public Iterable<Memtable> getAllMemtables()
+ {
+ return concat(flushingMemtables, liveMemtables);
+ }
+
+ public Sets.SetView<SSTableReader> nonCompactingSStables()
+ {
+ return Sets.difference(sstables, compacting);
+ }
+
+ public Iterable<SSTableReader> getUncompacting(Iterable<SSTableReader> candidates)
+ {
+ return filter(candidates, new Predicate<SSTableReader>()
+ {
+ public boolean apply(SSTableReader sstable)
+ {
+ return !compacting.contains(sstable);
+ }
+ });
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("View(pending_count=%d, sstables=%s, compacting=%s)", liveMemtables.size() + flushingMemtables.size() - 1, sstables, compacting);
+ }
+
+ /**
+ * Returns the sstables that have any partition between {@code left} and {@code right}, when both bounds are taken inclusively.
+ * The interval formed by {@code left} and {@code right} shouldn't wrap.
+ */
+ public List<SSTableReader> sstablesInBounds(RowPosition left, RowPosition right)
+ {
++ return sstablesInBounds(left, right, intervalTree);
++ }
++
++ public static List<SSTableReader> sstablesInBounds(RowPosition left, RowPosition right, SSTableIntervalTree intervalTree)
++ {
+ assert !AbstractBounds.strictlyWrapsAround(left, right);
+
+ if (intervalTree.isEmpty())
+ return Collections.emptyList();
+
+ RowPosition stopInTree = right.isMinimum() ? intervalTree.max() : right;
+ return intervalTree.search(Interval.<RowPosition, SSTableReader>create(left, stopInTree));
+ }
+
+ // METHODS TO CONSTRUCT FUNCTIONS FOR MODIFYING A VIEW:
+
+ // return a function to un/mark the provided readers compacting in a view
+ static Function<View, View> updateCompacting(final Set<SSTableReader> unmark, final Iterable<SSTableReader> mark)
+ {
+ if (unmark.isEmpty() && Iterables.isEmpty(mark))
+ return Functions.identity();
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ assert all(mark, Helpers.idIn(view.sstablesMap));
+ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap,
+ replace(view.compacting, unmark, mark),
+ view.premature, view.intervalTree);
+ }
+ };
+ }
+
+ // construct a predicate to reject views that do not permit us to mark these readers compacting;
+ // i.e. one of them is either already compacting, has been compacted, or has been replaced
+ static Predicate<View> permitCompacting(final Iterable<SSTableReader> readers)
+ {
+ return new Predicate<View>()
+ {
+ public boolean apply(View view)
+ {
+ for (SSTableReader reader : readers)
+ if (view.compacting.contains(reader) || view.sstablesMap.get(reader) != reader || reader.isMarkedCompacted() || view.premature.contains(reader))
+ return false;
+ return true;
+ }
+ };
+ }
+
+ // construct a function to change the liveset in a Snapshot
+ static Function<View, View> updateLiveSet(final Set<SSTableReader> remove, final Iterable<SSTableReader> add)
+ {
+ if (remove.isEmpty() && Iterables.isEmpty(add))
+ return Functions.identity();
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, remove, add);
+ return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting, view.premature,
+ SSTableIntervalTree.build(sstableMap.keySet()));
+ }
+ };
+ }
+
+ // called prior to initiating flush: add newMemtable to liveMemtables, making it the latest memtable
+ static Function<View, View> switchMemtable(final Memtable newMemtable)
+ {
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ List<Memtable> newLive = ImmutableList.<Memtable>builder().addAll(view.liveMemtables).add(newMemtable).build();
+ assert newLive.size() == view.liveMemtables.size() + 1;
+ return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
+ }
+ };
+ }
+
+ // called before flush: move toFlush from liveMemtables to flushingMemtables
+ static Function<View, View> markFlushing(final Memtable toFlush)
+ {
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ List<Memtable> live = view.liveMemtables, flushing = view.flushingMemtables;
+ List<Memtable> newLive = copyOf(filter(live, not(equalTo(toFlush))));
+ List<Memtable> newFlushing = copyOf(concat(filter(flushing, lessThan(toFlush)),
+ of(toFlush),
+ filter(flushing, not(lessThan(toFlush)))));
+ assert newLive.size() == live.size() - 1;
+ assert newFlushing.size() == flushing.size() + 1;
+ return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
+ }
+ };
+ }
+
+ // called after flush: removes memtable from flushingMemtables, and inserts flushed into the live sstable set
+ static Function<View, View> replaceFlushed(final Memtable memtable, final SSTableReader flushed)
+ {
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ List<Memtable> flushingMemtables = copyOf(filter(view.flushingMemtables, not(equalTo(memtable))));
+ assert flushingMemtables.size() == view.flushingMemtables.size() - 1;
+
+ if (flushed == null)
+ return new View(view.liveMemtables, flushingMemtables, view.sstablesMap,
+ view.compacting, view.premature, view.intervalTree);
+
+ Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), singleton(flushed));
+ Set<SSTableReader> compacting = replace(view.compacting, emptySet(), singleton(flushed));
+ Set<SSTableReader> premature = replace(view.premature, emptySet(), singleton(flushed));
+ return new View(view.liveMemtables, flushingMemtables, sstableMap, compacting, premature,
+ SSTableIntervalTree.build(sstableMap.keySet()));
+ }
+ };
+ }
+
+ static Function<View, View> permitCompactionOfFlushed(final SSTableReader reader)
+ {
+ return new Function<View, View>()
+ {
+
+ @Nullable
+ public View apply(View view)
+ {
+ Set<SSTableReader> premature = ImmutableSet.copyOf(filter(view.premature, not(equalTo(reader))));
+ Set<SSTableReader> compacting = ImmutableSet.copyOf(filter(view.compacting, not(equalTo(reader))));
+ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compacting, premature, view.intervalTree);
+ }
+ };
+ }
+
+
+ private static <T extends Comparable<T>> Predicate<T> lessThan(final T lessThan)
+ {
+ return new Predicate<T>()
+ {
+ public boolean apply(T t)
+ {
+ return t.compareTo(lessThan) < 0;
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/src/java/org/apache/cassandra/streaming/StreamSession.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/streaming/StreamSession.java
index f14b94c,273631c..f4c900e
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@@ -25,18 -25,22 +25,22 @@@ import java.util.*
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
-import javax.annotation.Nullable;
-
+ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.collect.*;
+
++import org.apache.cassandra.db.lifecycle.SSTableIntervalTree;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DataTracker;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowPosition;
+ import org.apache.cassandra.dht.AbstractBounds;
+ import org.apache.cassandra.dht.Bounds;
-import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.*;
@@@ -279,7 -270,7 +283,7 @@@ public class StreamSession implements I
flushSSTables(stores);
List<Range<Token>> normalizedRanges = Range.normalize(ranges);
- List<SSTableStreamingSections> sections = getSSTableSectionsForRanges(normalizedRanges, stores, repairedAt);
- List<SSTableStreamingSections> sections = getSSTableSectionsForRanges(normalizedRanges, stores, repairedAt, repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE);
++ List<SSTableStreamingSections> sections = getSSTableSectionsForRanges(normalizedRanges, stores, repairedAt, isIncremental);
try
{
addTransferFiles(sections);
@@@ -314,33 -306,21 +319,26 @@@
{
for (ColumnFamilyStore cfStore : stores)
{
- final List<AbstractBounds<RowPosition>> rowBoundsList = new ArrayList<>(ranges.size());
- final IPartitioner partitioner = cfStore.partitioner;
+ final List<Range<RowPosition>> keyRanges = new ArrayList<>(ranges.size());
for (Range<Token> range : ranges)
- rowBoundsList.add(range.toRowBounds());
- refs.addAll(cfStore.selectAndReference(new Function<DataTracker.View, List<SSTableReader>>()
+ keyRanges.add(Range.makeRowRange(range));
+ refs.addAll(cfStore.selectAndReference(new Function<View, List<SSTableReader>>()
{
- public List<SSTableReader> apply(DataTracker.View view)
+ public List<SSTableReader> apply(View view)
{
- Map<SSTableReader, SSTableReader> permittedInstances = new HashMap<>();
- for (SSTableReader reader : ColumnFamilyStore.CANONICAL_SSTABLES.apply(view))
- permittedInstances.put(reader, reader);
-
- DataTracker.SSTableIntervalTree intervalTree = DataTracker.buildIntervalTree(ColumnFamilyStore.CANONICAL_SSTABLES.apply(view));
++ SSTableIntervalTree intervalTree = SSTableIntervalTree.build(ColumnFamilyStore.CANONICAL_SSTABLES.apply(view));
Set<SSTableReader> sstables = Sets.newHashSet();
- for (AbstractBounds<RowPosition> rowBounds : rowBoundsList)
+ for (Range<RowPosition> keyRange : keyRanges)
{
- for (SSTableReader sstable : DataTracker.View.sstablesInBounds(rowBounds, intervalTree, partitioner))
+ // keyRange excludes its start, while sstableInBounds is inclusive (of both start and end).
+ // This is fine however, because keyRange has been created from a token range through Range.makeRowRange (see above).
+ // And that later method uses the Token.maxKeyBound() method to creates the range, which return a "fake" key that
+ // sort after all keys having the token. That "fake" key cannot however be equal to any real key, so that even
+ // including keyRange.left will still exclude any key having the token of the original token range, and so we're
+ // still actually selecting what we wanted.
- for (SSTableReader sstable : view.sstablesInBounds(keyRange.left, keyRange.right))
++ for (SSTableReader sstable : View.sstablesInBounds(keyRange.left, keyRange.right, intervalTree))
{
- // sstableInBounds may contain early opened sstables
- if (isIncremental && sstable.isRepaired())
- continue;
- sstable = permittedInstances.get(sstable);
- if (sstable != null)
+ if (!isIncremental || !sstable.isRepaired())
sstables.add(sstable);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index c2cc6e3,1fb28f5..f50953a
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@@ -21,15 -21,11 +21,18 @@@ import java.io.File
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
+ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
++import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
@@@ -41,25 -35,24 +44,29 @@@ import org.apache.cassandra.db.*
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.LazilyCompactedRow;
import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.db.compaction.SSTableSplitter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.metrics.StorageMetrics;
+ import org.apache.cassandra.notifications.INotification;
+ import org.apache.cassandra.notifications.INotificationConsumer;
+ import org.apache.cassandra.notifications.SSTableListChangedNotification;
import org.apache.cassandra.service.StorageService;
+ import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
public class SSTableRewriterTest extends SchemaLoader
{
@@@ -807,10 -769,76 +814,69 @@@
}
}
}
- writer.abort();
- cfs.getDataTracker().unmarkCompacting(sstables);
- truncate(cfs);
+ truncateCF();
+ validateCFS(cfs);
}
+ @Test
+ public void testSSTableSectionsForRanges() throws IOException, InterruptedException, ExecutionException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+ truncate(cfs);
+
+ cfs.addSSTable(writeFile(cfs, 1000));
+
+ Collection<SSTableReader> allSSTables = cfs.getSSTables();
+ assertEquals(1, allSSTables.size());
+ final Token firstToken = allSSTables.iterator().next().first.getToken();
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(1);
+
+ List<StreamSession.SSTableStreamingSections> sectionsBeforeRewrite = StreamSession.getSSTableSectionsForRanges(
+ Collections.singleton(new Range<Token>(firstToken, firstToken)),
+ Collections.singleton(cfs), 0L, false);
+ assertEquals(1, sectionsBeforeRewrite.size());
+ for (StreamSession.SSTableStreamingSections section : sectionsBeforeRewrite)
+ section.ref.release();
+ final AtomicInteger checkCount = new AtomicInteger();
+ // needed since we get notified when compaction is done as well - we can't get sections for ranges for obsoleted sstables
- INotificationConsumer consumer = new INotificationConsumer()
++ final AtomicBoolean done = new AtomicBoolean(false);
++ final AtomicBoolean failed = new AtomicBoolean(false);
++ Runnable r = new Runnable()
++ {
++ public void run()
++ {
++ while (!done.get())
+ {
- public void handleNotification(INotification notification, Object sender)
- {
- if (notification instanceof SSTableListChangedNotification)
- {
- Collection<SSTableReader> added = ((SSTableListChangedNotification) notification).added;
- Collection<SSTableReader> removed = ((SSTableListChangedNotification) notification).removed;
- // note that we need to check if added.equals(removed) because once the compaction is done the old sstable will have
- // selfRef().globalCount() == 0 and we cant get the SectionsForRanges then. During incremental opening we always add and remove the same
- // sstable (note that the sstables are x.equal(y) but not x == y since the new one will be a new instance with a moved starting point
- // In this case we must avoid trying to call getSSTableSectionsForRanges since we are in the notification
- // method and trying to reference an sstable with globalcount == 0 puts it into a loop, and this blocks the tracker from removing the
- // unreferenced sstable.
- if (added.isEmpty() || !added.iterator().next().getColumnFamilyName().equals(cfs.getColumnFamilyName()) || !added.equals(removed))
- return;
-
- // at no point must the rewrite process hide
- // sections returned by getSSTableSectionsForRanges
- Set<Range<Token>> range = Collections.singleton(new Range<Token>(firstToken, firstToken));
- List<StreamSession.SSTableStreamingSections> sections = StreamSession.getSSTableSectionsForRanges(range, Collections.singleton(cfs), 0L, false);
- assertEquals(1, sections.size());
- for (StreamSession.SSTableStreamingSections section : sections)
- section.ref.release();
- checkCount.incrementAndGet();
- }
- }
- };
- cfs.getDataTracker().subscribe(consumer);
++ Set<Range<Token>> range = Collections.singleton(new Range<Token>(firstToken, firstToken));
++ List<StreamSession.SSTableStreamingSections> sections = StreamSession.getSSTableSectionsForRanges(range, Collections.singleton(cfs), 0L, false);
++ if (sections.size() != 1)
++ failed.set(true);
++ for (StreamSession.SSTableStreamingSections section : sections)
++ section.ref.release();
++ checkCount.incrementAndGet();
++ Uninterruptibles.sleepUninterruptibly(5, TimeUnit.MILLISECONDS);
++ }
++ }
++ };
++ Thread t = new Thread(r);
+ try
+ {
++ t.start();
+ cfs.forceMajorCompaction();
+ // reset
+ }
+ finally
+ {
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(50);
- cfs.getDataTracker().unsubscribe(consumer);
++ done.set(true);
++ t.join(20);
+ }
++ assertFalse(failed.get());
+ assertTrue(checkCount.get() >= 2);
+ truncate(cfs);
+ }
+
/**
* emulates anticompaction - writing from one source sstable to two new sstables
*
[06/10] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/05f8a008
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/05f8a008
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/05f8a008
Branch: refs/heads/cassandra-2.2
Commit: 05f8a008f696d9624ec85176fa0e2a1ce06a1ad5
Parents: 593bbf5 72acbcd
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Jun 13 14:34:01 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Jun 13 15:00:08 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 4 ++
.../org/apache/cassandra/db/lifecycle/View.java | 5 ++
.../cassandra/streaming/StreamSession.java | 22 +++----
.../io/sstable/SSTableRewriterTest.java | 66 ++++++++++++++++++++
5 files changed, 86 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index d639d43,ebcc90c..491f72a
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,31 -1,7 +1,32 @@@
-2.1.15
+2.2.7
+ * StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
+ * Persist local metadata earlier in startup sequence (CASSANDRA-11742)
+ * Run CommitLog tests with different compression settings (CASSANDRA-9039)
+ * cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
+ * Avoid showing estimated key as -1 in tablestats (CASSANDRA-11587)
+ * Fix possible race condition in CommitLog.recover (CASSANDRA-11743)
+ * Enable client encryption in sstableloader with cli options (CASSANDRA-11708)
+ * Possible memory leak in NIODataInputStream (CASSANDRA-11867)
+ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+ * Add seconds to cqlsh tracing session duration (CASSANDRA-11753)
+ * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
+ * cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
+ * Exit JVM if JMX server fails to startup (CASSANDRA-11540)
+ * Produce a heap dump when exiting on OOM (CASSANDRA-9861)
+ * Avoid read repairing purgeable tombstones on range slices (CASSANDRA-11427)
+ * Restore ability to filter on clustering columns when using a 2i (CASSANDRA-11510)
+ * JSON datetime formatting needs timezone (CASSANDRA-11137)
+ * Fix is_dense recalculation for Thrift-updated tables (CASSANDRA-11502)
+ * Remove unnescessary file existence check during anticompaction (CASSANDRA-11660)
+ * Add missing files to debian packages (CASSANDRA-11642)
+ * Avoid calling Iterables::concat in loops during ModificationStatement::getFunctions (CASSANDRA-11621)
+ * cqlsh: COPY FROM should use regular inserts for single statement batches and
+ report errors correctly if workers processes crash on initialization (CASSANDRA-11474)
+ * Always close cluster with connection in CqlRecordWriter (CASSANDRA-11553)
+ * Fix slice queries on ordered COMPACT tables (CASSANDRA-10988)
+Merged from 2.1:
+ * Create interval tree over canonical sstables to avoid missing sstables during streaming (CASSANDRA-11886)
* cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
- * Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
* Backport CASSANDRA-11578 (CASSANDRA-11750)
* Clear out parent repair session if repair coordinator dies (CASSANDRA-11824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index e24917c,559ba0b..d3a5028
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@@ -1588,8 -1529,12 +1588,12 @@@ public class DatabaseDescripto
public static int getSSTablePreempiveOpenIntervalInMB()
{
- return conf.sstable_preemptive_open_interval_in_mb;
+ return FBUtilities.isWindows() ? -1 : conf.sstable_preemptive_open_interval_in_mb;
}
+ public static void setSSTablePreempiveOpenIntervalInMB(int mb)
+ {
+ conf.sstable_preemptive_open_interval_in_mb = mb;
+ }
public static boolean getTrickleFsync()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/View.java
index fba1627,0000000..e303801
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@@ -1,281 -1,0 +1,286 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.lifecycle;
+
+import java.util.*;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.*;
+
+import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.RowPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.Interval;
+
+import static com.google.common.base.Predicates.equalTo;
+import static com.google.common.base.Predicates.not;
+import static com.google.common.collect.ImmutableList.copyOf;
+import static com.google.common.collect.ImmutableList.of;
+import static com.google.common.collect.Iterables.all;
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.filter;
+import static java.util.Collections.singleton;
+import static org.apache.cassandra.db.lifecycle.Helpers.emptySet;
+import static org.apache.cassandra.db.lifecycle.Helpers.replace;
+
+/**
+ * An immutable structure holding the current memtable, the memtables pending
+ * flush, the sstables for a column family, and the sstables that are active
+ * in compaction (a subset of the sstables).
+ *
+ * Modifications to instances are all performed via a Function produced by the static methods in this class.
+ * These are composed as necessary and provided to the Tracker.apply() methods, which atomically reject or
+ * accept and apply the changes to the View.
+ *
+ */
+public class View
+{
+ /**
+ * ordinarily a list of size 1, but when preparing to flush will contain both the memtable we will flush
+ * and the new replacement memtable, until all outstanding write operations on the old table complete.
+ * The last item in the list is always the "current" memtable.
+ */
+ public final List<Memtable> liveMemtables;
+ /**
+ * contains all memtables that are no longer referenced for writing and are queued for / in the process of being
+ * flushed. In chronologically ascending order.
+ */
+ public final List<Memtable> flushingMemtables;
+ public final Set<SSTableReader> compacting;
+ public final Set<SSTableReader> sstables;
+ public final Set<SSTableReader> premature;
+ // we use a Map here so that we can easily perform identity checks as well as equality checks.
+ // When marking compacting, we now indicate if we expect the sstables to be present (by default we do),
+ // and we then check that not only are they all present in the live set, but that the exact instance present is
+ // the one we made our decision to compact against.
+ public final Map<SSTableReader, SSTableReader> sstablesMap;
+
+ public final SSTableIntervalTree intervalTree;
+
+ View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
+ {
+ assert liveMemtables != null;
+ assert flushingMemtables != null;
+ assert sstables != null;
+ assert compacting != null;
+ assert intervalTree != null;
+
+ this.liveMemtables = liveMemtables;
+ this.flushingMemtables = flushingMemtables;
+
+ this.sstablesMap = sstables;
+ this.sstables = sstablesMap.keySet();
+ this.compacting = compacting;
+ this.premature = premature;
+ this.intervalTree = intervalTree;
+ }
+
+ public Memtable getCurrentMemtable()
+ {
+ return liveMemtables.get(liveMemtables.size() - 1);
+ }
+
+ /**
+ * @return the active memtable and all the memtables that are pending flush.
+ */
+ public Iterable<Memtable> getAllMemtables()
+ {
+ return concat(flushingMemtables, liveMemtables);
+ }
+
+ public Sets.SetView<SSTableReader> nonCompactingSStables()
+ {
+ return Sets.difference(sstables, compacting);
+ }
+
+ public Iterable<SSTableReader> getUncompacting(Iterable<SSTableReader> candidates)
+ {
+ return filter(candidates, new Predicate<SSTableReader>()
+ {
+ public boolean apply(SSTableReader sstable)
+ {
+ return !compacting.contains(sstable);
+ }
+ });
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("View(pending_count=%d, sstables=%s, compacting=%s)", liveMemtables.size() + flushingMemtables.size() - 1, sstables, compacting);
+ }
+
+ /**
+ * Returns the sstables that have any partition between {@code left} and {@code right}, when both bounds are taken inclusively.
+ * The interval formed by {@code left} and {@code right} shouldn't wrap.
+ */
+ public List<SSTableReader> sstablesInBounds(RowPosition left, RowPosition right)
+ {
++ return sstablesInBounds(left, right, intervalTree);
++ }
++
++ public static List<SSTableReader> sstablesInBounds(RowPosition left, RowPosition right, SSTableIntervalTree intervalTree)
++ {
+ assert !AbstractBounds.strictlyWrapsAround(left, right);
+
+ if (intervalTree.isEmpty())
+ return Collections.emptyList();
+
+ RowPosition stopInTree = right.isMinimum() ? intervalTree.max() : right;
+ return intervalTree.search(Interval.<RowPosition, SSTableReader>create(left, stopInTree));
+ }
+
+ // METHODS TO CONSTRUCT FUNCTIONS FOR MODIFYING A VIEW:
+
+ // return a function to un/mark the provided readers compacting in a view
+ static Function<View, View> updateCompacting(final Set<SSTableReader> unmark, final Iterable<SSTableReader> mark)
+ {
+ if (unmark.isEmpty() && Iterables.isEmpty(mark))
+ return Functions.identity();
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ assert all(mark, Helpers.idIn(view.sstablesMap));
+ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap,
+ replace(view.compacting, unmark, mark),
+ view.premature, view.intervalTree);
+ }
+ };
+ }
+
+ // construct a predicate to reject views that do not permit us to mark these readers compacting;
+ // i.e. one of them is either already compacting, has been compacted, or has been replaced
+ static Predicate<View> permitCompacting(final Iterable<SSTableReader> readers)
+ {
+ return new Predicate<View>()
+ {
+ public boolean apply(View view)
+ {
+ for (SSTableReader reader : readers)
+ if (view.compacting.contains(reader) || view.sstablesMap.get(reader) != reader || reader.isMarkedCompacted() || view.premature.contains(reader))
+ return false;
+ return true;
+ }
+ };
+ }
+
+ // construct a function to change the liveset in a Snapshot
+ static Function<View, View> updateLiveSet(final Set<SSTableReader> remove, final Iterable<SSTableReader> add)
+ {
+ if (remove.isEmpty() && Iterables.isEmpty(add))
+ return Functions.identity();
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, remove, add);
+ return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting, view.premature,
+ SSTableIntervalTree.build(sstableMap.keySet()));
+ }
+ };
+ }
+
+ // called prior to initiating flush: add newMemtable to liveMemtables, making it the latest memtable
+ static Function<View, View> switchMemtable(final Memtable newMemtable)
+ {
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ List<Memtable> newLive = ImmutableList.<Memtable>builder().addAll(view.liveMemtables).add(newMemtable).build();
+ assert newLive.size() == view.liveMemtables.size() + 1;
+ return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
+ }
+ };
+ }
+
+ // called before flush: move toFlush from liveMemtables to flushingMemtables
+ static Function<View, View> markFlushing(final Memtable toFlush)
+ {
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ List<Memtable> live = view.liveMemtables, flushing = view.flushingMemtables;
+ List<Memtable> newLive = copyOf(filter(live, not(equalTo(toFlush))));
+ List<Memtable> newFlushing = copyOf(concat(filter(flushing, lessThan(toFlush)),
+ of(toFlush),
+ filter(flushing, not(lessThan(toFlush)))));
+ assert newLive.size() == live.size() - 1;
+ assert newFlushing.size() == flushing.size() + 1;
+ return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
+ }
+ };
+ }
+
+ // called after flush: removes memtable from flushingMemtables, and inserts flushed into the live sstable set
+ static Function<View, View> replaceFlushed(final Memtable memtable, final SSTableReader flushed)
+ {
+ return new Function<View, View>()
+ {
+ public View apply(View view)
+ {
+ List<Memtable> flushingMemtables = copyOf(filter(view.flushingMemtables, not(equalTo(memtable))));
+ assert flushingMemtables.size() == view.flushingMemtables.size() - 1;
+
+ if (flushed == null)
+ return new View(view.liveMemtables, flushingMemtables, view.sstablesMap,
+ view.compacting, view.premature, view.intervalTree);
+
+ Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), singleton(flushed));
+ Set<SSTableReader> compacting = replace(view.compacting, emptySet(), singleton(flushed));
+ Set<SSTableReader> premature = replace(view.premature, emptySet(), singleton(flushed));
+ return new View(view.liveMemtables, flushingMemtables, sstableMap, compacting, premature,
+ SSTableIntervalTree.build(sstableMap.keySet()));
+ }
+ };
+ }
+
+ static Function<View, View> permitCompactionOfFlushed(final SSTableReader reader)
+ {
+ return new Function<View, View>()
+ {
+
+ @Nullable
+ public View apply(View view)
+ {
+ Set<SSTableReader> premature = ImmutableSet.copyOf(filter(view.premature, not(equalTo(reader))));
+ Set<SSTableReader> compacting = ImmutableSet.copyOf(filter(view.compacting, not(equalTo(reader))));
+ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compacting, premature, view.intervalTree);
+ }
+ };
+ }
+
+
+ private static <T extends Comparable<T>> Predicate<T> lessThan(final T lessThan)
+ {
+ return new Predicate<T>()
+ {
+ public boolean apply(T t)
+ {
+ return t.compareTo(lessThan) < 0;
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/src/java/org/apache/cassandra/streaming/StreamSession.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/streaming/StreamSession.java
index f14b94c,273631c..f4c900e
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@@ -25,18 -25,22 +25,22 @@@ import java.util.*
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
-import javax.annotation.Nullable;
-
+ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.collect.*;
+
++import org.apache.cassandra.db.lifecycle.SSTableIntervalTree;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DataTracker;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowPosition;
+ import org.apache.cassandra.dht.AbstractBounds;
+ import org.apache.cassandra.dht.Bounds;
-import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.*;
@@@ -279,7 -270,7 +283,7 @@@ public class StreamSession implements I
flushSSTables(stores);
List<Range<Token>> normalizedRanges = Range.normalize(ranges);
- List<SSTableStreamingSections> sections = getSSTableSectionsForRanges(normalizedRanges, stores, repairedAt);
- List<SSTableStreamingSections> sections = getSSTableSectionsForRanges(normalizedRanges, stores, repairedAt, repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE);
++ List<SSTableStreamingSections> sections = getSSTableSectionsForRanges(normalizedRanges, stores, repairedAt, isIncremental);
try
{
addTransferFiles(sections);
@@@ -314,33 -306,21 +319,26 @@@
{
for (ColumnFamilyStore cfStore : stores)
{
- final List<AbstractBounds<RowPosition>> rowBoundsList = new ArrayList<>(ranges.size());
- final IPartitioner partitioner = cfStore.partitioner;
+ final List<Range<RowPosition>> keyRanges = new ArrayList<>(ranges.size());
for (Range<Token> range : ranges)
- rowBoundsList.add(range.toRowBounds());
- refs.addAll(cfStore.selectAndReference(new Function<DataTracker.View, List<SSTableReader>>()
+ keyRanges.add(Range.makeRowRange(range));
+ refs.addAll(cfStore.selectAndReference(new Function<View, List<SSTableReader>>()
{
- public List<SSTableReader> apply(DataTracker.View view)
+ public List<SSTableReader> apply(View view)
{
- Map<SSTableReader, SSTableReader> permittedInstances = new HashMap<>();
- for (SSTableReader reader : ColumnFamilyStore.CANONICAL_SSTABLES.apply(view))
- permittedInstances.put(reader, reader);
-
- DataTracker.SSTableIntervalTree intervalTree = DataTracker.buildIntervalTree(ColumnFamilyStore.CANONICAL_SSTABLES.apply(view));
++ SSTableIntervalTree intervalTree = SSTableIntervalTree.build(ColumnFamilyStore.CANONICAL_SSTABLES.apply(view));
Set<SSTableReader> sstables = Sets.newHashSet();
- for (AbstractBounds<RowPosition> rowBounds : rowBoundsList)
+ for (Range<RowPosition> keyRange : keyRanges)
{
- for (SSTableReader sstable : DataTracker.View.sstablesInBounds(rowBounds, intervalTree, partitioner))
+ // keyRange excludes its start, while sstableInBounds is inclusive (of both start and end).
+ // This is fine however, because keyRange has been created from a token range through Range.makeRowRange (see above).
+ // And that later method uses the Token.maxKeyBound() method to creates the range, which return a "fake" key that
+ // sort after all keys having the token. That "fake" key cannot however be equal to any real key, so that even
+ // including keyRange.left will still exclude any key having the token of the original token range, and so we're
+ // still actually selecting what we wanted.
- for (SSTableReader sstable : view.sstablesInBounds(keyRange.left, keyRange.right))
++ for (SSTableReader sstable : View.sstablesInBounds(keyRange.left, keyRange.right, intervalTree))
{
- // sstableInBounds may contain early opened sstables
- if (isIncremental && sstable.isRepaired())
- continue;
- sstable = permittedInstances.get(sstable);
- if (sstable != null)
+ if (!isIncremental || !sstable.isRepaired())
sstables.add(sstable);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/05f8a008/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index c2cc6e3,1fb28f5..f50953a
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@@ -21,15 -21,11 +21,18 @@@ import java.io.File
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
+ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
++import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
@@@ -41,25 -35,24 +44,29 @@@ import org.apache.cassandra.db.*
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.LazilyCompactedRow;
import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.db.compaction.SSTableSplitter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.metrics.StorageMetrics;
+ import org.apache.cassandra.notifications.INotification;
+ import org.apache.cassandra.notifications.INotificationConsumer;
+ import org.apache.cassandra.notifications.SSTableListChangedNotification;
import org.apache.cassandra.service.StorageService;
+ import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
public class SSTableRewriterTest extends SchemaLoader
{
@@@ -807,10 -769,76 +814,69 @@@
}
}
}
- writer.abort();
- cfs.getDataTracker().unmarkCompacting(sstables);
- truncate(cfs);
+ truncateCF();
+ validateCFS(cfs);
}
+ @Test
+ public void testSSTableSectionsForRanges() throws IOException, InterruptedException, ExecutionException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+ truncate(cfs);
+
+ cfs.addSSTable(writeFile(cfs, 1000));
+
+ Collection<SSTableReader> allSSTables = cfs.getSSTables();
+ assertEquals(1, allSSTables.size());
+ final Token firstToken = allSSTables.iterator().next().first.getToken();
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(1);
+
+ List<StreamSession.SSTableStreamingSections> sectionsBeforeRewrite = StreamSession.getSSTableSectionsForRanges(
+ Collections.singleton(new Range<Token>(firstToken, firstToken)),
+ Collections.singleton(cfs), 0L, false);
+ assertEquals(1, sectionsBeforeRewrite.size());
+ for (StreamSession.SSTableStreamingSections section : sectionsBeforeRewrite)
+ section.ref.release();
+ final AtomicInteger checkCount = new AtomicInteger();
+ // needed since we get notified when compaction is done as well - we can't get sections for ranges for obsoleted sstables
- INotificationConsumer consumer = new INotificationConsumer()
++ final AtomicBoolean done = new AtomicBoolean(false);
++ final AtomicBoolean failed = new AtomicBoolean(false);
++ Runnable r = new Runnable()
++ {
++ public void run()
++ {
++ while (!done.get())
+ {
- public void handleNotification(INotification notification, Object sender)
- {
- if (notification instanceof SSTableListChangedNotification)
- {
- Collection<SSTableReader> added = ((SSTableListChangedNotification) notification).added;
- Collection<SSTableReader> removed = ((SSTableListChangedNotification) notification).removed;
- // note that we need to check if added.equals(removed) because once the compaction is done the old sstable will have
- // selfRef().globalCount() == 0 and we cant get the SectionsForRanges then. During incremental opening we always add and remove the same
- // sstable (note that the sstables are x.equal(y) but not x == y since the new one will be a new instance with a moved starting point
- // In this case we must avoid trying to call getSSTableSectionsForRanges since we are in the notification
- // method and trying to reference an sstable with globalcount == 0 puts it into a loop, and this blocks the tracker from removing the
- // unreferenced sstable.
- if (added.isEmpty() || !added.iterator().next().getColumnFamilyName().equals(cfs.getColumnFamilyName()) || !added.equals(removed))
- return;
-
- // at no point must the rewrite process hide
- // sections returned by getSSTableSectionsForRanges
- Set<Range<Token>> range = Collections.singleton(new Range<Token>(firstToken, firstToken));
- List<StreamSession.SSTableStreamingSections> sections = StreamSession.getSSTableSectionsForRanges(range, Collections.singleton(cfs), 0L, false);
- assertEquals(1, sections.size());
- for (StreamSession.SSTableStreamingSections section : sections)
- section.ref.release();
- checkCount.incrementAndGet();
- }
- }
- };
- cfs.getDataTracker().subscribe(consumer);
++ Set<Range<Token>> range = Collections.singleton(new Range<Token>(firstToken, firstToken));
++ List<StreamSession.SSTableStreamingSections> sections = StreamSession.getSSTableSectionsForRanges(range, Collections.singleton(cfs), 0L, false);
++ if (sections.size() != 1)
++ failed.set(true);
++ for (StreamSession.SSTableStreamingSections section : sections)
++ section.ref.release();
++ checkCount.incrementAndGet();
++ Uninterruptibles.sleepUninterruptibly(5, TimeUnit.MILLISECONDS);
++ }
++ }
++ };
++ Thread t = new Thread(r);
+ try
+ {
++ t.start();
+ cfs.forceMajorCompaction();
+ // reset
+ }
+ finally
+ {
+ DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(50);
- cfs.getDataTracker().unsubscribe(consumer);
++ done.set(true);
++ t.join(20);
+ }
++ assertFalse(failed.get());
+ assertTrue(checkCount.get() >= 2);
+ truncate(cfs);
+ }
+
/**
* emulates anticompaction - writing from one source sstable to two new sstables
*
[08/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/73a8341f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/73a8341f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/73a8341f
Branch: refs/heads/cassandra-3.0
Commit: 73a8341fef25de7236bc591e84cddc637c0b7b2f
Parents: 3d211e9 05f8a00
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Jun 13 15:14:28 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Jun 13 15:14:28 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 4 ++
.../org/apache/cassandra/db/lifecycle/View.java | 11 +++
.../cassandra/streaming/StreamSession.java | 18 +++--
.../io/sstable/SSTableRewriterTest.java | 75 +++++++++++++++++---
5 files changed, 88 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/73a8341f/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 47aef7e,491f72a..8a04077
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,28 -1,5 +1,29 @@@
-2.2.7
+3.0.8
+ * Add TimeWindowCompactionStrategy (CASSANDRA-9666)
+Merged from 2.2:
* StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
+Merged from 2.1:
++ * Create interval tree over canonical sstables to avoid missing sstables during streaming (CASSANDRA-11886)
+ * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
+
+
+3.0.7
+ * Fix legacy serialization of Thrift-generated non-compound range tombstones
+ when communicating with 2.x nodes (CASSANDRA-11930)
+ * Fix Directories instantiations where CFS.initialDirectories should be used (CASSANDRA-11849)
+ * Avoid referencing DatabaseDescriptor in AbstractType (CASSANDRA-11912)
+ * Fix sstables not being protected from removal during index build (CASSANDRA-11905)
+ * cqlsh: Suppress stack trace from Read/WriteFailures (CASSANDRA-11032)
+ * Remove unneeded code to repair index summaries that have
+ been improperly down-sampled (CASSANDRA-11127)
+ * Avoid WriteTimeoutExceptions during commit log replay due to materialized
+ view lock contention (CASSANDRA-11891)
+ * Prevent OOM failures on SSTable corruption, improve tests for corruption detection (CASSANDRA-9530)
+ * Use CFS.initialDirectories when clearing snapshots (CASSANDRA-11705)
+ * Allow compaction strategies to disable early open (CASSANDRA-11754)
+ * Refactor Materialized View code (CASSANDRA-11475)
+ * Update Java Driver (CASSANDRA-11615)
+Merged from 2.2:
* Persist local metadata earlier in startup sequence (CASSANDRA-11742)
* Run CommitLog tests with different compression settings (CASSANDRA-9039)
* cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/73a8341f/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/73a8341f/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/View.java
index 17062b4,e303801..99903fc
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@@ -179,52 -131,23 +179,63 @@@ public class Vie
}
/**
- * Returns the sstables that have any partition between {@code left} and {@code right}, when both bounds are taken inclusively.
- * The interval formed by {@code left} and {@code right} shouldn't wrap.
- */
- public List<SSTableReader> sstablesInBounds(RowPosition left, RowPosition right)
+ * Returns the sstables that have any partition between {@code left} and {@code right}, when both bounds are taken inclusively.
+ * The interval formed by {@code left} and {@code right} shouldn't wrap.
+ */
+ public Iterable<SSTableReader> sstablesInBounds(SSTableSet sstableSet, PartitionPosition left, PartitionPosition right)
{
- return sstablesInBounds(left, right, intervalTree);
+ assert !AbstractBounds.strictlyWrapsAround(left, right);
+
+ if (intervalTree.isEmpty())
+ return Collections.emptyList();
+
+ PartitionPosition stopInTree = right.isMinimum() ? intervalTree.max() : right;
+ return select(sstableSet, intervalTree.search(Interval.create(left, stopInTree)));
}
- public static List<SSTableReader> sstablesInBounds(RowPosition left, RowPosition right, SSTableIntervalTree intervalTree)
++ public static List<SSTableReader> sstablesInBounds(PartitionPosition left, PartitionPosition right, SSTableIntervalTree intervalTree)
+ {
+ assert !AbstractBounds.strictlyWrapsAround(left, right);
+
+ if (intervalTree.isEmpty())
+ return Collections.emptyList();
+
- RowPosition stopInTree = right.isMinimum() ? intervalTree.max() : right;
- return intervalTree.search(Interval.<RowPosition, SSTableReader>create(left, stopInTree));
++ PartitionPosition stopInTree = right.isMinimum() ? intervalTree.max() : right;
++ return intervalTree.search(Interval.create(left, stopInTree));
++ }
++
+ public static Function<View, Iterable<SSTableReader>> select(SSTableSet sstableSet)
+ {
+ return (view) -> view.sstables(sstableSet);
+ }
+
+ public static Function<View, Iterable<SSTableReader>> select(SSTableSet sstableSet, Predicate<SSTableReader> filter)
+ {
+ return (view) -> view.sstables(sstableSet, filter);
+ }
+
+ /**
+ * @return a ViewFragment containing the sstables and memtables that may need to be merged
+ * for the given @param key, according to the interval tree
+ */
+ public static Function<View, Iterable<SSTableReader>> select(SSTableSet sstableSet, DecoratedKey key)
+ {
+ assert sstableSet == SSTableSet.LIVE;
+ return (view) -> view.intervalTree.search(key);
+ }
+
+ /**
+ * @return a ViewFragment containing the sstables and memtables that may need to be merged
+ * for rows within @param rowBounds, inclusive, according to the interval tree.
+ */
+ public static Function<View, Iterable<SSTableReader>> select(SSTableSet sstableSet, AbstractBounds<PartitionPosition> rowBounds)
+ {
+ // Note that View.sstablesInBounds always includes it's bound while rowBounds may not. This is ok however
+ // because the fact we restrict the sstables returned by this function is an optimization in the first
+ // place and the returned sstables will (almost) never cover *exactly* rowBounds anyway. It's also
+ // *very* unlikely that a sstable is included *just* because we consider one of the bound inclusively
+ // instead of exclusively, so the performance impact is negligible in practice.
+ return (view) -> view.sstablesInBounds(sstableSet, rowBounds.left, rowBounds.right);
}
// METHODS TO CONSTRUCT FUNCTIONS FOR MODIFYING A VIEW:
http://git-wip-us.apache.org/repos/asf/cassandra/blob/73a8341f/src/java/org/apache/cassandra/streaming/StreamSession.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/streaming/StreamSession.java
index 2ed6ad1,f4c900e..d5c060e
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@@ -25,10 -25,12 +25,13 @@@ import java.util.*
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
+ import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
import com.google.common.collect.*;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+ import org.apache.cassandra.db.lifecycle.SSTableIntervalTree;
+ import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -320,30 -319,33 +324,24 @@@ public class StreamSession implements I
{
for (ColumnFamilyStore cfStore : stores)
{
- final List<Range<RowPosition>> keyRanges = new ArrayList<>(ranges.size());
+ final List<Range<PartitionPosition>> keyRanges = new ArrayList<>(ranges.size());
for (Range<Token> range : ranges)
keyRanges.add(Range.makeRowRange(range));
- refs.addAll(cfStore.selectAndReference(new Function<View, List<SSTableReader>>()
- {
- public List<SSTableReader> apply(View view)
+ refs.addAll(cfStore.selectAndReference(view -> {
+ Set<SSTableReader> sstables = Sets.newHashSet();
++ SSTableIntervalTree intervalTree = SSTableIntervalTree.build(view.sstables(SSTableSet.CANONICAL));
+ for (Range<PartitionPosition> keyRange : keyRanges)
{
- // keyRange excludes its start, while sstableInBounds is inclusive (of both start and end).
- // This is fine however, because keyRange has been created from a token range through Range.makeRowRange (see above).
- // And that later method uses the Token.maxKeyBound() method to creates the range, which return a "fake" key that
- // sort after all keys having the token. That "fake" key cannot however be equal to any real key, so that even
- // including keyRange.left will still exclude any key having the token of the original token range, and so we're
- // still actually selecting what we wanted.
- for (SSTableReader sstable : view.sstablesInBounds(SSTableSet.CANONICAL, keyRange.left, keyRange.right))
- SSTableIntervalTree intervalTree = SSTableIntervalTree.build(ColumnFamilyStore.CANONICAL_SSTABLES.apply(view));
- Set<SSTableReader> sstables = Sets.newHashSet();
- for (Range<RowPosition> keyRange : keyRanges)
++ for (SSTableReader sstable : View.sstablesInBounds(keyRange.left, keyRange.right, intervalTree))
{
- // sstableInBounds may contain early opened sstables
- // keyRange excludes its start, while sstableInBounds is inclusive (of both start and end).
- // This is fine however, because keyRange has been created from a token range through Range.makeRowRange (see above).
- // And that later method uses the Token.maxKeyBound() method to creates the range, which return a "fake" key that
- // sort after all keys having the token. That "fake" key cannot however be equal to any real key, so that even
- // including keyRange.left will still exclude any key having the token of the original token range, and so we're
- // still actually selecting what we wanted.
- for (SSTableReader sstable : View.sstablesInBounds(keyRange.left, keyRange.right, intervalTree))
- {
- if (!isIncremental || !sstable.isRepaired())
- sstables.add(sstable);
- }
+ if (!isIncremental || !sstable.isRepaired())
+ sstables.add(sstable);
}
-
- logger.debug("ViewFilter for {}/{} sstables", sstables.size(), view.sstables.size());
- return ImmutableList.copyOf(sstables);
}
+
+ if (logger.isDebugEnabled())
+ logger.debug("ViewFilter for {}/{} sstables", sstables.size(), Iterables.size(view.sstables(SSTableSet.CANONICAL)));
+ return sstables;
}).refs);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/73a8341f/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index 008df06,f50953a..18bc760
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@@ -20,48 -19,49 +20,42 @@@ package org.apache.cassandra.io.sstable
import java.io.File;
import java.io.IOException;
--import java.nio.ByteBuffer;
import java.util.*;
- import java.util.concurrent.ThreadLocalRandom;
+ import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
--import org.junit.After;
--import org.junit.AfterClass;
--import org.junit.BeforeClass;
import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Test;
--import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.config.Config;
+import org.apache.cassandra.UpdateBuilder;
- import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.KSMetaData;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.compaction.AbstractCompactedRow;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.db.SerializationHeader;
++import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionController;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.compaction.LazilyCompactedRow;
+import org.apache.cassandra.db.compaction.CompactionIterator;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.db.compaction.SSTableSplitter;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
- import org.apache.cassandra.db.lifecycle.View;
- import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
- import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.metrics.StorageMetrics;
- import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.notifications.INotification;
-import org.apache.cassandra.notifications.INotificationConsumer;
-import org.apache.cassandra.notifications.SSTableListChangedNotification;
-import org.apache.cassandra.service.StorageService;
+ import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;