You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2019/12/16 12:35:25 UTC
[cassandra] branch trunk updated: Make it possible to disable
STCS-in-L0 during runtime
This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5463310 Make it possible to disable STCS-in-L0 during runtime
5463310 is described below
commit 546331037ee8218208b61a26ecbc12ab8dd7eace
Author: Marcus Eriksson <ma...@apache.org>
AuthorDate: Mon Dec 9 14:29:20 2019 +0100
Make it possible to disable STCS-in-L0 during runtime
Patch by marcuse; reviewed by Jordan West for CASSANDRA-15445
---
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 7 ++-
.../cassandra/db/compaction/CompactionManager.java | 12 ++++
.../db/compaction/CompactionManagerMBean.java | 6 ++
.../compaction/LeveledCompactionStrategyTest.java | 70 ++++++++++++++++++++++
.../org/apache/cassandra/schema/MockSchema.java | 15 +++--
6 files changed, 102 insertions(+), 9 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 4c4af72..b88a65b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.0-alpha3
+ * Make it possible to disable STCS-in-L0 during runtime (CASSANDRA-15445)
* Removed obsolete OldNetworkTopologyStrategy (CASSANDRA-13990)
* Align record header of FQL and audit binary log (CASSANDRA-15076)
* Shuffle forwarding replica for messages to non-local DC (CASSANDRA-15318)
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 3c184bd..e6fa280 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -145,7 +145,7 @@ public class DatabaseDescriptor
private static final int searchConcurrencyFactor = Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "search_concurrency_factor", "1"));
- private static final boolean disableSTCSInL0 = Boolean.getBoolean(Config.PROPERTY_PREFIX + "disable_stcs_in_l0");
+ private static volatile boolean disableSTCSInL0 = Boolean.getBoolean(Config.PROPERTY_PREFIX + "disable_stcs_in_l0");
private static final boolean unsafeSystem = Boolean.getBoolean(Config.PROPERTY_PREFIX + "unsafesystem");
// turns some warnings into exceptions for testing
@@ -1737,6 +1737,11 @@ public class DatabaseDescriptor
return disableSTCSInL0;
}
+ public static void setDisableSTCSInL0(boolean disabled)
+ {
+ disableSTCSInL0 = disabled;
+ }
+
public static int getStreamThroughputOutboundMegabitsPerSec()
{
return conf.stream_throughput_outbound_megabits_per_sec;
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 9dab6d7..28db027 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -2065,6 +2065,18 @@ public class CompactionManager implements CompactionManagerMBean
validationExecutor.setMaximumPoolSize(number);
}
+ public boolean getDisableSTCSInL0()
+ {
+ return DatabaseDescriptor.getDisableSTCSInL0();
+ }
+
+ public void setDisableSTCSInL0(boolean disabled)
+ {
+ if (disabled != DatabaseDescriptor.getDisableSTCSInL0())
+ logger.info("Changing STCS in L0 disabled from {} to {}", DatabaseDescriptor.getDisableSTCSInL0(), disabled);
+ DatabaseDescriptor.setDisableSTCSInL0(disabled);
+ }
+
public int getCoreViewBuildThreads()
{
return viewBuildExecutor.getCorePoolSize();
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java b/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
index e4d5392..d298c8b 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
@@ -123,6 +123,12 @@ public interface CompactionManagerMBean
public int getCoreViewBuildThreads();
/**
+ * Enable / disable STCS in L0
+ */
+ public boolean getDisableSTCSInL0();
+ public void setDisableSTCSInL0(boolean disabled);
+
+ /**
* Allows user to resize maximum size of the view build thread pool.
* @param number New maximum of view build threads
*/
diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
index 6c75e7b..b925bab 100644
--- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
@@ -17,13 +17,16 @@
*/
package org.apache.cassandra.db.compaction;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.UUID;
@@ -52,6 +55,7 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.notifications.SSTableAddedNotification;
import org.apache.cassandra.notifications.SSTableRepairStatusChanged;
import org.apache.cassandra.repair.ValidationManager;
+import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.repair.Validator;
@@ -490,4 +494,70 @@ public class LeveledCompactionStrategyTest
lastMaxTimeStamp = sstable.getMaxTimestamp();
}
}
+
+ @Test
+ public void testDisableSTCSInL0() throws IOException
+ {
+ /*
+ First creates a bunch of sstables in L1, then overloads L0 with 50 sstables. Now with STCS in L0 enabled
+ we should get a compaction task where the target level is 0, then we disable STCS-in-L0 and make sure that
+ the compaction task we get targets L1 or higher.
+ */
+ ColumnFamilyStore cfs = MockSchema.newCFS();
+ Map<String, String> localOptions = new HashMap<>();
+ localOptions.put("class", "LeveledCompactionStrategy");
+ localOptions.put("sstable_size_in_mb", "1");
+ cfs.setCompactionParameters(localOptions);
+ List<SSTableReader> sstables = new ArrayList<>();
+ for (int i = 0; i < 11; i++)
+ {
+ SSTableReader l1sstable = MockSchema.sstable(i, 1 * 1024 * 1024, cfs);
+ l1sstable.descriptor.getMetadataSerializer().mutateLevel(l1sstable.descriptor, 1);
+ l1sstable.reloadSSTableMetadata();
+ sstables.add(l1sstable);
+ }
+
+ for (int i = 100; i < 150; i++)
+ sstables.add(MockSchema.sstable(i, 1 * 1024 * 1024, cfs));
+
+ cfs.disableAutoCompaction();
+ cfs.addSSTables(sstables);
+ assertEquals(0, getTaskLevel(cfs));
+
+ try
+ {
+ CompactionManager.instance.setDisableSTCSInL0(true);
+ assertTrue(getTaskLevel(cfs) > 0);
+ }
+ finally
+ {
+ CompactionManager.instance.setDisableSTCSInL0(false);
+ }
+ }
+
+ private int getTaskLevel(ColumnFamilyStore cfs)
+ {
+ int level = -1;
+ for (List<AbstractCompactionStrategy> strategies : cfs.getCompactionStrategyManager().getStrategies())
+ {
+ for (AbstractCompactionStrategy strategy : strategies)
+ {
+ AbstractCompactionTask task = strategy.getNextBackgroundTask(0);
+ if (task != null)
+ {
+ try
+ {
+ assertTrue(task instanceof LeveledCompactionTask);
+ LeveledCompactionTask lcsTask = (LeveledCompactionTask) task;
+ level = Math.max(level, lcsTask.getLevel());
+ }
+ finally
+ {
+ task.transaction.abort();
+ }
+ }
+ }
+ }
+ return level;
+ }
}
diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java
index dfa8731..40b0f87 100644
--- a/test/unit/org/apache/cassandra/schema/MockSchema.java
+++ b/test/unit/org/apache/cassandra/schema/MockSchema.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Consumer;
import java.util.function.Function;
import com.google.common.collect.ImmutableSet;
@@ -40,14 +39,10 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.ChannelProxy;
import org.apache.cassandra.io.util.FileHandle;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.Memory;
-import org.apache.cassandra.schema.CachingParams;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.utils.AlwaysPresentFilter;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -65,7 +60,8 @@ public class MockSchema
public static final Keyspace ks = Keyspace.mockKS(KeyspaceMetadata.create("mockks", KeyspaceParams.simpleTransient(1)));
public static final IndexSummary indexSummary;
- private static final FileHandle RANDOM_ACCESS_READER_FACTORY = new FileHandle.Builder(temp("mocksegmentedfile").getAbsolutePath()).complete();
+
+ private static final File tempFile = temp("mocksegmentedfile");
public static Memtable memtable(ColumnFamilyStore cfs)
{
@@ -114,6 +110,9 @@ public class MockSchema
{
}
}
+ // .complete() with size to make sstable.onDiskLength work
+ @SuppressWarnings("resource")
+ FileHandle fileHandle = new FileHandle.Builder(new ChannelProxy(tempFile)).bufferSize(size).complete(size);
if (size > 0)
{
try
@@ -134,7 +133,7 @@ public class MockSchema
.finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, header)
.get(MetadataType.STATS);
SSTableReader reader = SSTableReader.internalOpen(descriptor, components, cfs.metadata,
- RANDOM_ACCESS_READER_FACTORY.sharedCopy(), RANDOM_ACCESS_READER_FACTORY.sharedCopy(), indexSummary.sharedCopy(),
+ fileHandle.sharedCopy(), fileHandle.sharedCopy(), indexSummary.sharedCopy(),
new AlwaysPresentFilter(), 1L, metadata, SSTableReader.OpenReason.NORMAL, header);
reader.first = readerBounds(firstToken);
reader.last = readerBounds(lastToken);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org