You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by bd...@apache.org on 2019/04/25 17:27:08 UTC
[cassandra] branch cassandra-3.11 updated (839fc7e -> 01d6548)
This is an automated email from the ASF dual-hosted git repository.
bdeggleston pushed a change to branch cassandra-3.11
in repository https://gitbox.apache.org/repos/asf/cassandra.git.
from 839fc7e Merge branch 'cassandra-3.0' into cassandra-3.11
new c3ce32e Fix assorted gossip races and add related runtime checks
new 01d6548 Merge branch 'cassandra-3.0' into cassandra-3.11
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.txt | 1 +
build.xml | 1 +
ide/idea/workspace.xml | 2 +-
.../cassandra/config/DatabaseDescriptor.java | 8 +
src/java/org/apache/cassandra/gms/Gossiper.java | 192 +++++++++++++++------
.../apache/cassandra/service/StorageService.java | 10 +-
.../cassandra/distributed/impl/Instance.java | 20 ++-
.../org/apache/cassandra/gms/GossiperTest.java | 1 +
.../gms/PendingRangeCalculatorServiceTest.java | 1 +
.../cassandra/locator/CloudstackSnitchTest.java | 1 +
.../apache/cassandra/locator/EC2SnitchTest.java | 1 +
.../cassandra/locator/GoogleCloudSnitchTest.java | 1 +
.../cassandra/locator/PropertyFileSnitchTest.java | 1 +
13 files changed, 170 insertions(+), 70 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra] 01/01: Merge branch 'cassandra-3.0' into cassandra-3.11
Posted by bd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bdeggleston pushed a commit to branch cassandra-3.11
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 01d6548e0dec465c01df4248e1d740c51710a36b
Merge: 839fc7e c3ce32e
Author: Blake Eggleston <bd...@gmail.com>
AuthorDate: Thu Apr 25 10:21:22 2019 -0700
Merge branch 'cassandra-3.0' into cassandra-3.11
CHANGES.txt | 1 +
build.xml | 1 +
ide/idea/workspace.xml | 2 +-
.../cassandra/config/DatabaseDescriptor.java | 8 +
src/java/org/apache/cassandra/gms/Gossiper.java | 192 +++++++++++++++------
.../apache/cassandra/service/StorageService.java | 10 +-
.../cassandra/distributed/impl/Instance.java | 20 ++-
.../org/apache/cassandra/gms/GossiperTest.java | 1 +
.../gms/PendingRangeCalculatorServiceTest.java | 1 +
.../cassandra/locator/CloudstackSnitchTest.java | 1 +
.../apache/cassandra/locator/EC2SnitchTest.java | 1 +
.../cassandra/locator/GoogleCloudSnitchTest.java | 1 +
.../cassandra/locator/PropertyFileSnitchTest.java | 1 +
13 files changed, 170 insertions(+), 70 deletions(-)
diff --cc CHANGES.txt
index 9ce2972,596d902..0506da7
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,9 -1,8 +1,10 @@@
-3.0.19
+3.11.5
+ * Fixed nodetool cfstats printing index name twice (CASSANDRA-14903)
+ * Add flag to disable SASI indexes, and warnings on creation (CASSANDRA-14866)
+Merged from 3.0:
+ * Fix assorted gossip races and add related runtime checks (CASSANDRA-15059)
* Fix mixed mode partition range scans with limit (CASSANDRA-15072)
* cassandra-stress works with frozen collections: list and set (CASSANDRA-14907)
- * For nodetool listsnapshots output, put spaces between columns, and increase snapshot padding (CASSANDRA-14876)
* Fix handling FS errors on writing and reading flat files - LogTransaction and hints (CASSANDRA-15053)
* Avoid double closing the iterator to avoid overcounting the number of requests (CASSANDRA-15058)
* Improve `nodetool status -r` speed (CASSANDRA-14847)
diff --cc ide/idea/workspace.xml
index a7c991e,3424af3..8d1b0fc
--- a/ide/idea/workspace.xml
+++ b/ide/idea/workspace.xml
@@@ -167,7 -168,7 +167,7 @@@
<option name="MAIN_CLASS_NAME" value="" />
<option name="METHOD_NAME" value="" />
<option name="TEST_OBJECT" value="class" />
- <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/test/conf/cassandra.yaml -Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml -Dcassandra.logdir=$PROJECT_DIR$/build/test/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -ea -XX:MaxMetaspaceSize=256M -XX:SoftRefLRUPolicyMSPerMB=0" />
- <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/test/conf/cassandra.yaml -Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml -Dcassandra.logdir=$PROJECT_DIR$/build/test/logs -ea -XX:MaxMetaspaceSize=256M -XX:SoftRefLRUPolicyMSPerMB=0 -Dcassandra.strict.runtime.checks=true" />
++ <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/test/conf/cassandra.yaml -Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml -Dcassandra.logdir=$PROJECT_DIR$/build/test/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -ea -XX:MaxMetaspaceSize=256M -XX:SoftRefLRUPolicyMSPerMB=0 -Dcassandra.strict.runtime.checks=true" />
<option name="PARAMETERS" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="ENV_VARIABLES" />
diff --cc src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 99f8575,db55c20..e452830
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@@ -112,153 -102,46 +112,156 @@@ public class DatabaseDescripto
private static String localDC;
private static Comparator<InetAddress> localComparator;
+ private static EncryptionContext encryptionContext;
private static boolean hasLoggedConfig;
+ private static BackPressureStrategy backPressureStrategy;
+ private static DiskOptimizationStrategy diskOptimizationStrategy;
+
+ private static boolean clientInitialized;
+ private static boolean toolInitialized;
private static boolean daemonInitialized;
+ private static final int searchConcurrencyFactor = Integer.parseInt(System.getProperty(Config.PROPERTY_PREFIX + "search_concurrency_factor", "1"));
+
+ private static final boolean disableSTCSInL0 = Boolean.getBoolean(Config.PROPERTY_PREFIX + "disable_stcs_in_l0");
+ private static final boolean unsafeSystem = Boolean.getBoolean(Config.PROPERTY_PREFIX + "unsafesystem");
+
+ // turns some warnings into exceptions for testing
+ private static final boolean strictRuntimeChecks = Boolean.getBoolean("cassandra.strict.runtime.checks");
+
- public static boolean isDaemonInitialized()
+ public static void daemonInitialization() throws ConfigurationException
{
- return daemonInitialized;
+ daemonInitialization(DatabaseDescriptor::loadConfig);
}
- public static void setDaemonInitialized()
+ public static void daemonInitialization(Supplier<Config> config) throws ConfigurationException
{
+ if (toolInitialized)
+ throw new AssertionError("toolInitialization() already called");
+ if (clientInitialized)
+ throw new AssertionError("clientInitialization() already called");
+
+ // Some unit tests require this :(
+ if (daemonInitialized)
+ return;
daemonInitialized = true;
+
+ setConfig(config.get());
+ applyAll();
+ AuthConfig.applyAuth();
+ }
+
+ /**
+ * Equivalent to {@link #toolInitialization(boolean) toolInitialization(true)}.
+ */
+ public static void toolInitialization()
+ {
+ toolInitialization(true);
}
- public static void forceStaticInitialization() {}
- static
+ /**
+ * Initializes this class as a tool, which means that the configuration is loaded
+ * using {@link #loadConfig()} and all non-daemon configuration parts will be setup.
+ *
+ * @param failIfDaemonOrClient if {@code true} and a call to {@link #daemonInitialization()} or
+ * {@link #clientInitialization()} has been performed before, an
+ * {@link AssertionError} will be thrown.
+ */
+ public static void toolInitialization(boolean failIfDaemonOrClient)
{
- // In client mode, we use a default configuration. Note that the fields of this class will be
- // left unconfigured however (the partitioner or localDC will be null for instance) so this
- // should be used with care.
- try
+ if (!failIfDaemonOrClient && (daemonInitialized || clientInitialized))
{
- if (Config.isClientMode())
- {
- conf = new Config();
- }
- else
- {
- applyConfig(loadConfig());
- }
+ return;
}
- catch (Exception e)
+ else
+ {
+ if (daemonInitialized)
+ throw new AssertionError("daemonInitialization() already called");
+ if (clientInitialized)
+ throw new AssertionError("clientInitialization() already called");
+ }
+
+ if (toolInitialized)
+ return;
+ toolInitialized = true;
+
+ setConfig(loadConfig());
+
+ applySimpleConfig();
+
+ applyPartitioner();
+
+ applySnitch();
+
+ applyEncryptionContext();
+ }
+
+ /**
+ * Equivalent to {@link #clientInitialization(boolean) clientInitialization(true)}.
+ */
+ public static void clientInitialization()
+ {
+ clientInitialization(true);
+ }
+
+ /**
+ * Initializes this class as a client, which means that just an empty configuration will
+ * be used.
+ *
+ * @param failIfDaemonOrTool if {@code true} and a call to {@link #daemonInitialization()} or
+ * {@link #toolInitialization()} has been performed before, an
+ * {@link AssertionError} will be thrown.
+ */
+ public static void clientInitialization(boolean failIfDaemonOrTool)
+ {
+ if (!failIfDaemonOrTool && (daemonInitialized || toolInitialized))
+ {
+ return;
+ }
+ else
{
- throw new ExceptionInInitializerError(e);
+ if (daemonInitialized)
+ throw new AssertionError("daemonInitialization() already called");
+ if (toolInitialized)
+ throw new AssertionError("toolInitialization() already called");
}
+
+ if (clientInitialized)
+ return;
+ clientInitialized = true;
+
+ Config.setClientMode(true);
+ conf = new Config();
+ diskOptimizationStrategy = new SpinningDiskOptimizationStrategy();
+ }
+
+ public static boolean isClientInitialized()
+ {
+ return clientInitialized;
+ }
+
+ public static boolean isToolInitialized()
+ {
+ return toolInitialized;
+ }
+
+ public static boolean isClientOrToolInitialized()
+ {
+ return clientInitialized || toolInitialized;
+ }
+
+ public static boolean isDaemonInitialized()
+ {
+ return daemonInitialized;
}
+ public static Config getRawConfig()
+ {
+ return conf;
+ }
+
+ @VisibleForTesting
public static Config loadConfig() throws ConfigurationException
{
if (Config.getOverrideLoadConfig() != null)
@@@ -2462,71 -2109,8 +2465,76 @@@
return conf.gc_warn_threshold_in_ms;
}
+ public static boolean isCDCEnabled()
+ {
+ return conf.cdc_enabled;
+ }
+
+ public static void setCDCEnabled(boolean cdc_enabled)
+ {
+ conf.cdc_enabled = cdc_enabled;
+ }
+
+ public static String getCDCLogLocation()
+ {
+ return conf.cdc_raw_directory;
+ }
+
+ public static int getCDCSpaceInMB()
+ {
+ return conf.cdc_total_space_in_mb;
+ }
+
+ @VisibleForTesting
+ public static void setCDCSpaceInMB(int input)
+ {
+ conf.cdc_total_space_in_mb = input;
+ }
+
+ public static int getCDCDiskCheckInterval()
+ {
+ return conf.cdc_free_space_check_interval_ms;
+ }
+
+ @VisibleForTesting
+ public static void setEncryptionContext(EncryptionContext ec)
+ {
+ encryptionContext = ec;
+ }
+
+ public static int searchConcurrencyFactor()
+ {
+ return searchConcurrencyFactor;
+ }
+
+ public static boolean isUnsafeSystem()
+ {
+ return unsafeSystem;
+ }
+
+ public static void setBackPressureEnabled(boolean backPressureEnabled)
+ {
+ conf.back_pressure_enabled = backPressureEnabled;
+ }
+
+ public static boolean backPressureEnabled()
+ {
+ return conf.back_pressure_enabled;
+ }
+
+ @VisibleForTesting
+ public static void setBackPressureStrategy(BackPressureStrategy strategy)
+ {
+ backPressureStrategy = strategy;
+ }
+
+ public static BackPressureStrategy getBackPressureStrategy()
+ {
+ return backPressureStrategy;
+ }
++
+ public static boolean strictRuntimeChecks()
+ {
+ return strictRuntimeChecks;
+ }
}
diff --cc src/java/org/apache/cassandra/gms/Gossiper.java
index 0fd52ca,4ea0a4a..5d2e997
--- a/src/java/org/apache/cassandra/gms/Gossiper.java
+++ b/src/java/org/apache/cassandra/gms/Gossiper.java
@@@ -24,14 -23,17 +24,18 @@@ import java.util.*
import java.util.Map.Entry;
import java.util.concurrent.*;
import java.util.concurrent.locks.ReentrantLock;
+import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
+ import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+ import com.google.common.util.concurrent.ListenableFutureTask;
import com.google.common.util.concurrent.Uninterruptibles;
+ import io.netty.util.concurrent.FastThreadLocal;
import org.apache.cassandra.utils.MBeanWrapper;
+ import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --cc src/java/org/apache/cassandra/service/StorageService.java
index 8f4b1e7,a1f361d..e64cbaa
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@@ -712,42 -699,6 +712,42 @@@ public class StorageService extends Not
doAuthSetup();
logger.info("Not joining ring as requested. Use JMX (StorageService->joinRing()) to initiate ring joining");
}
+
+ initialized = true;
+ }
+
+ private void loadRingState()
+ {
+ if (Boolean.parseBoolean(System.getProperty("cassandra.load_ring_state", "true")))
+ {
+ logger.info("Loading persisted ring state");
+ Multimap<InetAddress, Token> loadedTokens = SystemKeyspace.loadTokens();
+ Map<InetAddress, UUID> loadedHostIds = SystemKeyspace.loadHostIds();
+ for (InetAddress ep : loadedTokens.keySet())
+ {
+ if (ep.equals(FBUtilities.getBroadcastAddress()))
+ {
+ // entry has been mistakenly added, delete it
+ SystemKeyspace.removeEndpoint(ep);
+ }
+ else
+ {
+ if (loadedHostIds.containsKey(ep))
+ tokenMetadata.updateHostId(loadedHostIds.get(ep), ep);
- Gossiper.instance.addSavedEndpoint(ep);
++ Gossiper.runInGossipStageBlocking(() -> Gossiper.instance.addSavedEndpoint(ep));
+ }
+ }
+ }
+ }
+
+ private boolean isReplacing()
+ {
+ if (System.getProperty("cassandra.replace_address_first_boot", null) != null && SystemKeyspace.bootstrapComplete())
+ {
+ logger.info("Replace address on first boot requested; this node is already bootstrapped");
+ return false;
+ }
+ return DatabaseDescriptor.getReplaceAddress() != null;
}
/**
diff --cc test/distributed/org/apache/cassandra/distributed/impl/Instance.java
index 07fd35d,94dbc96..382388b
--- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
@@@ -349,15 -348,18 +349,19 @@@ public class Instance extends IsolatedE
for (int i = 0; i < tokens.size(); i++)
{
InetAddressAndPort ep = hosts.get(i);
- Gossiper.instance.initializeNodeUnsafe(ep.address, hostIds.get(i), 1);
- Gossiper.instance.injectApplicationState(ep.address,
- ApplicationState.TOKENS,
- new VersionedValue.VersionedValueFactory(partitioner).tokens(Collections.singleton(tokens.get(i))));
- storageService.onChange(ep.address,
- ApplicationState.STATUS,
- new VersionedValue.VersionedValueFactory(partitioner).normal(Collections.singleton(tokens.get(i))));
- Gossiper.instance.realMarkAlive(ep.address, Gossiper.instance.getEndpointStateForEndpoint(ep.address));
+ UUID hostId = hostIds.get(i);
+ Token token = tokens.get(i);
+ Gossiper.runInGossipStageBlocking(() -> {
+ Gossiper.instance.initializeNodeUnsafe(ep.address, hostId, 1);
+ Gossiper.instance.injectApplicationState(ep.address,
+ ApplicationState.TOKENS,
+ new VersionedValue.VersionedValueFactory(partitioner).tokens(Collections.singleton(token)));
+ storageService.onChange(ep.address,
+ ApplicationState.STATUS,
+ new VersionedValue.VersionedValueFactory(partitioner).normal(Collections.singleton(token)));
+ Gossiper.instance.realMarkAlive(ep.address, Gossiper.instance.getEndpointStateForEndpoint(ep.address));
+ });
+
int version = Math.min(MessagingService.current_version, cluster.get(ep).getMessagingVersion());
MessagingService.instance().setVersion(ep.address, version);
}
diff --cc test/unit/org/apache/cassandra/gms/GossiperTest.java
index def0530,f23c016..448620a
--- a/test/unit/org/apache/cassandra/gms/GossiperTest.java
+++ b/test/unit/org/apache/cassandra/gms/GossiperTest.java
@@@ -47,16 -45,11 +47,17 @@@ import static org.junit.Assert.assertTr
public class GossiperTest
{
- static
+ @BeforeClass
+ public static void before()
{
+ System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
- DatabaseDescriptor.setDaemonInitialized();
+ DatabaseDescriptor.daemonInitialization();
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace("schema_test_ks",
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD("schema_test_ks", "schema_test_cf"));
}
+
static final IPartitioner partitioner = new RandomPartitioner();
StorageService ss = StorageService.instance;
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
diff --cc test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java
index 34f8204,5dc34df..bc3e837
--- a/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/CloudstackSnitchTest.java
@@@ -45,7 -45,8 +45,8 @@@ public class CloudstackSnitchTes
@BeforeClass
public static void setup() throws Exception
{
+ System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
- DatabaseDescriptor.setDaemonInitialized();
+ DatabaseDescriptor.daemonInitialization();
SchemaLoader.mkdirs();
SchemaLoader.cleanup();
Keyspace.setInitialized();
diff --cc test/unit/org/apache/cassandra/locator/EC2SnitchTest.java
index 5e99523,9d078ce..0c71c92
--- a/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/EC2SnitchTest.java
@@@ -50,7 -50,8 +50,8 @@@ public class EC2SnitchTes
@BeforeClass
public static void setup() throws Exception
{
+ System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
- DatabaseDescriptor.setDaemonInitialized();
+ DatabaseDescriptor.daemonInitialization();
SchemaLoader.mkdirs();
SchemaLoader.cleanup();
Keyspace.setInitialized();
diff --cc test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java
index c63044c,04b71e9..2491ba9
--- a/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java
+++ b/test/unit/org/apache/cassandra/locator/GoogleCloudSnitchTest.java
@@@ -46,7 -46,8 +46,8 @@@ public class GoogleCloudSnitchTes
@BeforeClass
public static void setup() throws Exception
{
+ System.setProperty(Gossiper.Props.DISABLE_THREAD_VALIDATION, "true");
- DatabaseDescriptor.setDaemonInitialized();
+ DatabaseDescriptor.daemonInitialization();
SchemaLoader.mkdirs();
SchemaLoader.cleanup();
Keyspace.setInitialized();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org