You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jz...@apache.org on 2019/01/15 17:55:19 UTC
[cassandra-dtest] branch master updated: Check nativetransport
while bootstrap not complete
This is an automated email from the ASF dual-hosted git repository.
jzhuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/cassandra-dtest.git
The following commit(s) were added to refs/heads/master by this push:
new e6f58cb Check nativetransport while bootstrap not complete
e6f58cb is described below
commit e6f58cb33f7a09f273c5990d5d21c7b529ba80bf
Author: jaydeepkumar1984 <ch...@gmail.com>
AuthorDate: Sun Jun 17 22:01:46 2018 -0700
Check nativetransport while bootstrap not complete
patch by Jaydeepkumar Chovatia; reviewed by Jay Zhuang for CASSANDRA-14526
---
bootstrap_test.py | 103 +++++++++++++++++++++++++++++++++-----
byteman/pre4.0/stream_failure.btm | 2 +-
secondary_indexes_test.py | 5 +-
tools/assertions.py | 6 ++-
4 files changed, 98 insertions(+), 18 deletions(-)
diff --git a/bootstrap_test.py b/bootstrap_test.py
index 6e7682f..e33749e 100644
--- a/bootstrap_test.py
+++ b/bootstrap_test.py
@@ -10,7 +10,8 @@ import signal
from cassandra import ConsistencyLevel
from cassandra.concurrent import execute_concurrent_with_args
-from ccmlib.node import NodeError
+from ccmlib.node import NodeError, TimeoutError, ToolError
+from ccmlib.node import TimeoutError
import pytest
@@ -20,12 +21,14 @@ from tools.assertions import (assert_almost_equal, assert_bootstrap_state, asser
from tools.data import query_c1c2
from tools.intervention import InterruptBootstrap, KillOnBootstrap
from tools.misc import new_node
-from tools.misc import generate_ssl_stores, retry_till_success
+from tools.misc import generate_ssl_stores
since = pytest.mark.since
logger = logging.getLogger(__name__)
class TestBootstrap(Tester):
+ byteman_submit_path_pre_4_0 = './byteman/pre4.0/stream_failure.btm'
+ byteman_submit_path_4_0 = './byteman/4.0/stream_failure.btm'
@pytest.fixture(autouse=True)
def fixture_add_additional_log_patterns(self, fixture_dtest_setup):
@@ -308,26 +311,22 @@ class TestBootstrap(Tester):
cluster.start(wait_other_notice=True)
# kill stream to node3 in the middle of streaming to let it fail
if cluster.version() < '4.0':
- node1.byteman_submit(['./byteman/pre4.0/stream_failure.btm'])
+ node1.byteman_submit([self.byteman_submit_path_pre_4_0])
else:
- node1.byteman_submit(['./byteman/4.0/stream_failure.btm'])
+ node1.byteman_submit([self.byteman_submit_path_4_0])
node1.stress(['write', 'n=1K', 'no-warmup', 'cl=TWO', '-schema', 'replication(factor=2)', '-rate', 'threads=50'])
cluster.flush()
# start bootstrapping node3 and wait for streaming
node3 = new_node(cluster)
- node3.start(wait_other_notice=False, wait_for_binary_proto=True)
+ node3.start(wait_other_notice=False)
- # wait for node3 ready to query
- node3.watch_log_for("Starting listening for CQL clients")
- mark = node3.mark_log()
- # check if node3 is still in bootstrap mode
- retry_till_success(assert_bootstrap_state, tester=self, node=node3, expected_bootstrap_state='IN_PROGRESS', timeout=120)
+ # let streaming fail as we expect
+ node3.watch_log_for('Some data streaming failed')
- # bring back node1 and invoke nodetool bootstrap to resume bootstrapping
+ # bring back node3 and invoke nodetool bootstrap to resume bootstrapping
node3.nodetool('bootstrap resume')
-
- node3.watch_log_for("Resume complete", from_mark=mark)
+ node3.wait_for_binary_interface()
assert_bootstrap_state(self, node3, 'COMPLETED')
# cleanup to guarantee each node will only have sstables of its ranges
@@ -706,3 +705,81 @@ class TestBootstrap(Tester):
logger.debug("Deleting {}".format(data_dir))
shutil.rmtree(data_dir)
shutil.rmtree(commitlog_dir)
+
+ @since('2.2')
+ def test_bootstrap_binary_disabled(self):
+ """
+ Test binary while bootstrapping and streaming fails
+ @jira_ticket CASSANDRA-14526, CASSANDRA-14525
+ """
+ config = {'authenticator': 'org.apache.cassandra.auth.PasswordAuthenticator',
+ 'authorizer': 'org.apache.cassandra.auth.CassandraAuthorizer',
+ 'role_manager': 'org.apache.cassandra.auth.CassandraRoleManager',
+ 'permissions_validity_in_ms': 0,
+ 'roles_validity_in_ms': 0}
+
+ cluster = self.cluster
+ cluster.populate(1)
+
+ node1 = cluster.nodes['node1']
+ # set up byteman
+ node1.byteman_port = '8100'
+ node1.import_config_files()
+
+ cluster.start(wait_other_notice=True)
+ # kill stream to node2 in the middle of streaming to let it fail
+ if cluster.version() < '4.0':
+ node1.byteman_submit([self.byteman_submit_path_pre_4_0])
+ else:
+ node1.byteman_submit([self.byteman_submit_path_4_0])
+ node1.stress(['write', 'n=1K', 'no-warmup', 'cl=ONE', '-schema', 'replication(factor=3)', '-rate', 'threads=50', '-mode', 'native', 'cql3', 'user=cassandra', 'password=cassandra'])
+ cluster.flush()
+
+ # start bootstrapping node2 and wait for streaming
+ node2 = new_node(cluster)
+ node2.set_configuration_options(values=config)
+ node2.byteman_port = '8101' # set for when we add node3
+ node2.import_config_files()
+ node2.start(jvm_args=["-Dcassandra.ring_delay_ms=5000"], wait_other_notice=True)
+ self.assert_log_had_msg(node2, 'Some data streaming failed', timeout=30)
+ self.assert_log_had_msg(node2, 'Not starting client transports as bootstrap has not completed', timeout=30)
+
+ try:
+ node2.nodetool('join')
+ pytest.fail('nodetool should have errored and failed to join ring')
+ except ToolError as t:
+ assert "Cannot join the ring until bootstrap completes" in t.stdout
+
+ node2.nodetool('bootstrap resume')
+ node2.wait_for_binary_interface()
+ assert_bootstrap_state(self, node2, 'COMPLETED', user='cassandra', password='cassandra')
+
+ # Test write survey behaviour
+ node3 = new_node(cluster)
+ node3.set_configuration_options(values=config)
+
+ # kill stream to node3 in the middle of streaming to let it fail
+ if cluster.version() < '4.0':
+ node1.byteman_submit([self.byteman_submit_path_pre_4_0])
+ node2.byteman_submit([self.byteman_submit_path_pre_4_0])
+ else:
+ node1.byteman_submit([self.byteman_submit_path_4_0])
+ node2.byteman_submit([self.byteman_submit_path_4_0])
+ node3.start(jvm_args=["-Dcassandra.write_survey=true", "-Dcassandra.ring_delay_ms=5000"], wait_other_notice=True)
+ self.assert_log_had_msg(node3, 'Some data streaming failed', timeout=30)
+ self.assert_log_had_msg(node3, "Not starting client transports in write_survey mode as it's bootstrapping or auth is enabled", timeout=30)
+
+ try:
+ node3.nodetool('join')
+ pytest.fail('nodetool should have errored and failed to join ring')
+ except ToolError as t:
+ assert "Cannot join the ring until bootstrap completes" in t.stdout
+
+ node3.nodetool('bootstrap resume')
+ self.assert_log_had_msg(node3, "Not starting client transports in write_survey mode as it's bootstrapping or auth is enabled", timeout=30)
+
+ # Should succeed in joining
+ node3.nodetool('join')
+ self.assert_log_had_msg(node3, "Leaving write survey mode and joining ring at operator request", timeout=30)
+ assert_bootstrap_state(self, node3, 'COMPLETED', user='cassandra', password='cassandra')
+ node3.wait_for_binary_interface(timeout=30)
\ No newline at end of file
diff --git a/byteman/pre4.0/stream_failure.btm b/byteman/pre4.0/stream_failure.btm
index 23c63cd..9546572 100644
--- a/byteman/pre4.0/stream_failure.btm
+++ b/byteman/pre4.0/stream_failure.btm
@@ -7,7 +7,7 @@
RULE inject stream failure
CLASS org.apache.cassandra.streaming.StreamSession
METHOD prepare
-AT INVOKE startStreamingFiles
+AT INVOKE maybeCompleted
BIND peer = $0.peer
# set flag to only run this rule once.
IF NOT flagged("done")
diff --git a/secondary_indexes_test.py b/secondary_indexes_test.py
index cccd27a..c8b5fd8 100644
--- a/secondary_indexes_test.py
+++ b/secondary_indexes_test.py
@@ -1204,10 +1204,11 @@ class TestPreJoinCallback(Tester):
yaml_opts['streaming_socket_timeout_in_ms'] = 1000
node2.set_configuration_options(values=yaml_opts)
- node2.start(wait_other_notice=False, wait_for_binary_proto=True)
- assert_bootstrap_state(self, node2, 'IN_PROGRESS')
+ node2.start(wait_other_notice=True, wait_for_binary_proto=False)
+ node2.watch_log_for('Some data streaming failed. Use nodetool to check bootstrap state and resume.')
node2.nodetool("bootstrap resume")
+ node2.watch_log_for('Starting listening for CQL clients')
assert_bootstrap_state(self, node2, 'COMPLETED')
assert node2.grep_log('Executing pre-join post-bootstrap tasks')
diff --git a/tools/assertions.py b/tools/assertions.py
index d91e6fd..1b82251 100644
--- a/tools/assertions.py
+++ b/tools/assertions.py
@@ -311,17 +311,19 @@ def assert_stderr_clean(err, acceptable_errors=None):
"stderr is {}".format(err_str, regex_str)
-def assert_bootstrap_state(tester, node, expected_bootstrap_state):
+def assert_bootstrap_state(tester, node, expected_bootstrap_state, user=None, password=None):
"""
Assert that a node is on a given bootstrap state
@param tester The dtest.Tester object to fetch the exclusive connection to the node
@param node The node to check bootstrap state
@param expected_bootstrap_state Bootstrap state to expect
+ @param user To connect as for authenticated nodes
+ @param password for corresponding user
Examples:
assert_bootstrap_state(self, node3, 'COMPLETED')
"""
- session = tester.patient_exclusive_cql_connection(node)
+ session = tester.patient_exclusive_cql_connection(node, user=user, password=password)
assert_one(session, "SELECT bootstrapped FROM system.local WHERE key='local'", [expected_bootstrap_state])
session.shutdown()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org