You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by st...@apache.org on 2016/06/10 21:00:27 UTC
[01/10] cassandra git commit: cqlsh COPY FROM: shutdown parent
cluster after forking, to avoid corrupting SSL connections
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 06bb6b9d0 -> 68319f7c3
refs/heads/cassandra-2.2 1dffa0225 -> 593bbf57d
refs/heads/cassandra-3.0 c59897b6c -> 3d211e9fb
refs/heads/trunk f0613bf6d -> db8df9153
cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections
patch by Stefania Alborghetti; reviewed by Tyler Hobbs for CASSANDRA-11749
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/68319f7c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/68319f7c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/68319f7c
Branch: refs/heads/cassandra-2.1
Commit: 68319f7c3be232a58e68ca91206283076aa3dedb
Parents: 06bb6b9
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Fri May 27 11:00:27 2016 +0800
Committer: Stefania Alborghetti <st...@datastax.com>
Committed: Fri Jun 10 15:49:51 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
pylib/cqlshlib/copyutil.py | 38 +++++++++++++++++++++++++++++++++++---
2 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68319f7c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 619dc61..af641e1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.15
+ * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
* Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
* Backport CASSANDRA-11578 (CASSANDRA-11750)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68319f7c/pylib/cqlshlib/copyutil.py
----------------------------------------------------------------------
diff --git a/pylib/cqlshlib/copyutil.py b/pylib/cqlshlib/copyutil.py
index d68812c..0016dfd 100644
--- a/pylib/cqlshlib/copyutil.py
+++ b/pylib/cqlshlib/copyutil.py
@@ -59,6 +59,7 @@ PROFILE_ON = False
STRACE_ON = False
DEBUG = False # This may be set to True when initializing the task
IS_LINUX = platform.system() == 'Linux'
+IS_WINDOWS = platform.system() == 'Windows'
CopyOptions = namedtuple('CopyOptions', 'copy dialect unrecognized')
@@ -421,9 +422,13 @@ class CopyTask(object):
def make_params(self):
"""
Return a dictionary of parameters to be used by the worker processes.
- On Windows this dictionary must be pickle-able.
+ On Windows this dictionary must be pickle-able, therefore we do not pass the
+ parent connection since it may not be pickle-able. Also, on Windows child
+ processes are spawned and not forked, and therefore we don't need to shutdown
+ the parent connection anyway, see CASSANDRA-11749 for more details.
"""
shell = self.shell
+
return dict(ks=self.ks,
table=self.table,
local_dc=self.local_dc,
@@ -434,6 +439,7 @@ class CopyTask(object):
port=shell.port,
ssl=shell.ssl,
auth_provider=shell.auth_provider,
+ parent_cluster=shell.conn if not IS_WINDOWS else None,
cql_version=shell.conn.cql_version,
config_file=self.config_file,
protocol_version=self.protocol_version,
@@ -1072,7 +1078,8 @@ class ImportTask(CopyTask):
self.processes.append(ImportProcess(self.update_params(params, i)))
feeder = FeedingProcess(self.outmsg.channels[-1], self.inmsg.channels[-1],
- self.outmsg.channels[:-1], self.fname, self.options)
+ self.outmsg.channels[:-1], self.fname, self.options,
+ self.shell.conn if not IS_WINDOWS else None)
self.processes.append(feeder)
self.start_processes()
@@ -1179,7 +1186,7 @@ class FeedingProcess(mp.Process):
"""
A process that reads from import sources and sends chunks to worker processes.
"""
- def __init__(self, inmsg, outmsg, worker_channels, fname, options):
+ def __init__(self, inmsg, outmsg, worker_channels, fname, options, parent_cluster):
mp.Process.__init__(self, target=self.run)
self.inmsg = inmsg
self.outmsg = outmsg
@@ -1189,6 +1196,15 @@ class FeedingProcess(mp.Process):
self.ingest_rate = options.copy['ingestrate']
self.num_worker_processes = options.copy['numprocesses']
self.chunk_id = 0
+ self.parent_cluster = parent_cluster
+
+ def on_fork(self):
+ """
+ Release any parent connections after forking, see CASSANDRA-11749 for details.
+ """
+ if self.parent_cluster:
+ printdebugmsg("Closing parent cluster sockets")
+ self.parent_cluster.shutdown()
def run(self):
pr = profile_on() if PROFILE_ON else None
@@ -1205,6 +1221,9 @@ class FeedingProcess(mp.Process):
here we throttle using the ingest rate in the feeding process because of memory usage concerns.
When finished we send back to the parent process the total number of rows sent.
"""
+
+ self.on_fork()
+
reader = self.reader
reader.start()
channels = self.worker_channels
@@ -1268,6 +1287,7 @@ class ChildProcess(mp.Process):
self.connect_timeout = params['connect_timeout']
self.cql_version = params['cql_version']
self.auth_provider = params['auth_provider']
+ self.parent_cluster = params['parent_cluster']
self.ssl = params['ssl']
self.protocol_version = params['protocol_version']
self.config_file = params['config_file']
@@ -1285,6 +1305,14 @@ class ChildProcess(mp.Process):
else:
self.test_failures = None
+ def on_fork(self):
+ """
+ Release any parent connections after forking, see CASSANDRA-11749 for details.
+ """
+ if self.parent_cluster:
+ printdebugmsg("Closing parent cluster sockets")
+ self.parent_cluster.shutdown()
+
def close(self):
printdebugmsg("Closing queues...")
self.inmsg.close()
@@ -1411,6 +1439,9 @@ class ExportProcess(ChildProcess):
we can signal a global error by sending (None, error).
We terminate when the inbound queue is closed.
"""
+
+ self.on_fork()
+
while True:
if self.num_requests() > self.max_requests:
time.sleep(0.001) # 1 millisecond
@@ -2059,6 +2090,7 @@ class ImportProcess(ChildProcess):
try:
pr = profile_on() if PROFILE_ON else None
+ self.on_fork()
self.inner_run(*self.make_params())
if pr:
[05/10] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by st...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/593bbf57
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/593bbf57
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/593bbf57
Branch: refs/heads/cassandra-3.0
Commit: 593bbf57dd2e87df031d13edb2fad8234610521e
Parents: 1dffa02 68319f7
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Fri Jun 10 15:51:11 2016 -0500
Committer: Stefania Alborghetti <st...@datastax.com>
Committed: Fri Jun 10 15:52:44 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
pylib/cqlshlib/copyutil.py | 38 +++++++++++++++++++++++++++++++++++---
2 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/593bbf57/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 7ec3ae9,af641e1..d639d43
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,30 -1,6 +1,31 @@@
-2.1.15
+2.2.7
+ * StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
+ * Persist local metadata earlier in startup sequence (CASSANDRA-11742)
+ * Run CommitLog tests with different compression settings (CASSANDRA-9039)
+ * cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
+ * Avoid showing estimated key as -1 in tablestats (CASSANDRA-11587)
+ * Fix possible race condition in CommitLog.recover (CASSANDRA-11743)
+ * Enable client encryption in sstableloader with cli options (CASSANDRA-11708)
+ * Possible memory leak in NIODataInputStream (CASSANDRA-11867)
+ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+ * Add seconds to cqlsh tracing session duration (CASSANDRA-11753)
+ * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
+ * cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
+ * Exit JVM if JMX server fails to startup (CASSANDRA-11540)
+ * Produce a heap dump when exiting on OOM (CASSANDRA-9861)
+ * Avoid read repairing purgeable tombstones on range slices (CASSANDRA-11427)
+ * Restore ability to filter on clustering columns when using a 2i (CASSANDRA-11510)
+ * JSON datetime formatting needs timezone (CASSANDRA-11137)
+ * Fix is_dense recalculation for Thrift-updated tables (CASSANDRA-11502)
+ * Remove unnescessary file existence check during anticompaction (CASSANDRA-11660)
+ * Add missing files to debian packages (CASSANDRA-11642)
+ * Avoid calling Iterables::concat in loops during ModificationStatement::getFunctions (CASSANDRA-11621)
+ * cqlsh: COPY FROM should use regular inserts for single statement batches and
+ report errors correctly if workers processes crash on initialization (CASSANDRA-11474)
+ * Always close cluster with connection in CqlRecordWriter (CASSANDRA-11553)
+ * Fix slice queries on ordered COMPACT tables (CASSANDRA-10988)
+Merged from 2.1:
+ * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
- * Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
* Backport CASSANDRA-11578 (CASSANDRA-11750)
* Clear out parent repair session if repair coordinator dies (CASSANDRA-11824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/593bbf57/pylib/cqlshlib/copyutil.py
----------------------------------------------------------------------
[10/10] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Posted by st...@apache.org.
Merge branch 'cassandra-3.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/db8df915
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/db8df915
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/db8df915
Branch: refs/heads/trunk
Commit: db8df91530512cc192b95252ccad89f0edee8540
Parents: f0613bf 3d211e9
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Fri Jun 10 15:56:47 2016 -0500
Committer: Stefania Alborghetti <st...@datastax.com>
Committed: Fri Jun 10 15:56:47 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 3 +++
pylib/cqlshlib/copyutil.py | 38 +++++++++++++++++++++++++++++++++++---
2 files changed, 38 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/db8df915/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index d699f93,47aef7e..a944bd1
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -14,11 -2,11 +14,14 @@@ Merged from 3.0
* Add TimeWindowCompactionStrategy (CASSANDRA-9666)
Merged from 2.2:
* StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
+ Merged from 2.1:
+ * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
+
-3.0.7
+3.7
+ * Support multiple folders for user defined compaction tasks (CASSANDRA-11765)
+ * Fix race in CompactionStrategyManager's pause/resume (CASSANDRA-11922)
+Merged from 3.0:
* Fix legacy serialization of Thrift-generated non-compound range tombstones
when communicating with 2.x nodes (CASSANDRA-11930)
* Fix Directories instantiations where CFS.initialDirectories should be used (CASSANDRA-11849)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/db8df915/pylib/cqlshlib/copyutil.py
----------------------------------------------------------------------
[04/10] cassandra git commit: cqlsh COPY FROM: shutdown parent
cluster after forking, to avoid corrupting SSL connections
Posted by st...@apache.org.
cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections
patch by Stefania Alborghetti; reviewed by Tyler Hobbs for CASSANDRA-11749
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/68319f7c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/68319f7c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/68319f7c
Branch: refs/heads/trunk
Commit: 68319f7c3be232a58e68ca91206283076aa3dedb
Parents: 06bb6b9
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Fri May 27 11:00:27 2016 +0800
Committer: Stefania Alborghetti <st...@datastax.com>
Committed: Fri Jun 10 15:49:51 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
pylib/cqlshlib/copyutil.py | 38 +++++++++++++++++++++++++++++++++++---
2 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68319f7c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 619dc61..af641e1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.15
+ * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
* Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
* Backport CASSANDRA-11578 (CASSANDRA-11750)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68319f7c/pylib/cqlshlib/copyutil.py
----------------------------------------------------------------------
diff --git a/pylib/cqlshlib/copyutil.py b/pylib/cqlshlib/copyutil.py
index d68812c..0016dfd 100644
--- a/pylib/cqlshlib/copyutil.py
+++ b/pylib/cqlshlib/copyutil.py
@@ -59,6 +59,7 @@ PROFILE_ON = False
STRACE_ON = False
DEBUG = False # This may be set to True when initializing the task
IS_LINUX = platform.system() == 'Linux'
+IS_WINDOWS = platform.system() == 'Windows'
CopyOptions = namedtuple('CopyOptions', 'copy dialect unrecognized')
@@ -421,9 +422,13 @@ class CopyTask(object):
def make_params(self):
"""
Return a dictionary of parameters to be used by the worker processes.
- On Windows this dictionary must be pickle-able.
+ On Windows this dictionary must be pickle-able, therefore we do not pass the
+ parent connection since it may not be pickle-able. Also, on Windows child
+ processes are spawned and not forked, and therefore we don't need to shutdown
+ the parent connection anyway, see CASSANDRA-11749 for more details.
"""
shell = self.shell
+
return dict(ks=self.ks,
table=self.table,
local_dc=self.local_dc,
@@ -434,6 +439,7 @@ class CopyTask(object):
port=shell.port,
ssl=shell.ssl,
auth_provider=shell.auth_provider,
+ parent_cluster=shell.conn if not IS_WINDOWS else None,
cql_version=shell.conn.cql_version,
config_file=self.config_file,
protocol_version=self.protocol_version,
@@ -1072,7 +1078,8 @@ class ImportTask(CopyTask):
self.processes.append(ImportProcess(self.update_params(params, i)))
feeder = FeedingProcess(self.outmsg.channels[-1], self.inmsg.channels[-1],
- self.outmsg.channels[:-1], self.fname, self.options)
+ self.outmsg.channels[:-1], self.fname, self.options,
+ self.shell.conn if not IS_WINDOWS else None)
self.processes.append(feeder)
self.start_processes()
@@ -1179,7 +1186,7 @@ class FeedingProcess(mp.Process):
"""
A process that reads from import sources and sends chunks to worker processes.
"""
- def __init__(self, inmsg, outmsg, worker_channels, fname, options):
+ def __init__(self, inmsg, outmsg, worker_channels, fname, options, parent_cluster):
mp.Process.__init__(self, target=self.run)
self.inmsg = inmsg
self.outmsg = outmsg
@@ -1189,6 +1196,15 @@ class FeedingProcess(mp.Process):
self.ingest_rate = options.copy['ingestrate']
self.num_worker_processes = options.copy['numprocesses']
self.chunk_id = 0
+ self.parent_cluster = parent_cluster
+
+ def on_fork(self):
+ """
+ Release any parent connections after forking, see CASSANDRA-11749 for details.
+ """
+ if self.parent_cluster:
+ printdebugmsg("Closing parent cluster sockets")
+ self.parent_cluster.shutdown()
def run(self):
pr = profile_on() if PROFILE_ON else None
@@ -1205,6 +1221,9 @@ class FeedingProcess(mp.Process):
here we throttle using the ingest rate in the feeding process because of memory usage concerns.
When finished we send back to the parent process the total number of rows sent.
"""
+
+ self.on_fork()
+
reader = self.reader
reader.start()
channels = self.worker_channels
@@ -1268,6 +1287,7 @@ class ChildProcess(mp.Process):
self.connect_timeout = params['connect_timeout']
self.cql_version = params['cql_version']
self.auth_provider = params['auth_provider']
+ self.parent_cluster = params['parent_cluster']
self.ssl = params['ssl']
self.protocol_version = params['protocol_version']
self.config_file = params['config_file']
@@ -1285,6 +1305,14 @@ class ChildProcess(mp.Process):
else:
self.test_failures = None
+ def on_fork(self):
+ """
+ Release any parent connections after forking, see CASSANDRA-11749 for details.
+ """
+ if self.parent_cluster:
+ printdebugmsg("Closing parent cluster sockets")
+ self.parent_cluster.shutdown()
+
def close(self):
printdebugmsg("Closing queues...")
self.inmsg.close()
@@ -1411,6 +1439,9 @@ class ExportProcess(ChildProcess):
we can signal a global error by sending (None, error).
We terminate when the inbound queue is closed.
"""
+
+ self.on_fork()
+
while True:
if self.num_requests() > self.max_requests:
time.sleep(0.001) # 1 millisecond
@@ -2059,6 +2090,7 @@ class ImportProcess(ChildProcess):
try:
pr = profile_on() if PROFILE_ON else None
+ self.on_fork()
self.inner_run(*self.make_params())
if pr:
[07/10] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by st...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/593bbf57
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/593bbf57
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/593bbf57
Branch: refs/heads/trunk
Commit: 593bbf57dd2e87df031d13edb2fad8234610521e
Parents: 1dffa02 68319f7
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Fri Jun 10 15:51:11 2016 -0500
Committer: Stefania Alborghetti <st...@datastax.com>
Committed: Fri Jun 10 15:52:44 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
pylib/cqlshlib/copyutil.py | 38 +++++++++++++++++++++++++++++++++++---
2 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/593bbf57/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 7ec3ae9,af641e1..d639d43
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,30 -1,6 +1,31 @@@
-2.1.15
+2.2.7
+ * StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
+ * Persist local metadata earlier in startup sequence (CASSANDRA-11742)
+ * Run CommitLog tests with different compression settings (CASSANDRA-9039)
+ * cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
+ * Avoid showing estimated key as -1 in tablestats (CASSANDRA-11587)
+ * Fix possible race condition in CommitLog.recover (CASSANDRA-11743)
+ * Enable client encryption in sstableloader with cli options (CASSANDRA-11708)
+ * Possible memory leak in NIODataInputStream (CASSANDRA-11867)
+ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+ * Add seconds to cqlsh tracing session duration (CASSANDRA-11753)
+ * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
+ * cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
+ * Exit JVM if JMX server fails to startup (CASSANDRA-11540)
+ * Produce a heap dump when exiting on OOM (CASSANDRA-9861)
+ * Avoid read repairing purgeable tombstones on range slices (CASSANDRA-11427)
+ * Restore ability to filter on clustering columns when using a 2i (CASSANDRA-11510)
+ * JSON datetime formatting needs timezone (CASSANDRA-11137)
+ * Fix is_dense recalculation for Thrift-updated tables (CASSANDRA-11502)
+ * Remove unnescessary file existence check during anticompaction (CASSANDRA-11660)
+ * Add missing files to debian packages (CASSANDRA-11642)
+ * Avoid calling Iterables::concat in loops during ModificationStatement::getFunctions (CASSANDRA-11621)
+ * cqlsh: COPY FROM should use regular inserts for single statement batches and
+ report errors correctly if workers processes crash on initialization (CASSANDRA-11474)
+ * Always close cluster with connection in CqlRecordWriter (CASSANDRA-11553)
+ * Fix slice queries on ordered COMPACT tables (CASSANDRA-10988)
+Merged from 2.1:
+ * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
- * Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
* Backport CASSANDRA-11578 (CASSANDRA-11750)
* Clear out parent repair session if repair coordinator dies (CASSANDRA-11824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/593bbf57/pylib/cqlshlib/copyutil.py
----------------------------------------------------------------------
[08/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by st...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3d211e9f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3d211e9f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3d211e9f
Branch: refs/heads/cassandra-3.0
Commit: 3d211e9fbf1c4c61fffe7f589d64dd5ca7074c48
Parents: c59897b 593bbf5
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Fri Jun 10 15:54:22 2016 -0500
Committer: Stefania Alborghetti <st...@datastax.com>
Committed: Fri Jun 10 15:56:15 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 3 +++
pylib/cqlshlib/copyutil.py | 38 +++++++++++++++++++++++++++++++++++---
2 files changed, 38 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d211e9f/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index fd2fe79,d639d43..47aef7e
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,25 -1,5 +1,28 @@@
-2.2.7
+3.0.8
+ * Add TimeWindowCompactionStrategy (CASSANDRA-9666)
+Merged from 2.2:
* StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
++Merged from 2.1:
++ * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
++
+
+3.0.7
+ * Fix legacy serialization of Thrift-generated non-compound range tombstones
+ when communicating with 2.x nodes (CASSANDRA-11930)
+ * Fix Directories instantiations where CFS.initialDirectories should be used (CASSANDRA-11849)
+ * Avoid referencing DatabaseDescriptor in AbstractType (CASSANDRA-11912)
+ * Fix sstables not being protected from removal during index build (CASSANDRA-11905)
+ * cqlsh: Suppress stack trace from Read/WriteFailures (CASSANDRA-11032)
+ * Remove unneeded code to repair index summaries that have
+ been improperly down-sampled (CASSANDRA-11127)
+ * Avoid WriteTimeoutExceptions during commit log replay due to materialized
+ view lock contention (CASSANDRA-11891)
+ * Prevent OOM failures on SSTable corruption, improve tests for corruption detection (CASSANDRA-9530)
+ * Use CFS.initialDirectories when clearing snapshots (CASSANDRA-11705)
+ * Allow compaction strategies to disable early open (CASSANDRA-11754)
+ * Refactor Materialized View code (CASSANDRA-11475)
+ * Update Java Driver (CASSANDRA-11615)
+Merged from 2.2:
* Persist local metadata earlier in startup sequence (CASSANDRA-11742)
* Run CommitLog tests with different compression settings (CASSANDRA-9039)
* cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d211e9f/pylib/cqlshlib/copyutil.py
----------------------------------------------------------------------
diff --cc pylib/cqlshlib/copyutil.py
index d3ae1eb,a1adbaa..700b062
--- a/pylib/cqlshlib/copyutil.py
+++ b/pylib/cqlshlib/copyutil.py
@@@ -1222,8 -1223,16 +1229,17 @@@ class FeedingProcess(mp.Process)
self.send_meter = RateMeter(log_fcn=None, update_interval=1)
self.ingest_rate = options.copy['ingestrate']
self.num_worker_processes = options.copy['numprocesses']
+ self.max_pending_chunks = options.copy['maxpendingchunks']
self.chunk_id = 0
+ self.parent_cluster = parent_cluster
+
+ def on_fork(self):
+ """
+ Release any parent connections after forking, see CASSANDRA-11749 for details.
+ """
+ if self.parent_cluster:
+ printdebugmsg("Closing parent cluster sockets")
+ self.parent_cluster.shutdown()
def run(self):
pr = profile_on() if PROFILE_ON else None
[09/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by st...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3d211e9f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3d211e9f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3d211e9f
Branch: refs/heads/trunk
Commit: 3d211e9fbf1c4c61fffe7f589d64dd5ca7074c48
Parents: c59897b 593bbf5
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Fri Jun 10 15:54:22 2016 -0500
Committer: Stefania Alborghetti <st...@datastax.com>
Committed: Fri Jun 10 15:56:15 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 3 +++
pylib/cqlshlib/copyutil.py | 38 +++++++++++++++++++++++++++++++++++---
2 files changed, 38 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d211e9f/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index fd2fe79,d639d43..47aef7e
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,25 -1,5 +1,28 @@@
-2.2.7
+3.0.8
+ * Add TimeWindowCompactionStrategy (CASSANDRA-9666)
+Merged from 2.2:
* StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
++Merged from 2.1:
++ * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
++
+
+3.0.7
+ * Fix legacy serialization of Thrift-generated non-compound range tombstones
+ when communicating with 2.x nodes (CASSANDRA-11930)
+ * Fix Directories instantiations where CFS.initialDirectories should be used (CASSANDRA-11849)
+ * Avoid referencing DatabaseDescriptor in AbstractType (CASSANDRA-11912)
+ * Fix sstables not being protected from removal during index build (CASSANDRA-11905)
+ * cqlsh: Suppress stack trace from Read/WriteFailures (CASSANDRA-11032)
+ * Remove unneeded code to repair index summaries that have
+ been improperly down-sampled (CASSANDRA-11127)
+ * Avoid WriteTimeoutExceptions during commit log replay due to materialized
+ view lock contention (CASSANDRA-11891)
+ * Prevent OOM failures on SSTable corruption, improve tests for corruption detection (CASSANDRA-9530)
+ * Use CFS.initialDirectories when clearing snapshots (CASSANDRA-11705)
+ * Allow compaction strategies to disable early open (CASSANDRA-11754)
+ * Refactor Materialized View code (CASSANDRA-11475)
+ * Update Java Driver (CASSANDRA-11615)
+Merged from 2.2:
* Persist local metadata earlier in startup sequence (CASSANDRA-11742)
* Run CommitLog tests with different compression settings (CASSANDRA-9039)
* cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d211e9f/pylib/cqlshlib/copyutil.py
----------------------------------------------------------------------
diff --cc pylib/cqlshlib/copyutil.py
index d3ae1eb,a1adbaa..700b062
--- a/pylib/cqlshlib/copyutil.py
+++ b/pylib/cqlshlib/copyutil.py
@@@ -1222,8 -1223,16 +1229,17 @@@ class FeedingProcess(mp.Process)
self.send_meter = RateMeter(log_fcn=None, update_interval=1)
self.ingest_rate = options.copy['ingestrate']
self.num_worker_processes = options.copy['numprocesses']
+ self.max_pending_chunks = options.copy['maxpendingchunks']
self.chunk_id = 0
+ self.parent_cluster = parent_cluster
+
+ def on_fork(self):
+ """
+ Release any parent connections after forking, see CASSANDRA-11749 for details.
+ """
+ if self.parent_cluster:
+ printdebugmsg("Closing parent cluster sockets")
+ self.parent_cluster.shutdown()
def run(self):
pr = profile_on() if PROFILE_ON else None
[06/10] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by st...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/593bbf57
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/593bbf57
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/593bbf57
Branch: refs/heads/cassandra-2.2
Commit: 593bbf57dd2e87df031d13edb2fad8234610521e
Parents: 1dffa02 68319f7
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Fri Jun 10 15:51:11 2016 -0500
Committer: Stefania Alborghetti <st...@datastax.com>
Committed: Fri Jun 10 15:52:44 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
pylib/cqlshlib/copyutil.py | 38 +++++++++++++++++++++++++++++++++++---
2 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/593bbf57/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 7ec3ae9,af641e1..d639d43
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,30 -1,6 +1,31 @@@
-2.1.15
+2.2.7
+ * StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
+ * Persist local metadata earlier in startup sequence (CASSANDRA-11742)
+ * Run CommitLog tests with different compression settings (CASSANDRA-9039)
+ * cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
+ * Avoid showing estimated key as -1 in tablestats (CASSANDRA-11587)
+ * Fix possible race condition in CommitLog.recover (CASSANDRA-11743)
+ * Enable client encryption in sstableloader with cli options (CASSANDRA-11708)
+ * Possible memory leak in NIODataInputStream (CASSANDRA-11867)
+ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+ * Add seconds to cqlsh tracing session duration (CASSANDRA-11753)
+ * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
+ * cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
+ * Exit JVM if JMX server fails to startup (CASSANDRA-11540)
+ * Produce a heap dump when exiting on OOM (CASSANDRA-9861)
+ * Avoid read repairing purgeable tombstones on range slices (CASSANDRA-11427)
+ * Restore ability to filter on clustering columns when using a 2i (CASSANDRA-11510)
+ * JSON datetime formatting needs timezone (CASSANDRA-11137)
+ * Fix is_dense recalculation for Thrift-updated tables (CASSANDRA-11502)
+ * Remove unnescessary file existence check during anticompaction (CASSANDRA-11660)
+ * Add missing files to debian packages (CASSANDRA-11642)
+ * Avoid calling Iterables::concat in loops during ModificationStatement::getFunctions (CASSANDRA-11621)
+ * cqlsh: COPY FROM should use regular inserts for single statement batches and
+ report errors correctly if workers processes crash on initialization (CASSANDRA-11474)
+ * Always close cluster with connection in CqlRecordWriter (CASSANDRA-11553)
+ * Fix slice queries on ordered COMPACT tables (CASSANDRA-10988)
+Merged from 2.1:
+ * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
- * Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
* Backport CASSANDRA-11578 (CASSANDRA-11750)
* Clear out parent repair session if repair coordinator dies (CASSANDRA-11824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/593bbf57/pylib/cqlshlib/copyutil.py
----------------------------------------------------------------------
[02/10] cassandra git commit: cqlsh COPY FROM: shutdown parent
cluster after forking, to avoid corrupting SSL connections
Posted by st...@apache.org.
cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections
patch by Stefania Alborghetti; reviewed by Tyler Hobbs for CASSANDRA-11749
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/68319f7c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/68319f7c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/68319f7c
Branch: refs/heads/cassandra-2.2
Commit: 68319f7c3be232a58e68ca91206283076aa3dedb
Parents: 06bb6b9
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Fri May 27 11:00:27 2016 +0800
Committer: Stefania Alborghetti <st...@datastax.com>
Committed: Fri Jun 10 15:49:51 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
pylib/cqlshlib/copyutil.py | 38 +++++++++++++++++++++++++++++++++++---
2 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68319f7c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 619dc61..af641e1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.15
+ * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
* Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
* Backport CASSANDRA-11578 (CASSANDRA-11750)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68319f7c/pylib/cqlshlib/copyutil.py
----------------------------------------------------------------------
diff --git a/pylib/cqlshlib/copyutil.py b/pylib/cqlshlib/copyutil.py
index d68812c..0016dfd 100644
--- a/pylib/cqlshlib/copyutil.py
+++ b/pylib/cqlshlib/copyutil.py
@@ -59,6 +59,7 @@ PROFILE_ON = False
STRACE_ON = False
DEBUG = False # This may be set to True when initializing the task
IS_LINUX = platform.system() == 'Linux'
+IS_WINDOWS = platform.system() == 'Windows'
CopyOptions = namedtuple('CopyOptions', 'copy dialect unrecognized')
@@ -421,9 +422,13 @@ class CopyTask(object):
def make_params(self):
"""
Return a dictionary of parameters to be used by the worker processes.
- On Windows this dictionary must be pickle-able.
+ On Windows this dictionary must be pickle-able, therefore we do not pass the
+ parent connection since it may not be pickle-able. Also, on Windows child
+ processes are spawned and not forked, and therefore we don't need to shutdown
+ the parent connection anyway, see CASSANDRA-11749 for more details.
"""
shell = self.shell
+
return dict(ks=self.ks,
table=self.table,
local_dc=self.local_dc,
@@ -434,6 +439,7 @@ class CopyTask(object):
port=shell.port,
ssl=shell.ssl,
auth_provider=shell.auth_provider,
+ parent_cluster=shell.conn if not IS_WINDOWS else None,
cql_version=shell.conn.cql_version,
config_file=self.config_file,
protocol_version=self.protocol_version,
@@ -1072,7 +1078,8 @@ class ImportTask(CopyTask):
self.processes.append(ImportProcess(self.update_params(params, i)))
feeder = FeedingProcess(self.outmsg.channels[-1], self.inmsg.channels[-1],
- self.outmsg.channels[:-1], self.fname, self.options)
+ self.outmsg.channels[:-1], self.fname, self.options,
+ self.shell.conn if not IS_WINDOWS else None)
self.processes.append(feeder)
self.start_processes()
@@ -1179,7 +1186,7 @@ class FeedingProcess(mp.Process):
"""
A process that reads from import sources and sends chunks to worker processes.
"""
- def __init__(self, inmsg, outmsg, worker_channels, fname, options):
+ def __init__(self, inmsg, outmsg, worker_channels, fname, options, parent_cluster):
mp.Process.__init__(self, target=self.run)
self.inmsg = inmsg
self.outmsg = outmsg
@@ -1189,6 +1196,15 @@ class FeedingProcess(mp.Process):
self.ingest_rate = options.copy['ingestrate']
self.num_worker_processes = options.copy['numprocesses']
self.chunk_id = 0
+ self.parent_cluster = parent_cluster
+
+ def on_fork(self):
+ """
+ Release any parent connections after forking, see CASSANDRA-11749 for details.
+ """
+ if self.parent_cluster:
+ printdebugmsg("Closing parent cluster sockets")
+ self.parent_cluster.shutdown()
def run(self):
pr = profile_on() if PROFILE_ON else None
@@ -1205,6 +1221,9 @@ class FeedingProcess(mp.Process):
here we throttle using the ingest rate in the feeding process because of memory usage concerns.
When finished we send back to the parent process the total number of rows sent.
"""
+
+ self.on_fork()
+
reader = self.reader
reader.start()
channels = self.worker_channels
@@ -1268,6 +1287,7 @@ class ChildProcess(mp.Process):
self.connect_timeout = params['connect_timeout']
self.cql_version = params['cql_version']
self.auth_provider = params['auth_provider']
+ self.parent_cluster = params['parent_cluster']
self.ssl = params['ssl']
self.protocol_version = params['protocol_version']
self.config_file = params['config_file']
@@ -1285,6 +1305,14 @@ class ChildProcess(mp.Process):
else:
self.test_failures = None
+ def on_fork(self):
+ """
+ Release any parent connections after forking, see CASSANDRA-11749 for details.
+ """
+ if self.parent_cluster:
+ printdebugmsg("Closing parent cluster sockets")
+ self.parent_cluster.shutdown()
+
def close(self):
printdebugmsg("Closing queues...")
self.inmsg.close()
@@ -1411,6 +1439,9 @@ class ExportProcess(ChildProcess):
we can signal a global error by sending (None, error).
We terminate when the inbound queue is closed.
"""
+
+ self.on_fork()
+
while True:
if self.num_requests() > self.max_requests:
time.sleep(0.001) # 1 millisecond
@@ -2059,6 +2090,7 @@ class ImportProcess(ChildProcess):
try:
pr = profile_on() if PROFILE_ON else None
+ self.on_fork()
self.inner_run(*self.make_params())
if pr:
[03/10] cassandra git commit: cqlsh COPY FROM: shutdown parent
cluster after forking, to avoid corrupting SSL connections
Posted by st...@apache.org.
cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections
patch by Stefania Alborghetti; reviewed by Tyler Hobbs for CASSANDRA-11749
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/68319f7c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/68319f7c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/68319f7c
Branch: refs/heads/cassandra-3.0
Commit: 68319f7c3be232a58e68ca91206283076aa3dedb
Parents: 06bb6b9
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Fri May 27 11:00:27 2016 +0800
Committer: Stefania Alborghetti <st...@datastax.com>
Committed: Fri Jun 10 15:49:51 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
pylib/cqlshlib/copyutil.py | 38 +++++++++++++++++++++++++++++++++++---
2 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68319f7c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 619dc61..af641e1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.15
+ * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
* Updated cqlsh Python driver to fix DESCRIBE problem for legacy tables (CASSANDRA-11055)
* cqlsh: apply current keyspace to source command (CASSANDRA-11152)
* Backport CASSANDRA-11578 (CASSANDRA-11750)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68319f7c/pylib/cqlshlib/copyutil.py
----------------------------------------------------------------------
diff --git a/pylib/cqlshlib/copyutil.py b/pylib/cqlshlib/copyutil.py
index d68812c..0016dfd 100644
--- a/pylib/cqlshlib/copyutil.py
+++ b/pylib/cqlshlib/copyutil.py
@@ -59,6 +59,7 @@ PROFILE_ON = False
STRACE_ON = False
DEBUG = False # This may be set to True when initializing the task
IS_LINUX = platform.system() == 'Linux'
+IS_WINDOWS = platform.system() == 'Windows'
CopyOptions = namedtuple('CopyOptions', 'copy dialect unrecognized')
@@ -421,9 +422,13 @@ class CopyTask(object):
def make_params(self):
"""
Return a dictionary of parameters to be used by the worker processes.
- On Windows this dictionary must be pickle-able.
+ On Windows this dictionary must be pickle-able, therefore we do not pass the
+ parent connection since it may not be pickle-able. Also, on Windows child
+ processes are spawned and not forked, and therefore we don't need to shutdown
+ the parent connection anyway, see CASSANDRA-11749 for more details.
"""
shell = self.shell
+
return dict(ks=self.ks,
table=self.table,
local_dc=self.local_dc,
@@ -434,6 +439,7 @@ class CopyTask(object):
port=shell.port,
ssl=shell.ssl,
auth_provider=shell.auth_provider,
+ parent_cluster=shell.conn if not IS_WINDOWS else None,
cql_version=shell.conn.cql_version,
config_file=self.config_file,
protocol_version=self.protocol_version,
@@ -1072,7 +1078,8 @@ class ImportTask(CopyTask):
self.processes.append(ImportProcess(self.update_params(params, i)))
feeder = FeedingProcess(self.outmsg.channels[-1], self.inmsg.channels[-1],
- self.outmsg.channels[:-1], self.fname, self.options)
+ self.outmsg.channels[:-1], self.fname, self.options,
+ self.shell.conn if not IS_WINDOWS else None)
self.processes.append(feeder)
self.start_processes()
@@ -1179,7 +1186,7 @@ class FeedingProcess(mp.Process):
"""
A process that reads from import sources and sends chunks to worker processes.
"""
- def __init__(self, inmsg, outmsg, worker_channels, fname, options):
+ def __init__(self, inmsg, outmsg, worker_channels, fname, options, parent_cluster):
mp.Process.__init__(self, target=self.run)
self.inmsg = inmsg
self.outmsg = outmsg
@@ -1189,6 +1196,15 @@ class FeedingProcess(mp.Process):
self.ingest_rate = options.copy['ingestrate']
self.num_worker_processes = options.copy['numprocesses']
self.chunk_id = 0
+ self.parent_cluster = parent_cluster
+
+ def on_fork(self):
+ """
+ Release any parent connections after forking, see CASSANDRA-11749 for details.
+ """
+ if self.parent_cluster:
+ printdebugmsg("Closing parent cluster sockets")
+ self.parent_cluster.shutdown()
def run(self):
pr = profile_on() if PROFILE_ON else None
@@ -1205,6 +1221,9 @@ class FeedingProcess(mp.Process):
here we throttle using the ingest rate in the feeding process because of memory usage concerns.
When finished we send back to the parent process the total number of rows sent.
"""
+
+ self.on_fork()
+
reader = self.reader
reader.start()
channels = self.worker_channels
@@ -1268,6 +1287,7 @@ class ChildProcess(mp.Process):
self.connect_timeout = params['connect_timeout']
self.cql_version = params['cql_version']
self.auth_provider = params['auth_provider']
+ self.parent_cluster = params['parent_cluster']
self.ssl = params['ssl']
self.protocol_version = params['protocol_version']
self.config_file = params['config_file']
@@ -1285,6 +1305,14 @@ class ChildProcess(mp.Process):
else:
self.test_failures = None
+ def on_fork(self):
+ """
+ Release any parent connections after forking, see CASSANDRA-11749 for details.
+ """
+ if self.parent_cluster:
+ printdebugmsg("Closing parent cluster sockets")
+ self.parent_cluster.shutdown()
+
def close(self):
printdebugmsg("Closing queues...")
self.inmsg.close()
@@ -1411,6 +1439,9 @@ class ExportProcess(ChildProcess):
we can signal a global error by sending (None, error).
We terminate when the inbound queue is closed.
"""
+
+ self.on_fork()
+
while True:
if self.num_requests() > self.max_requests:
time.sleep(0.001) # 1 millisecond
@@ -2059,6 +2090,7 @@ class ImportProcess(ChildProcess):
try:
pr = profile_on() if PROFILE_ON else None
+ self.on_fork()
self.inner_run(*self.make_params())
if pr: