You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by di...@apache.org on 2024/06/27 20:00:36 UTC
(allura) branch dw/8565 created (now 34f2f3449)
This is an automated email from the ASF dual-hosted git repository.
dill0wn pushed a change to branch dw/8565
in repository https://gitbox.apache.org/repos/asf/allura.git
at 34f2f3449 add additional logging of concurrency values in run_tests
This branch includes the following new commits:
new 4ddfcc1df update ming calls from find_and_modify to find_one_and_update
new 58badb930 replace deprecated ming/mongo database_names() with list_database_names()
new e86a182b8 replace deprecated ming/mongo collection_names() with list_collection_names()
new 6cd8656f4 remove ming/pymongo map_reduce and inline_map_reduce. no longer provided by mongo
new 30e8afb9a use pymongo's create_index rather than ensure_index
new 99609d85a pymongo - use drop instead of remove in some places
new 34f2f3449 add additional logging of concurrency values in run_tests
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(allura) 07/07: add additional logging of concurrency values in run_tests
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dill0wn pushed a commit to branch dw/8565
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 34f2f3449adad25da3dfce36f5b6cb7f7bef28d0
Author: Dillon Walls <di...@slashdotmedia.com>
AuthorDate: Tue Jun 25 18:11:18 2024 -0400
add additional logging of concurrency values in run_tests
---
run_tests | 29 +++++++++++++++++++++++------
1 file changed, 23 insertions(+), 6 deletions(-)
diff --git a/run_tests b/run_tests
index 28d2e0b3f..99aa681a9 100755
--- a/run_tests
+++ b/run_tests
@@ -18,7 +18,9 @@
# under the License.
import argparse
+from collections import defaultdict
from glob import glob
+import json
import multiprocessing
from multiprocessing.pool import ThreadPool
import subprocess
@@ -125,13 +127,18 @@ def run_tests_in_parallel(options, runner_args):
def get_pkg_path(pkg):
return ALT_PKG_PATHS.get(pkg, '')
+
+ def get_concurrent_tests(pkg):
+ if pkg in NOT_MULTIPROC_SAFE:
+ return 1
+ return options.concurrent_tests
def get_multiproc_args(pkg):
- if options.concurrent_tests == 1:
- return ''
- return '-n {procs_per_suite} --dist loadfile'.format(
- procs_per_suite=options.concurrent_tests
- ) if pkg not in NOT_MULTIPROC_SAFE else ''
+ if get_concurrent_tests(pkg) > 1:
+ return '-n {procs_per_suite} --dist loadfile'.format(
+ procs_per_suite=options.concurrent_tests
+ )
+ return ''
def get_concurrent_suites():
if '-n' in sys.argv:
@@ -140,6 +147,7 @@ def run_tests_in_parallel(options, runner_args):
cmds = []
env = dict(os.environ)
+ concurrent_tests_map = dict()
for package in check_packages(options.packages):
runner = 'pytest'
if options.coverage:
@@ -163,6 +171,8 @@ def run_tests_in_parallel(options, runner_args):
args=' '.join(default_args + runner_args),
multiproc_args=multiproc_args,
)
+ num_concurrent_tests = get_concurrent_tests(package)
+ concurrent_tests_map[package] = num_concurrent_tests if num_concurrent_tests > 1 else '.'
if options.coverage:
cmd += ' && coverage combine' # merge separate files present from multiprocessing config being on
cmd += ' && coverage report --include=./* --omit="*/tests/*"'
@@ -171,7 +181,14 @@ def run_tests_in_parallel(options, runner_args):
# TODO: add a way to include this or not; and add xml output for Jenkins
cmds.append(('npm run lint-es6', {}))
- ret_codes = run_many(cmds, processes=get_concurrent_suites())
+ num_concurrent_suites = get_concurrent_suites()
+
+ print('Running tests in parallel...')
+ print('\t{:>20}: {}'.format('# CPUS', CPUS))
+ print('\t{:>20}: {}'.format('# Suites in Parallel', num_concurrent_suites))
+ print('\t{:>20}: {}'.format('# Test in Parallel Per Suite', json.dumps(concurrent_tests_map, indent=2)))
+
+ ret_codes = run_many(cmds, processes=num_concurrent_suites)
if options.coverage and not any(ret_codes) and len(options.packages) > 1:
subprocess.call('rm .coverage', shell=True)
(allura) 03/07: replace deprecated ming/mongo collection_names() with list_collection_names()
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dill0wn pushed a commit to branch dw/8565
in repository https://gitbox.apache.org/repos/asf/allura.git
commit e86a182b88a75227613de79326a79d39e83ff9df
Author: Dillon Walls <di...@slashdotmedia.com>
AuthorDate: Wed Jun 19 15:28:05 2024 -0400
replace deprecated ming/mongo collection_names() with list_collection_names()
---
Allura/allura/websetup/bootstrap.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Allura/allura/websetup/bootstrap.py b/Allura/allura/websetup/bootstrap.py
index 981df8438..78a602d54 100644
--- a/Allura/allura/websetup/bootstrap.py
+++ b/Allura/allura/websetup/bootstrap.py
@@ -287,7 +287,7 @@ def wipe_database():
continue
log.info('Wiping database %s', database)
db = conn[database]
- for coll in list(db.collection_names()):
+ for coll in list(db.list_collection_names()):
if coll.startswith('system.'):
continue
log.info('Dropping collection %s:%s', database, coll)
@@ -301,7 +301,7 @@ def clear_all_database_tables():
conn = M.main_doc_session.bind.conn
for db in conn.list_database_names():
db = conn[db]
- for coll in list(db.collection_names()):
+ for coll in list(db.list_collection_names()):
if coll == 'system.indexes':
continue
db.drop_collection(coll)
(allura) 05/07: use pymongo's create_index rather than ensure_index
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dill0wn pushed a commit to branch dw/8565
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 30e8afb9a56cfc27257685017573e49ba25bd51b
Author: Dillon Walls <di...@slashdotmedia.com>
AuthorDate: Tue Jun 25 18:09:35 2024 -0400
use pymongo's create_index rather than ensure_index
---
Allura/allura/command/show_models.py | 8 ++++----
Allura/allura/tests/test_commands.py | 20 +++++++++-----------
2 files changed, 13 insertions(+), 15 deletions(-)
diff --git a/Allura/allura/command/show_models.py b/Allura/allura/command/show_models.py
index b0b5fad19..7bffebffa 100644
--- a/Allura/allura/command/show_models.py
+++ b/Allura/allura/command/show_models.py
@@ -297,14 +297,14 @@ class EnsureIndexCommand(base.Command):
# _id is always non-sparse and unique anyway
del index_options['sparse']
del index_options['unique']
- collection.ensure_index(idx.index_spec, **index_options)
+ collection.create_index(idx.index_spec, **index_options)
break
except DuplicateKeyError as err:
base.log.info('Found dupe key(%s), eliminating dupes', err)
self._remove_dupes(collection, idx.index_spec)
for keys, idx in indexes.items():
base.log.info('...... ensure %s:%s', collection.name, idx)
- collection.ensure_index(idx.index_spec, background=True, **idx.index_options)
+ collection.create_index(idx.index_spec, background=True, **idx.index_options)
# Drop obsolete indexes
for iname, keys in prev_indexes.items():
if keys not in indexes:
@@ -329,12 +329,12 @@ class EnsureIndexCommand(base.Command):
superset_keys = keys + [('temporary_extra_field_for_indexing', 1)]
base.log.info('...... ensure index %s:%s',
collection.name, superset_keys)
- superset_index = collection.ensure_index(superset_keys)
+ superset_index = collection.create_index(superset_keys)
base.log.info('...... drop index %s:%s', collection.name, iname)
collection.drop_index(iname)
base.log.info('...... ensure index %s:%s %s',
collection.name, keys, creation_options)
- collection.ensure_index(keys, **creation_options)
+ collection.create_index(keys, **creation_options)
base.log.info('...... drop index %s:%s',
collection.name, superset_index)
collection.drop_index(superset_index)
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index 660b0f172..068fdaf89 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -199,7 +199,7 @@ class TestEnsureIndexCommand:
cmd = show_models.EnsureIndexCommand('ensure_index')
cmd.options = Object(clean=False)
cmd._update_indexes(collection, indexes)
- assert collection.ensure_index.called
+ assert collection.create_index.called
assert not collection.drop_index.called
def test_update_indexes_order(self):
@@ -220,13 +220,13 @@ class TestEnsureIndexCommand:
for i, call_ in enumerate(collection.mock_calls):
method_name = call_[0]
collection_call_order[method_name] = i
- assert collection_call_order['ensure_index'] < collection_call_order['drop_index'], collection.mock_calls
+ assert collection_call_order['create_index'] < collection_call_order['drop_index'], collection.mock_calls
def test_update_indexes_unique_changes(self):
collection = Mock(name='collection')
# expecting these ensure_index calls, we'll make their return values normal
# for easier assertions later
- collection.ensure_index.side_effect = [
+ collection.create_index.side_effect = [
'_foo_bar_temporary_extra_field_for_indexing',
'_foo_bar',
'_foo_baz_temporary_extra_field_for_indexing',
@@ -251,18 +251,16 @@ class TestEnsureIndexCommand:
assert collection.mock_calls == [
call.index_information(),
- call.ensure_index(
- [('foo', 1), ('bar', 1), ('temporary_extra_field_for_indexing', 1)]),
+ call.create_index([('foo', 1), ('bar', 1), ('temporary_extra_field_for_indexing', 1)]),
call.drop_index('_foo_bar'),
- call.ensure_index([('foo', 1), ('bar', 1)], unique=False),
+ call.create_index([('foo', 1), ('bar', 1)], unique=False),
call.drop_index('_foo_bar_temporary_extra_field_for_indexing'),
- call.ensure_index(
- [('foo', 1), ('baz', 1), ('temporary_extra_field_for_indexing', 1)]),
+ call.create_index([('foo', 1), ('baz', 1), ('temporary_extra_field_for_indexing', 1)]),
call.drop_index('_foo_baz'),
- call.ensure_index([('foo', 1), ('baz', 1)], unique=True),
+ call.create_index([('foo', 1), ('baz', 1)], unique=True),
call.drop_index('_foo_baz_temporary_extra_field_for_indexing'),
- call.ensure_index([('foo', 1), ('baz', 1)], unique=True, sparse=False),
- call.ensure_index([('foo', 1), ('bar', 1)], unique=False, sparse=False, background=True)
+ call.create_index([('foo', 1), ('baz', 1)], unique=True, sparse=False),
+ call.create_index([('foo', 1), ('bar', 1)], unique=False, sparse=False, background=True)
]
(allura) 02/07: replace deprecated ming/mongo database_names() with list_database_names()
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dill0wn pushed a commit to branch dw/8565
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 58badb930c316c8a1ea7799cb43509922fed7f80
Author: Dillon Walls <di...@slashdotmedia.com>
AuthorDate: Wed Jun 19 15:27:44 2024 -0400
replace deprecated ming/mongo database_names() with list_database_names()
---
Allura/allura/websetup/bootstrap.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/Allura/allura/websetup/bootstrap.py b/Allura/allura/websetup/bootstrap.py
index 21cfa6781..981df8438 100644
--- a/Allura/allura/websetup/bootstrap.py
+++ b/Allura/allura/websetup/bootstrap.py
@@ -279,10 +279,10 @@ def wipe_database():
conn = M.main_doc_session.bind.conn
if isinstance(conn, mim.Connection):
clear_all_database_tables()
- for db in conn.database_names():
+ for db in conn.list_database_names():
db = conn[db]
else:
- for database in conn.database_names():
+ for database in conn.list_database_names():
if database not in ('allura', 'pyforge', 'project-data'):
continue
log.info('Wiping database %s', database)
@@ -299,7 +299,7 @@ def wipe_database():
def clear_all_database_tables():
conn = M.main_doc_session.bind.conn
- for db in conn.database_names():
+ for db in conn.list_database_names():
db = conn[db]
for coll in list(db.collection_names()):
if coll == 'system.indexes':
(allura) 01/07: update ming calls from find_and_modify to find_one_and_update
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dill0wn pushed a commit to branch dw/8565
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 4ddfcc1dfe2e1b4f877ce94c791ec01c02a947f1
Author: Dillon Walls <di...@slashdotmedia.com>
AuthorDate: Tue Jun 18 18:21:38 2024 -0400
update ming calls from find_and_modify to find_one_and_update
---
Allura/allura/lib/custom_middleware.py | 5 +++--
Allura/allura/model/auth.py | 4 ++--
Allura/allura/model/monq_model.py | 4 ++--
Allura/allura/model/notification.py | 8 ++++----
Allura/allura/model/repository.py | 4 ++--
ForgeImporters/forgeimporters/base.py | 8 ++++----
ForgeTracker/forgetracker/model/ticket.py | 4 ++--
7 files changed, 19 insertions(+), 18 deletions(-)
diff --git a/Allura/allura/lib/custom_middleware.py b/Allura/allura/lib/custom_middleware.py
index e06f950c1..16ca53479 100644
--- a/Allura/allura/lib/custom_middleware.py
+++ b/Allura/allura/lib/custom_middleware.py
@@ -326,7 +326,8 @@ class AlluraTimerMiddleware(TimerMiddleware):
debug_each_call=False),
Timer('ming', ming.odm.odmsession.ODMSession,
'insert_now', 'update_now', 'delete_now',
- 'find', 'find_and_modify', 'remove', 'update', 'update_if_not_modified',
+ 'find', 'find_one_and_update', 'find_one_and_replace', 'find_one_and_delete',
+ 'remove', 'update', 'update_if_not_modified',
'aggregate', 'group', 'map_reduce', 'inline_map_reduce', 'distinct',
),
# Timer('ming', ming.schema.Document, 'validate',
@@ -336,7 +337,7 @@ class AlluraTimerMiddleware(TimerMiddleware):
# debug_each_call=False),
Timer('mongo', pymongo.collection.Collection, 'count', 'find',
'find_one', 'aggregate', 'group', 'map_reduce',
- 'inline_map_reduce', 'find_and_modify',
+ 'inline_map_reduce', 'find_one_and_update', 'find_one_and_replace', 'find_one_and_delete',
'insert', 'save', 'update', 'remove', 'drop'),
Timer('mongo', pymongo.cursor.Cursor, 'count', 'distinct',
'_refresh'),
diff --git a/Allura/allura/model/auth.py b/Allura/allura/model/auth.py
index 0b73f1b34..cb797e776 100644
--- a/Allura/allura/model/auth.py
+++ b/Allura/allura/model/auth.py
@@ -217,8 +217,8 @@ class AuthGlobals(MappedClass):
@classmethod
def get_next_uid(cls):
cls.upsert()
- g = cls.query.find_and_modify(
- query={}, update={'$inc': {'next_uid': 1}},
+ g = cls.query.find_one_and_update(
+ {}, update={'$inc': {'next_uid': 1}},
new=True)
return g.next_uid
diff --git a/Allura/allura/model/monq_model.py b/Allura/allura/model/monq_model.py
index 61ea3d863..17b4aa47e 100644
--- a/Allura/allura/model/monq_model.py
+++ b/Allura/allura/model/monq_model.py
@@ -199,8 +199,8 @@ class MonQTask(MappedClass):
query['time_queue'] = {'$lte': datetime.utcnow()}
if only:
query['task_name'] = {'$in': only}
- obj = cls.query.find_and_modify(
- query=query,
+ obj = cls.query.find_one_and_update(
+ query,
update={
'$set': dict(
state='busy',
diff --git a/Allura/allura/model/notification.py b/Allura/allura/model/notification.py
index 2bd205d35..942a34b66 100644
--- a/Allura/allura/model/notification.py
+++ b/Allura/allura/model/notification.py
@@ -592,8 +592,8 @@ class Mailbox(MappedClass):
next_scheduled={'$lt': now})
def find_and_modify_direct_mbox():
- return cls.query.find_and_modify(
- query=q_direct,
+ return cls.query.find_one_and_update(
+ q_direct,
update={'$set': dict(
queue=[],
queue_empty=True,
@@ -618,8 +618,8 @@ class Mailbox(MappedClass):
next_scheduled += timedelta(days=7 * mbox.frequency.n)
elif mbox.frequency.unit == 'month':
next_scheduled += timedelta(days=30 * mbox.frequency.n)
- mbox = cls.query.find_and_modify(
- query=dict(_id=mbox._id),
+ mbox = cls.query.find_one_and_update(
+ dict(_id=mbox._id),
update={'$set': dict(
next_scheduled=next_scheduled,
queue=[],
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index c947ec1b3..3266fe0a8 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -1072,8 +1072,8 @@ class CommitStatus(MappedClass):
@classmethod
def upsert(cls, **kw):
- obj = cls.query.find_and_modify(
- query=dict(commit_id=kw.get('commit_id'), context=kw.get('context')),
+ obj = cls.query.find_one_and_update(
+ dict(commit_id=kw.get('commit_id'), context=kw.get('context')),
update={'$set': kw},
new=True,
upsert=True,
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 1087bdc40..7c87c6a78 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -466,8 +466,8 @@ class ToolImporter(metaclass=ToolImporterMeta):
"""
limit = config.get('tool_import.rate_limit', 1)
pending_key = 'tool_data.%s.pending' % self.classname
- modified_project = M.Project.query.find_and_modify(
- query={
+ modified_project = M.Project.query.find_one_and_update(
+ {
'_id': project._id,
'$or': [
{pending_key: None},
@@ -485,8 +485,8 @@ class ToolImporter(metaclass=ToolImporterMeta):
to indicate that an import is complete.
"""
pending_key = 'tool_data.%s.pending' % self.classname
- M.Project.query.find_and_modify(
- query={'_id': project._id},
+ M.Project.query.find_one_and_update(
+ {'_id': project._id},
update={'$inc': {pending_key: -1}},
new=True,
)
diff --git a/ForgeTracker/forgetracker/model/ticket.py b/ForgeTracker/forgetracker/model/ticket.py
index de099a372..01714d071 100644
--- a/ForgeTracker/forgetracker/model/ticket.py
+++ b/ForgeTracker/forgetracker/model/ticket.py
@@ -139,8 +139,8 @@ class Globals(MappedClass):
})
def next_ticket_num(self):
- gbl = Globals.query.find_and_modify(
- query=dict(app_config_id=self.app_config_id),
+ gbl = Globals.query.find_one_and_update(
+ dict(app_config_id=self.app_config_id),
update={'$inc': {'last_ticket_num': 1}},
new=True)
session(gbl).expunge(gbl)
(allura) 04/07: remove ming/pymongo map_reduce and inline_map_reduce. no longer provided by mongo
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dill0wn pushed a commit to branch dw/8565
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 6cd8656f4bc93fa2abeb04f39c431ef7971b2d91
Author: Dillon Walls <di...@slashdotmedia.com>
AuthorDate: Wed Jun 19 15:28:45 2024 -0400
remove ming/pymongo map_reduce and inline_map_reduce. no longer provided by mongo
---
Allura/allura/lib/custom_middleware.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/Allura/allura/lib/custom_middleware.py b/Allura/allura/lib/custom_middleware.py
index 16ca53479..1baf4c4f1 100644
--- a/Allura/allura/lib/custom_middleware.py
+++ b/Allura/allura/lib/custom_middleware.py
@@ -328,7 +328,7 @@ class AlluraTimerMiddleware(TimerMiddleware):
'insert_now', 'update_now', 'delete_now',
'find', 'find_one_and_update', 'find_one_and_replace', 'find_one_and_delete',
'remove', 'update', 'update_if_not_modified',
- 'aggregate', 'group', 'map_reduce', 'inline_map_reduce', 'distinct',
+ 'aggregate', 'group', 'distinct',
),
# Timer('ming', ming.schema.Document, 'validate',
# debug_each_call=False),
@@ -336,8 +336,8 @@ class AlluraTimerMiddleware(TimerMiddleware):
# '_validate_fast_missing', '_validate_optional',
# debug_each_call=False),
Timer('mongo', pymongo.collection.Collection, 'count', 'find',
- 'find_one', 'aggregate', 'group', 'map_reduce',
- 'inline_map_reduce', 'find_one_and_update', 'find_one_and_replace', 'find_one_and_delete',
+ 'find_one', 'aggregate', 'group',
+ 'find_one_and_update', 'find_one_and_replace', 'find_one_and_delete',
'insert', 'save', 'update', 'remove', 'drop'),
Timer('mongo', pymongo.cursor.Cursor, 'count', 'distinct',
'_refresh'),
(allura) 06/07: pymongo - use drop instead of remove in some places
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dill0wn pushed a commit to branch dw/8565
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 99609d85a816529b686bd05c6a33153d14bce31c
Author: Dillon Walls <di...@slashdotmedia.com>
AuthorDate: Tue Jun 25 18:10:52 2024 -0400
pymongo - use drop instead of remove in some places
---
Allura/allura/tests/model/test_filesystem.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/Allura/allura/tests/model/test_filesystem.py b/Allura/allura/tests/model/test_filesystem.py
index 7c2c9520a..9fdb4028a 100644
--- a/Allura/allura/tests/model/test_filesystem.py
+++ b/Allura/allura/tests/model/test_filesystem.py
@@ -49,9 +49,9 @@ class TestFile(TestCase):
self.conn = M.session.main_doc_session.db._connection
self.db = M.session.main_doc_session.db
- self.db.fs.remove()
- self.db.fs.files.remove()
- self.db.fs.chunks.remove()
+ self.db.fs.drop()
+ self.db.fs.files.drop()
+ self.db.fs.chunks.drop()
def test_from_stream(self):
f = File.from_stream('test1.txt', BytesIO(b'test1'))