You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2021/05/24 21:06:16 UTC
[allura] branch master updated (8694800 -> 320a479)
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git.
from 8694800 [#8388] tidy up per-tool details on ticket search_help; mark as nofollow/noindex so not duplicate content for search engines
new e9b6252 add filter by age of task to TaskCommands
new 576a070 various TaskCommand improvements triggered by expansions to purge command
new 320a479 Add a test for task "count" command
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
Allura/allura/command/taskd.py | 79 ++++++++++++++++++++++++++++--------
Allura/allura/tests/test_commands.py | 48 ++++++++++++++++++++++
2 files changed, 111 insertions(+), 16 deletions(-)
[allura] 02/03: various TaskCommand improvements triggered by
expansions to purge command
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 576a0700f6ef7ff2d34e9c0e46ec3d87c966a88c
Author: Dillon Walls <di...@slashdotmedia.com>
AuthorDate: Fri May 21 20:14:49 2021 +0000
various TaskCommand improvements triggered by expansions to purge command
---
Allura/allura/command/taskd.py | 75 +++++++++++++++++++++++++++++++++---------
1 file changed, 59 insertions(+), 16 deletions(-)
diff --git a/Allura/allura/command/taskd.py b/Allura/allura/command/taskd.py
index 4d7d624..5f9d747 100644
--- a/Allura/allura/command/taskd.py
+++ b/Allura/allura/command/taskd.py
@@ -167,12 +167,20 @@ class TaskdCommand(base.Command):
class TaskCommand(base.Command):
+ cmd_default_states = {
+ 'list': 'ready',
+ 'count': 'ready',
+ 'purge': '*'
+ }
+
summary = 'Task command'
parser = base.Command.standard_parser(verbose=True)
- parser.add_option('-s', '--state', dest='state', default='ready',
- help='state of processes for "list" subcommand. * means all')
+ parser.add_option('-s', '--state', dest='state', default=None,
+ help='state of processes for "list", "count", or "purge" subcommands. * means all. '
+ '(Defaults per command: %s)' %
+ ", ".join(['%s="%s"' % (k, v) for k, v in cmd_default_states.items()]))
parser.add_option('-t', '--timeout', dest='timeout', type=int, default=60,
- help='timeout (in seconds) for busy tasks')
+ help='timeout (in seconds) for busy tasks (only applies to "timeout" command)')
parser.add_option('--filter-name-prefix', dest='filter_name_prefix', default=None,
help='limit to task names starting with this. Example allura.tasks.index_tasks.')
parser.add_option('--filter-result-regex', dest='filter_result_regex', default=None,
@@ -181,12 +189,13 @@ class TaskCommand(base.Command):
help='limit to tasks queued NUM days ago. Example "180"')
min_args = 2
max_args = None
- usage = '''<ini file> [list|retry|purge|timeout|commit]
+ usage = '''<ini file> [list|count|retry|purge|timeout|commit]
- list: list tasks with given --state value
- retry: re-run tasks with error state
- purge: remove all "complete" tasks with result_type "forget" (which is the default)
- timeout: retry all busy tasks older than --timeout seconds (does not stop existing task)
+ list: prints tasks matching --state (default: 'ready') and filters
+ count: counts tasks matching --state (default: 'ready') and filters
+ retry: re-run tasks with 'error' state. --state has no effect
+ purge: remove all tasks that match --state ( default: '*') with result_type "forget".
+ timeout: retry all tasks with state 'busy' and older than --timeout seconds (does not stop existing task). --state has no effect
commit: run a solr 'commit' as a background task
All subcommands except 'commit' can use the --filter-... options.
@@ -197,12 +206,27 @@ class TaskCommand(base.Command):
cmd = self.args[1]
tab = dict(
list=self._list,
+ count=self._count,
retry=self._retry,
purge=self._purge,
timeout=self._timeout,
commit=self._commit)
tab[cmd]()
+ def _get_state_query(self):
+ state = self.options.state
+ if not state:
+ cmd = self.args[1]
+ state = self.cmd_default_states.get(cmd, 'ready')
+
+ if state == '*':
+ from allura import model as M
+ # Providing all possible state values allows us to leverage the mongo index.
+ # omitting a state field might result in an entire COLLSCAN
+ state = {'$in': M.MonQTask.states}
+
+ return state
+
def _add_filters(self, q):
if self.options.filter_name_prefix:
q['task_name'] = {'$regex': r'^{}.*'.format(re.escape(self.options.filter_name_prefix))}
@@ -214,35 +238,52 @@ class TaskCommand(base.Command):
print(q)
return q
+ def _print_query(self, cmd, *args):
+ print('running mongod cmd: %s, %s' % (cmd, args))
+
def _list(self):
'''List tasks'''
from allura import model as M
- base.log.info('Listing tasks of state %s', self.options.state)
- if self.options.state == '*':
- q = dict()
- else:
- q = dict(state=self.options.state)
+ state = self._get_state_query()
+ base.log.info('Listing tasks of state %s', state)
+ q = dict(state=state)
q = self._add_filters(q)
+ self._print_query('find', q)
for t in M.MonQTask.query.find(q):
print(t)
+ def _count(self):
+ '''Count tasks'''
+ from allura import model as M
+ state = self._get_state_query()
+ base.log.info('Counting tasks of state %s', state)
+ q = dict(state=state)
+ q = self._add_filters(q)
+ self._print_query('find', q)
+ count = M.MonQTask.query.find(q).count()
+ print('Task Count %s' % count)
+
def _retry(self):
'''Retry tasks in an error state'''
from allura import model as M
base.log.info('Retry tasks in error state')
q = dict(state='error')
q = self._add_filters(q)
+ update = {'$set': dict(state='ready')}
+ self._print_query('update', q, update)
M.MonQTask.query.update(
q,
- {'$set': dict(state='ready')},
+ update,
multi=True)
def _purge(self):
'''Purge completed tasks'''
from allura import model as M
base.log.info('Purge complete/forget tasks')
- q = dict(state='complete', result_type='forget')
+ state = self._get_state_query()
+ q = dict(state=state, result_type='forget')
q = self._add_filters(q)
+ self._print_query('remove', q)
M.MonQTask.query.remove(q)
def _timeout(self):
@@ -256,9 +297,11 @@ class TaskCommand(base.Command):
time_start={'$lt': cutoff},
)
q = self._add_filters(q)
+ update = {'$set': dict(state='ready')}
+ self._print_query('update', q, update)
M.MonQTask.query.update(
q,
- {'$set': dict(state='ready')},
+ update,
multi=True)
def _commit(self):
[allura] 01/03: add filter by age of task to TaskCommands
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
commit e9b62528381b38604f807d9c71ed05ee7517b561
Author: Dillon Walls <di...@slashdotmedia.com>
AuthorDate: Mon May 17 22:00:18 2021 +0000
add filter by age of task to TaskCommands
---
Allura/allura/command/taskd.py | 4 ++++
Allura/allura/tests/test_commands.py | 44 ++++++++++++++++++++++++++++++++++++
2 files changed, 48 insertions(+)
diff --git a/Allura/allura/command/taskd.py b/Allura/allura/command/taskd.py
index 3a79cd5..4d7d624 100644
--- a/Allura/allura/command/taskd.py
+++ b/Allura/allura/command/taskd.py
@@ -177,6 +177,8 @@ class TaskCommand(base.Command):
help='limit to task names starting with this. Example allura.tasks.index_tasks.')
parser.add_option('--filter-result-regex', dest='filter_result_regex', default=None,
help='limit to tasks with result matching this regex. Example "pysolr"')
+ parser.add_option('--filter-queued-days-ago', dest='days_ago', type=int, default=None,
+ help='limit to tasks queued NUM days ago. Example "180"')
min_args = 2
max_args = None
usage = '''<ini file> [list|retry|purge|timeout|commit]
@@ -206,6 +208,8 @@ class TaskCommand(base.Command):
q['task_name'] = {'$regex': r'^{}.*'.format(re.escape(self.options.filter_name_prefix))}
if self.options.filter_result_regex:
q['result'] = {'$regex': self.options.filter_result_regex}
+ if self.options.days_ago:
+ q['time_queue'] = {'$lt': datetime.utcnow() - timedelta(days=self.options.days_ago)}
if self.verbose:
print(q)
return q
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index ccac09c..784116a 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -18,6 +18,8 @@
from __future__ import unicode_literals
from __future__ import absolute_import
+import datetime
+
import six
from alluratest.tools import assert_raises, assert_in
from testfixtures import OutputCapture
@@ -271,6 +273,9 @@ class TestEnsureIndexCommand(object):
class TestTaskCommand(object):
+ def tearDown(self):
+ M.MonQTask.query.remove({})
+
def test_commit(self):
exit_code = taskd.TaskCommand('task').run([test_config, 'commit'])
assert_equal(M.MonQTask.query.find({'task_name': 'allura.tasks.index_tasks.commit'}).count(), 1)
@@ -281,6 +286,7 @@ class TestTaskCommand(object):
assert_equal(exit_code, 0)
def test_retry(self):
+ # self.test_commit()
exit_code = taskd.TaskCommand('task').run([
test_config, 'retry',
'--filter-name-prefix', 'allura.tasks.index_tasks.',
@@ -288,6 +294,44 @@ class TestTaskCommand(object):
])
assert_equal(exit_code, 0)
+ def test_purge(self):
+ # create task
+ self.test_commit()
+ assert_equal(M.MonQTask.query.find().count(), 1)
+ M.MonQTask.query.update({'task_name': 'allura.tasks.index_tasks.commit'}, {'$set': {'state': 'complete'}})
+ # run purge; verify 0 records
+ exit_code = taskd.TaskCommand('task').run([
+ test_config, 'purge',
+ ])
+ assert_equal(exit_code, 0)
+ assert_equal(M.MonQTask.query.find().count(), 0)
+
+ def test_purge_old_only(self):
+ # create task
+ self.test_commit()
+ assert_equal(M.MonQTask.query.find().count(), 1)
+
+ # force task to be in complete state
+ M.MonQTask.query.update({'task_name': 'allura.tasks.index_tasks.commit'}, {'$set': {'state': 'complete'}})
+ # run purge; verify no records deleted
+ exit_code = taskd.TaskCommand('task').run([
+ test_config, 'purge', '--filter-queued-days-ago', '180',
+ ])
+ assert_equal(exit_code, 0)
+ assert_equal(M.MonQTask.query.find().count(), 1)
+
+ # modify task to be old
+ then = datetime.datetime.utcnow() - datetime.timedelta(days=200)
+ M.MonQTask.query.update({'task_name': 'allura.tasks.index_tasks.commit'},
+ {'$set': {'time_queue': then, 'time_start': then, 'time_stop': then}})
+
+ # run purge; verify old tasks deleted
+ exit_code = taskd.TaskCommand('task').run([
+ test_config, 'purge', '--filter-queued-days-ago', '180',
+ ])
+ assert_equal(exit_code, 0)
+ assert_equal(M.MonQTask.query.find().count(), 0)
+
class TestTaskdCleanupCommand(object):
[allura] 03/03: Add a test for task "count" command
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 320a479450a7eee16df6f00ab9718f9b88b772e6
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Mon May 24 21:03:34 2021 +0000
Add a test for task "count" command
---
Allura/allura/tests/test_commands.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index 784116a..af64994 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -285,6 +285,10 @@ class TestTaskCommand(object):
exit_code = taskd.TaskCommand('task').run([test_config, 'list'])
assert_equal(exit_code, 0)
+ def test_count(self):
+ exit_code = taskd.TaskCommand('task').run([test_config, 'count'])
+ assert_equal(exit_code, 0)
+
def test_retry(self):
# self.test_commit()
exit_code = taskd.TaskCommand('task').run([