You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2020/08/21 20:08:28 UTC
[allura] branch db/8374 created (now cc27dbe)
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a change to branch db/8374
in repository https://gitbox.apache.org/repos/asf/allura.git.
at cc27dbe [#8374] document these commands, related minor doc cleanup
This branch includes the following new commits:
new a6d4a85 [#8374] enhance task retry and related commands with filter options
new 1dcc9e2 [#8374] script/task for clearing old notifications
new cc27dbe [#8374] document these commands, related minor doc cleanup
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[allura] 03/03: [#8374] document these commands,
related minor doc cleanup
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8374
in repository https://gitbox.apache.org/repos/asf/allura.git
commit cc27dbe444f0e20c6f21f39e79e2d93f63f10cbe
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Fri Aug 21 16:08:18 2020 -0400
[#8374] document these commands, related minor doc cleanup
---
Allura/docs/getting_started/administration.rst | 24 ++++++++++++++++++------
requirements-dev.txt | 2 +-
2 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/Allura/docs/getting_started/administration.rst b/Allura/docs/getting_started/administration.rst
index 76469b7..7ab6e12 100644
--- a/Allura/docs/getting_started/administration.rst
+++ b/Allura/docs/getting_started/administration.rst
@@ -20,6 +20,7 @@ Administration
**************
.. contents::
+ :depth: 2
:local:
.. _site-admin-interface:
@@ -183,6 +184,13 @@ taskd
.. program-output:: paster taskd development.ini --help
+task
+-----
+
+.. program-output:: paster task development.ini --help | fmt -s -w 95
+ :shell:
+
+
taskd_cleanup
-------------
@@ -274,6 +282,16 @@ create_sitemap_files.py
:prog: paster script development.ini allura/scripts/create_sitemap_files.py --
+clear_old_notifications.py
+--------------------------
+
+*Can be run as a background task using task name:* :code:`allura.scripts.clear_old_notifications.ClearOldNotifications`
+
+.. argparse::
+ :module: allura.scripts.clear_old_notifications
+ :func: get_parser
+ :prog: paster script development.ini allura/scripts/clear_old_notifications.py --
+
publicize-neighborhood.py
-------------------------
@@ -423,9 +441,3 @@ Illustrates creating a new ticket, using the simple OAuth Bearer token.
:func: get_parser
:prog: python scripts/new_ticket.py
-
-wiki-post.py
-------------
-
-.. program-output:: python ../../scripts/wiki-post.py --help | sed 's/Usage: /Usage: python scripts\//'
- :shell:
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 2c3f366..ff7b4e8 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -22,7 +22,7 @@ pyparsing==2.4.0 # via packaging
pytz==2019.1 # via babel
q==2.3
requests==2.22.0 # via sphinx
-six==1.12.0 # via packaging, pip-tools, sphinx
+six==1.15.0 # via packaging, pip-tools, sphinx
snowballstemmer==1.9.0 # via sphinx
sphinx-argparse==0.2.5
sphinx-rtd-theme==0.1.6
[allura] 01/03: [#8374] enhance task retry and related commands
with filter options
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8374
in repository https://gitbox.apache.org/repos/asf/allura.git
commit a6d4a85342dec25e5f650385f27543e2aff0636f
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Fri Aug 21 16:07:02 2020 -0400
[#8374] enhance task retry and related commands with filter options
---
Allura/allura/command/taskd.py | 48 +++++++++++++++++++++++++++++++-----
Allura/allura/model/monq_model.py | 14 -----------
Allura/allura/tests/test_commands.py | 8 ++++++
3 files changed, 50 insertions(+), 20 deletions(-)
diff --git a/Allura/allura/command/taskd.py b/Allura/allura/command/taskd.py
index 1f5ec78..3a79cd5 100644
--- a/Allura/allura/command/taskd.py
+++ b/Allura/allura/command/taskd.py
@@ -18,6 +18,8 @@
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import absolute_import
+
+import re
import logging
import os
import time
@@ -168,12 +170,25 @@ class TaskCommand(base.Command):
summary = 'Task command'
parser = base.Command.standard_parser(verbose=True)
parser.add_option('-s', '--state', dest='state', default='ready',
- help='state of processes to examine')
+ help='state of processes for "list" subcommand. * means all')
parser.add_option('-t', '--timeout', dest='timeout', type=int, default=60,
help='timeout (in seconds) for busy tasks')
+ parser.add_option('--filter-name-prefix', dest='filter_name_prefix', default=None,
+ help='limit to task names starting with this. Example allura.tasks.index_tasks.')
+ parser.add_option('--filter-result-regex', dest='filter_result_regex', default=None,
+ help='limit to tasks with result matching this regex. Example "pysolr"')
min_args = 2
max_args = None
- usage = '<ini file> [list|retry|purge|timeout|commit]'
+ usage = '''<ini file> [list|retry|purge|timeout|commit]
+
+ list: list tasks with given --state value
+ retry: re-run tasks with error state
+ purge: remove all "complete" tasks with result_type "forget" (which is the default)
+ timeout: retry all busy tasks older than --timeout seconds (does not stop existing task)
+ commit: run a solr 'commit' as a background task
+
+ All subcommands except 'commit' can use the --filter-... options.
+ '''
def command(self):
self.basic_setup()
@@ -186,6 +201,15 @@ class TaskCommand(base.Command):
commit=self._commit)
tab[cmd]()
+ def _add_filters(self, q):
+ if self.options.filter_name_prefix:
+ q['task_name'] = {'$regex': r'^{}.*'.format(re.escape(self.options.filter_name_prefix))}
+ if self.options.filter_result_regex:
+ q['result'] = {'$regex': self.options.filter_result_regex}
+ if self.verbose:
+ print(q)
+ return q
+
def _list(self):
'''List tasks'''
from allura import model as M
@@ -194,6 +218,7 @@ class TaskCommand(base.Command):
q = dict()
else:
q = dict(state=self.options.state)
+ q = self._add_filters(q)
for t in M.MonQTask.query.find(q):
print(t)
@@ -201,8 +226,10 @@ class TaskCommand(base.Command):
'''Retry tasks in an error state'''
from allura import model as M
base.log.info('Retry tasks in error state')
+ q = dict(state='error')
+ q = self._add_filters(q)
M.MonQTask.query.update(
- dict(state='error'),
+ q,
{'$set': dict(state='ready')},
multi=True)
@@ -210,8 +237,9 @@ class TaskCommand(base.Command):
'''Purge completed tasks'''
from allura import model as M
base.log.info('Purge complete/forget tasks')
- M.MonQTask.query.remove(
- dict(state='complete', result_type='forget'))
+ q = dict(state='complete', result_type='forget')
+ q = self._add_filters(q)
+ M.MonQTask.query.remove(q)
def _timeout(self):
'''Reset tasks that have been busy too long to 'ready' state'''
@@ -219,7 +247,15 @@ class TaskCommand(base.Command):
base.log.info('Reset tasks stuck for %ss or more',
self.options.timeout)
cutoff = datetime.utcnow() - timedelta(seconds=self.options.timeout)
- M.MonQTask.timeout_tasks(cutoff)
+ q = dict(
+ state='busy',
+ time_start={'$lt': cutoff},
+ )
+ q = self._add_filters(q)
+ M.MonQTask.query.update(
+ q,
+ {'$set': dict(state='ready')},
+ multi=True)
def _commit(self):
'''Schedule a SOLR commit'''
diff --git a/Allura/allura/model/monq_model.py b/Allura/allura/model/monq_model.py
index 60edf10..2af31ed 100644
--- a/Allura/allura/model/monq_model.py
+++ b/Allura/allura/model/monq_model.py
@@ -217,20 +217,6 @@ class MonQTask(MappedClass):
return None
@classmethod
- def timeout_tasks(cls, older_than):
- '''Mark all busy tasks older than a certain datetime as 'ready' again.
- Used to retry 'stuck' tasks.'''
- spec = dict(state='busy')
- spec['time_start'] = {'$lt': older_than}
- cls.query.update(spec, {'$set': dict(state='ready')}, multi=True)
-
- @classmethod
- def clear_complete(cls):
- '''Delete the task objects for complete tasks'''
- spec = dict(state='complete')
- cls.query.remove(spec)
-
- @classmethod
def run_ready(cls, worker=None):
'''Run all the tasks that are currently ready'''
i = 0
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index ab077fd..ca6980f 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -280,6 +280,14 @@ class TestTaskCommand(object):
exit_code = taskd.TaskCommand('task').run([test_config, 'list'])
assert_equal(exit_code, 0)
+ def test_retry(self):
+ exit_code = taskd.TaskCommand('task').run([
+ test_config, 'retry',
+ '--filter-name-prefix', 'allura.tasks.index_tasks.',
+ '--filter-result-regex', 'pysolr',
+ ])
+ assert_equal(exit_code, 0)
+
class TestTaskdCleanupCommand(object):
[allura] 02/03: [#8374] script/task for clearing old notifications
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8374
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 1dcc9e291520d968bba7d26d060df91a1f200da1
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Fri Aug 21 16:07:34 2020 -0400
[#8374] script/task for clearing old notifications
---
Allura/allura/scripts/clear_old_notifications.py | 53 ++++++++++++++++++++++++
Allura/allura/tests/scripts/test_misc_scripts.py | 47 +++++++++++++++++++++
2 files changed, 100 insertions(+)
diff --git a/Allura/allura/scripts/clear_old_notifications.py b/Allura/allura/scripts/clear_old_notifications.py
new file mode 100644
index 0000000..5a2218a
--- /dev/null
+++ b/Allura/allura/scripts/clear_old_notifications.py
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from datetime import datetime, timedelta
+
+import logging
+
+import argparse
+
+from allura.scripts import ScriptTask
+from allura import model as M
+
+log = logging.getLogger('allura.scripts.clear_old_notifications')
+
+
+class ClearOldNotifications(ScriptTask):
+
+ @classmethod
+ def parser(cls):
+ parser = argparse.ArgumentParser(description="Remove old temporary notifications")
+ parser.add_argument('--back-days', dest='back_days', type=float, default=60,
+ help='How many days back to clear from (keeps newer notifications)')
+ return parser
+
+ @classmethod
+ def execute(cls, options):
+ before = datetime.utcnow() - timedelta(days=options.back_days)
+ M.Notification.query.remove({
+ 'pubdate': {'$lt': before}
+ })
+
+
+def get_parser():
+ return ClearOldNotifications.parser()
+
+
+if __name__ == '__main__':
+ ClearOldNotifications.main()
diff --git a/Allura/allura/tests/scripts/test_misc_scripts.py b/Allura/allura/tests/scripts/test_misc_scripts.py
new file mode 100644
index 0000000..7075c8a
--- /dev/null
+++ b/Allura/allura/tests/scripts/test_misc_scripts.py
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+from bson import ObjectId
+from nose.tools import assert_equal
+
+from allura.scripts.clear_old_notifications import ClearOldNotifications
+from alluratest.controller import setup_basic_test
+from allura import model as M
+from ming.odm import session
+
+
+class TestClearOldNotifications(object):
+
+ def setUp(self):
+ setup_basic_test()
+
+ def run_script(self, options):
+ cls = ClearOldNotifications
+ opts = cls.parser().parse_args(options)
+ cls.execute(opts)
+
+ def test(self):
+ n = M.Notification(app_config_id=ObjectId(), neighborhood_id=ObjectId(), project_id=ObjectId(),
+ tool_name='blah')
+ session(n).flush(n)
+ assert_equal(M.Notification.query.find().count(), 1)
+ self.run_script(['--back-days', '7'])
+ assert_equal(M.Notification.query.find().count(), 1)
+ self.run_script(['--back-days', '0'])
+ assert_equal(M.Notification.query.find().count(), 0)