You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2019/12/31 21:32:36 UTC
[allura] branch db/8345 created (now ea34e18)
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a change to branch db/8345
in repository https://gitbox.apache.org/repos/asf/allura.git.
at ea34e18 [#8345] default all event tasks created within web requests & tasks, to not flush immediately, so that they flush at the end of the task/req execution in MingMiddleware with everything else
This branch includes the following new commits:
new ea34e18 [#8345] default all event tasks created within web requests & tasks, to not flush immediately, so that they flush at the end of the task/req execution in MingMiddleware with everything else
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[allura] 01/01: [#8345] default all event tasks created within web
requests & tasks, to not flush immediately,
so that they flush at the end of the task/req execution in MingMiddleware
with everything else
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8345
in repository https://gitbox.apache.org/repos/asf/allura.git
commit ea34e18970a01b0bcb94a4c59656b0b3d4bddc4f
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Tue Dec 31 16:32:26 2019 -0500
[#8345] default all event tasks created within web requests & tasks, to not flush immediately, so that they flush at the end of the task/req execution in MingMiddleware with everything else
---
Allura/allura/lib/app_globals.py | 14 ++++++++++++
Allura/allura/lib/decorators.py | 3 ++-
Allura/allura/model/monq_model.py | 7 ++++--
Allura/allura/tests/test_tasks.py | 46 +++++++++++++++++++++++++++++++++++++--
Allura/test.ini | 3 +++
5 files changed, 68 insertions(+), 5 deletions(-)
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 1fb9162..6ef4626 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -346,6 +346,20 @@ class Globals(object):
return NullActivityStreamDirector()
def post_event(self, topic, *args, **kwargs):
+ if 'flush_immediately' not in kwargs:
+ try:
+ env = request.environ
+ except AttributeError:
+ script_without_ming_middleware = True
+ else:
+ script_without_ming_middleware = env['PATH_INFO'] == '--script--'
+ if script_without_ming_middleware:
+ kwargs['flush_immediately'] = True
+ else:
+ # within tasks and web requests, ming middleware will flush everything to mongo
+ # so best to *not* flush immediately and let all db writes happen in order
+ # so there's no chance of an event being created and started while the initiating code is still running
+ kwargs['flush_immediately'] = False
allura.tasks.event_tasks.event.post(topic, *args, **kwargs)
@LazyProperty
diff --git a/Allura/allura/lib/decorators.py b/Allura/allura/lib/decorators.py
index f84c93c..80c8d63 100644
--- a/Allura/allura/lib/decorators.py
+++ b/Allura/allura/lib/decorators.py
@@ -65,12 +65,13 @@ def task(*args, **kw):
def task_(func):
def post(*args, **kwargs):
delay = kwargs.pop('delay', 0)
+ flush_immediately = kwargs.pop('flush_immediately', True)
project = getattr(c, 'project', None)
cm = (h.notifications_disabled if project and
kw.get('notifications_disabled') else h.null_contextmanager)
with cm(project):
from allura import model as M
- return M.MonQTask.post(func, args, kwargs, delay=delay)
+ return M.MonQTask.post(func, args, kwargs, delay=delay, flush_immediately=flush_immediately)
# if decorating a class, have to make it a staticmethod
# or it gets a spurious cls argument
func.post = staticmethod(post) if inspect.isclass(func) else post
diff --git a/Allura/allura/model/monq_model.py b/Allura/allura/model/monq_model.py
index 3f48aae..a759c99 100644
--- a/Allura/allura/model/monq_model.py
+++ b/Allura/allura/model/monq_model.py
@@ -140,7 +140,9 @@ class MonQTask(MappedClass):
kwargs=None,
result_type='forget',
priority=10,
- delay=0):
+ delay=0,
+ flush_immediately=True,
+ ):
'''Create a new task object based on the current context.'''
if args is None:
args = ()
@@ -173,7 +175,8 @@ class MonQTask(MappedClass):
result=None,
context=context,
time_queue=datetime.utcnow() + timedelta(seconds=delay))
- session(obj).flush(obj)
+ if flush_immediately:
+ session(obj).flush(obj)
return obj
@classmethod
diff --git a/Allura/allura/tests/test_tasks.py b/Allura/allura/tests/test_tasks.py
index ec03dd9..6a03abb 100644
--- a/Allura/allura/tests/test_tasks.py
+++ b/Allura/allura/tests/test_tasks.py
@@ -23,10 +23,12 @@ import sys
import unittest
from base64 import b64encode
import logging
+import pkg_resources
import tg
import mock
from tg import tmpl_context as c, app_globals as g
+
from datadiff.tools import assert_equal
from nose.tools import assert_in, assert_less
from ming.orm import FieldProperty, Mapper
@@ -36,6 +38,7 @@ from testfixtures import LogCapture
from alluratest.controller import setup_basic_test, setup_global_objects, TestController
from allura import model as M
+from allura.command.taskd import TaskdCommand
from allura.lib import helpers as h
from allura.lib import search
from allura.lib.exceptions import CompoundError
@@ -86,21 +89,60 @@ class TestRepoTasks(unittest.TestCase):
mr.set_can_merge_cache.assert_called_once_with(val)
+# used in test_post_event_from_within_task below
+@task
+def _task_that_creates_event(event_name,):
+ g.post_event(event_name)
+ # event does not get flushed to db right away (at end of task, ming middleware will flush it)
+ assert not M.MonQTask.query.get(task_name='allura.tasks.event_tasks.event', args=[event_name])
+
+
class TestEventTasks(unittest.TestCase):
def setUp(self):
+ setup_basic_test()
+ setup_global_objects()
self.called_with = []
def test_fire_event(self):
event_tasks.event('my_event', self, 1, 2, a=5)
assert self.called_with == [((1, 2), {'a': 5})], self.called_with
+ def test_post_event_explicit_flush(self):
+ g.post_event('my_event1', flush_immediately=True)
+ assert M.MonQTask.query.get(task_name='allura.tasks.event_tasks.event', args=['my_event1'])
+
+ g.post_event('my_event2', flush_immediately=False)
+ assert not M.MonQTask.query.get(task_name='allura.tasks.event_tasks.event', args=['my_event2'])
+ ThreadLocalORMSession.flush_all()
+ assert M.MonQTask.query.get(task_name='allura.tasks.event_tasks.event', args=['my_event2'])
+
+ def test_post_event_from_script(self):
+ # simulate post_event being called from a paster script command:
+ with mock.patch.dict(tg.request.environ, PATH_INFO='--script--'):
+ g.post_event('my_event3')
+ # event task is flushed to db right away:
+ assert M.MonQTask.query.get(task_name='allura.tasks.event_tasks.event', args=['my_event3'])
+
+ def test_post_event_from_within_task(self):
+ # instead of M.MonQTask.run_ready() run real 'taskd' so we get all the setup we need
+ taskd = TaskdCommand('taskd')
+ taskd.parse_args([pkg_resources.resource_filename('allura', '../test.ini')])
+ taskd.keep_running = True
+ taskd.restart_when_done = False
+ _task_that_creates_event.post('my_event4')
+ with mock.patch('allura.command.taskd.setproctitle') as setproctitle:
+ def stop_taskd_after_this_task(*args):
+ taskd.keep_running = False
+ setproctitle.side_effect = stop_taskd_after_this_task # avoid proc title change; useful hook to stop taskd
+ taskd.worker()
+ # after the initial task is done, the event task has been persisted:
+ assert M.MonQTask.query.get(task_name='allura.tasks.event_tasks.event', args=['my_event4'])
+
def test_compound_error(self):
'''test_compound_exception -- make sure our multi-exception return works
OK
'''
- setup_basic_test()
- setup_global_objects()
t = raise_exc.post()
with LogCapture(level=logging.ERROR) as l, \
mock.patch.dict(tg.config, {'monq.raise_errors': False}): # match normal non-test behavior
diff --git a/Allura/test.ini b/Allura/test.ini
index ee6042b..82132b5 100644
--- a/Allura/test.ini
+++ b/Allura/test.ini
@@ -89,6 +89,9 @@ solr.use_new_types = false
; TODO: update tests and remove this setting override
auth.require_email_addr = false
+[app:task]
+use = main
+override_root = task ; TurboGears will use controllers/task.py as root controller
;
; Logging goes to a test.log file in current directory