You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2012/10/03 20:53:26 UTC
git commit: [#5023] add indexed queue_empty field so mailbox queries
aren't slow on large collections
Updated Branches:
refs/heads/db/5023 [created] 6f52980c9
[#5023] add indexed queue_empty field so mailbox queries aren't slow on large collections
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/6f52980c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/6f52980c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/6f52980c
Branch: refs/heads/db/5023
Commit: 6f52980c9c0026813e4a8ac5680e146b63e4cb4d
Parents: e846983
Author: Dave Brondsema <db...@geek.net>
Authored: Wed Oct 3 18:44:01 2012 +0000
Committer: Dave Brondsema <db...@geek.net>
Committed: Wed Oct 3 18:44:01 2012 +0000
----------------------------------------------------------------------
Allura/allura/lib/helpers.py | 1 +
Allura/allura/model/notification.py | 22 ++++++++++++++----
Allura/allura/tests/model/test_notification.py | 6 +++++
3 files changed, 24 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/6f52980c/Allura/allura/lib/helpers.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/helpers.py b/Allura/allura/lib/helpers.py
index a48822c..4cf3fbc 100644
--- a/Allura/allura/lib/helpers.py
+++ b/Allura/allura/lib/helpers.py
@@ -425,6 +425,7 @@ def pop_user_notifications(user=None):
if mbox:
notifications = M.Notification.query.find(dict(_id={'$in':mbox.queue}))
mbox.queue = []
+ mbox.queue_empty = True
for n in notifications:
M.Notification.query.remove({'_id': n._id}) # clean it up so it doesn't hang around
yield n
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/6f52980c/Allura/allura/model/notification.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/notification.py b/Allura/allura/model/notification.py
index 9bd9180..ff761a3 100644
--- a/Allura/allura/model/notification.py
+++ b/Allura/allura/model/notification.py
@@ -103,6 +103,7 @@ class Notification(MappedClass):
n = cls._make_notification(artifact, topic, **kw)
if n:
mbox.queue.append(n._id)
+ mbox.queue_empty = False
return n
@classmethod
@@ -291,7 +292,9 @@ class Mailbox(MappedClass):
indexes = [
('project_id', 'artifact_index_id'),
('is_flash', 'user_id'),
- ('type', 'next_scheduled')]
+ ('type', 'next_scheduled'), # for q_digest
+ ('type', 'queue_empty'), # for q_direct
+ ]
_id = FieldProperty(S.ObjectId)
user_id = ForeignIdProperty('User', if_missing=lambda:c.user._id)
@@ -314,6 +317,7 @@ class Mailbox(MappedClass):
# a list of notification _id values
queue = FieldProperty([str])
+ queue_empty = FieldProperty(bool)
project = RelationProperty('Project')
app_config = RelationProperty('AppConfig')
@@ -423,7 +427,9 @@ class Mailbox(MappedClass):
for mbox in cls.query.find(d):
mbox.query.update(
{'$push':dict(queue=nid),
- '$set':dict(last_modified=datetime.utcnow())})
+ '$set':dict(last_modified=datetime.utcnow(),
+ queue_empty=False),
+ })
# Make sure the mbox doesn't stick around to be flush()ed
session(mbox).expunge(mbox)
@@ -437,7 +443,9 @@ class Mailbox(MappedClass):
# Queries to find all matching subscription objects
q_direct = dict(
type='direct',
- queue={'$ne':[]})
+ queue={'$ne':[]},
+ queue_empty=False,
+ )
if MAILBOX_QUIESCENT:
q_direct['last_modified']={'$lt':now - MAILBOX_QUIESCENT}
q_digest = dict(
@@ -447,7 +455,9 @@ class Mailbox(MappedClass):
mbox = cls.query.find_and_modify(
query=dict(_id=mbox._id),
update={'$set': dict(
- queue=[])},
+ queue=[],
+ queue_empty=True,
+ )},
new=False)
mbox.fire(now)
for mbox in cls.query.find(q_digest):
@@ -462,7 +472,9 @@ class Mailbox(MappedClass):
query=dict(_id=mbox._id),
update={'$set': dict(
next_scheduled=next_scheduled,
- queue=[])},
+ queue=[],
+ queue_empty=True,
+ )},
new=False)
mbox.fire(now)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/6f52980c/Allura/allura/tests/model/test_notification.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/model/test_notification.py b/Allura/allura/tests/model/test_notification.py
index 4e9a969..d59fc1c 100644
--- a/Allura/allura/tests/model/test_notification.py
+++ b/Allura/allura/tests/model/test_notification.py
@@ -96,6 +96,7 @@ class TestPostNotifications(unittest.TestCase):
assert M.Mailbox.query.find().count()==1
mbox = M.Mailbox.query.get()
assert len(mbox.queue) == 1
+ assert not mbox.queue_empty
def test_email(self):
self._subscribe() # as current user: test-admin
@@ -111,7 +112,9 @@ class TestPostNotifications(unittest.TestCase):
mboxes = M.Mailbox.query.find().all()
assert_equal(len(mboxes), 2)
assert_equal(len(mboxes[0].queue), 1)
+ assert not mboxes[0].queue_empty
assert_equal(len(mboxes[1].queue), 1)
+ assert not mboxes[1].queue_empty
M.Mailbox.fire_ready()
email_tasks = M.MonQTask.query.find({'state': 'ready'}).all()
@@ -200,10 +203,13 @@ class TestSubscriptionTypes(unittest.TestCase):
def test_message(self):
self._test_message()
+
self.setUp()
self._test_message()
+
self.setUp()
M.notification.MAILBOX_QUIESCENT=timedelta(minutes=1)
+ # will raise "assert msg is not None" since the new message is not 1 min old:
self.assertRaises(AssertionError, self._test_message)
def _test_message(self):