You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2019/03/13 18:42:50 UTC
[allura] branch db/8272 created (now 02511fc)
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a change to branch db/8272
in repository https://gitbox.apache.org/repos/asf/allura.git.
at 02511fc [#8272] make per-forum feeds not create a potentially gigantic $in query by default
This branch includes the following new commits:
new 92c3694 [#8272] remove a dead function
new 02511fc [#8272] make per-forum feeds not create a potentially gigantic $in query by default
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[allura] 02/02: [#8272] make per-forum feeds not create a
potentially gigantic $in query by default
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8272
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 02511fca3c484c178cdbe65f8ee371a226d01464
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Wed Mar 13 14:42:27 2019 -0400
[#8272] make per-forum feeds not create a potentially gigantic $in query by default
---
Allura/allura/controllers/discuss.py | 14 ++++++++++++--
Allura/allura/controllers/feed.py | 9 +++++++++
Allura/allura/model/artifact.py | 2 ++
3 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/Allura/allura/controllers/discuss.py b/Allura/allura/controllers/discuss.py
index 662c085..fdc89cc 100644
--- a/Allura/allura/controllers/discuss.py
+++ b/Allura/allura/controllers/discuss.py
@@ -19,6 +19,7 @@ from urllib import unquote
from datetime import datetime
import logging
+import pymongo
from tg import expose, redirect, validate, request, flash, response
from tg.decorators import with_trailing_slash, without_trailing_slash, before_render, before_validate
from decorator import decorator
@@ -118,9 +119,18 @@ class DiscussionController(BaseController, FeedController):
Overrides :meth:`allura.controllers.feed.FeedController.get_feed`.
"""
+ def query(since, until, page, limit, **kwargs):
+ if not since and not until and not page:
+ # simplest default case, so make the threads list shorter by grabbing only needed ones
+ discussion_threads = self.discussion.thread_class().query.find(dict(
+ discussion_id=self.discussion._id,
+ num_replies={'$gt': 0}, # exclude empty threads (spam/deleted) like ForumController does
+ )).sort([('last_post_date', pymongo.DESCENDING)]).limit(limit)
+ else:
+ discussion_threads = self.discussion.threads
+ return dict(ref_id={'$in': [t.index_id() for t in discussion_threads]})
return FeedArgs(
- dict(ref_id={'$in': [t.index_id()
- for t in self.discussion.threads]}),
+ query,
'Recent posts to %s' % self.discussion.name,
self.discussion.url())
diff --git a/Allura/allura/controllers/feed.py b/Allura/allura/controllers/feed.py
index 7aaa789..2fef9f3 100644
--- a/Allura/allura/controllers/feed.py
+++ b/Allura/allura/controllers/feed.py
@@ -35,6 +35,15 @@ class FeedArgs(object):
"""
def __init__(self, query, title, url, description=None):
+ """
+ :param query: mongo criteria to query the Feed collection.
+ Pagination & filter criteria will be added in automatically
+ Can be a function to return criteria, which will be passed args (since, until, page, limit) for
+ advanced optimization.
+ :param title: feed title
+ :param url: feed's own url
+ :param description: feed description
+ """
self.query = query
self.title = title
self.url = url
diff --git a/Allura/allura/model/artifact.py b/Allura/allura/model/artifact.py
index 8f9c5b6..bf0716c 100644
--- a/Allura/allura/model/artifact.py
+++ b/Allura/allura/model/artifact.py
@@ -968,6 +968,8 @@ class Feed(MappedClass):
feed = RssFeed(**d)
limit, page = h.paging_sanitizer(limit or 10, page)
query = defaultdict(dict)
+ if callable(q):
+ q = q(since, until, page, limit)
query.update(q)
if since is not None:
query['pubdate']['$gte'] = since
[allura] 01/02: [#8272] remove a dead function
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8272
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 92c36946c0a99d7f329bb508cdcd272d07221ffa
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Wed Mar 13 14:41:03 2019 -0400
[#8272] remove a dead function
---
ForgeDiscussion/forgediscussion/model/forum.py | 10 ----------
ForgeDiscussion/forgediscussion/tests/test_app.py | 5 ++++-
2 files changed, 4 insertions(+), 11 deletions(-)
diff --git a/ForgeDiscussion/forgediscussion/model/forum.py b/ForgeDiscussion/forgediscussion/model/forum.py
index ce54156..481fe92 100644
--- a/ForgeDiscussion/forgediscussion/model/forum.py
+++ b/ForgeDiscussion/forgediscussion/model/forum.py
@@ -59,16 +59,6 @@ class Forum(M.Discussion):
def thread_class(cls):
return ForumThread
- @LazyProperty
- def sorted_threads(self):
- threads = self.thread_class().query.find(dict(discussion_id=self._id))
- threads = threads.sort([('last_post_date', pymongo.DESCENDING)]).all()
- sorted_threads = chain(
- (t for t in threads if 'Announcement' in t.flags),
- (t for t in threads if 'Sticky' in t.flags and 'Announcement' not in t.flags),
- (t for t in threads if 'Sticky' not in t.flags and 'Announcement' not in t.flags))
- return list(sorted_threads)
-
@property
def parent(self):
return Forum.query.get(_id=self.parent_id)
diff --git a/ForgeDiscussion/forgediscussion/tests/test_app.py b/ForgeDiscussion/forgediscussion/tests/test_app.py
index 39df2f0..90f83bf 100644
--- a/ForgeDiscussion/forgediscussion/tests/test_app.py
+++ b/ForgeDiscussion/forgediscussion/tests/test_app.py
@@ -22,6 +22,7 @@
import tempfile
import json
import os
+from operator import attrgetter
from nose.tools import assert_equal
from pylons import tmpl_context as c
@@ -69,7 +70,9 @@ class TestBulkExport(TestDiscussionApiBase):
def test_export_with_attachments(self):
project = M.Project.query.get(shortname='test')
discussion = project.app_instance('discussion')
- post = Forum.query.get(shortname='general').sorted_threads[0].first_post
+ thread = sorted(Forum.query.get(shortname='general').threads,
+ key=attrgetter('last_post_date'))[-1]
+ post = thread.first_post
test_file1 = FieldStorage()
test_file1.name = 'file_info'
test_file1.filename = 'test_file'