You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2019/10/01 15:50:27 UTC
[allura] branch db/8335 created (now 2493b54)
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a change to branch db/8335
in repository https://gitbox.apache.org/repos/asf/allura.git.
at 2493b54 [#8335] filter search results
This branch includes the following new commits:
new 42df011 [#8335] Improve ticket text field in solr, to not include python repr-like syntax
new 0a1817f [#8335] Strong search test
new 2493b54 [#8335] filter search results
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[allura] 01/03: [#8335] Improve ticket text field in solr,
to not include python repr-like syntax
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8335
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 42df0115c6d787054d75a3664c1164dd5a5dc2b2
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Mon Sep 30 16:18:26 2019 -0400
[#8335] Improve ticket text field in solr, to not include python repr-like syntax
---
ForgeTracker/forgetracker/model/ticket.py | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/ForgeTracker/forgetracker/model/ticket.py b/ForgeTracker/forgetracker/model/ticket.py
index 9a15d05..780ba95 100644
--- a/ForgeTracker/forgetracker/model/ticket.py
+++ b/ForgeTracker/forgetracker/model/ticket.py
@@ -20,13 +20,14 @@ import urllib
import json
import difflib
from datetime import datetime, timedelta
-from bson import ObjectId
import os
+from bson import ObjectId
+import six
+
import pymongo
from pymongo.errors import OperationFailure
from tg import tmpl_context as c, app_globals as g
-from pprint import pformat
from paste.deploy.converters import aslist, asbool
import jinja2
@@ -573,7 +574,11 @@ class TicketHistory(Snapshot):
text=self.data.summary)
# Tracker uses search with default solr parser. It would match only on
# `text`, so we're appending all other field values into `text`, to match on it too.
- result['text'] += pformat(result.values())
+ result['text'] += '\n'.join([six.text_type(v)
+ for k, v
+ in result.iteritems()
+ if k not in ('id', 'project_id_s')
+ ])
return result
@@ -726,7 +731,11 @@ class Ticket(VersionedArtifact, ActivityObject, VotableArtifact):
# Tracker uses search with default solr parser. It would match only on
# `text`, so we're appending all other field values into `text`, to
# match on it too.
- result['text'] += pformat(result.values())
+ result['text'] += '\n'.join([six.text_type(v)
+ for k, v
+ in result.iteritems()
+ if k not in ('id', 'project_id_s')
+ ])
return result
@classmethod
[allura] 03/03: [#8335] filter search results
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8335
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 2493b544fd4dc39aa9008fafa4b84df412cae8d5
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Tue Oct 1 11:29:54 2019 -0400
[#8335] filter search results
---
Allura/allura/lib/helpers.py | 25 ++++++++++++++++++
Allura/allura/lib/search.py | 25 ++++++++++++++----
Allura/allura/tests/functional/test_search.py | 37 +++++++++++++++++++++++++++
Allura/allura/tests/unit/test_solr.py | 5 +++-
ForgeWiki/forgewiki/model/wiki.py | 9 -------
5 files changed, 86 insertions(+), 15 deletions(-)
diff --git a/Allura/allura/lib/helpers.py b/Allura/allura/lib/helpers.py
index e53cdbe..28c08a1 100644
--- a/Allura/allura/lib/helpers.py
+++ b/Allura/allura/lib/helpers.py
@@ -290,6 +290,18 @@ def sharded_path(name, num_parts=2):
def set_context(project_shortname_or_id, mount_point=None, app_config_id=None, neighborhood=None):
+ """
+ Set ``c.project`` and ``c.app`` globals
+
+ :param project_id: _id or shortname of a project
+ :type project_id: ObjectId|str
+ :param mount_point: mount point to set c.app by
+ :type mount_point: str
+ :param app_config_id: alternative to mount_point parameter
+ :type app_config_id: ObjectId|str
+ :param neighborhood: neighborhood full name, required if project is specified by shortname
+ :type neighborhood: str
+ """
from allura import model
try:
p = model.Project.query.get(_id=ObjectId(str(project_shortname_or_id)))
@@ -331,6 +343,19 @@ def set_context(project_shortname_or_id, mount_point=None, app_config_id=None, n
@contextmanager
def push_context(project_id, mount_point=None, app_config_id=None, neighborhood=None):
+ """
+ A context manager to set ``c.project`` and ``c.app`` globals temporarily.
+ To set ``c.user`` or others, use ``push_config(c, user=...)``
+
+ :param project_id: _id or shortname of a project
+ :type project_id: ObjectId|str
+ :param mount_point: mount point to set c.app by
+ :type mount_point: str
+ :param app_config_id: alternative to mount_point parameter
+ :type app_config_id: ObjectId|str
+ :param neighborhood: neighborhood full name, required if project is specified by shortname
+ :type neighborhood: str
+ """
project = getattr(c, 'project', ())
app = getattr(c, 'app', ())
set_context(project_id, mount_point, app_config_id, neighborhood)
diff --git a/Allura/allura/lib/search.py b/Allura/allura/lib/search.py
index 91e75a9..c94c526 100644
--- a/Allura/allura/lib/search.py
+++ b/Allura/allura/lib/search.py
@@ -214,6 +214,9 @@ def search_app(q='', fq=None, app=True, **kw):
Uses dismax query parser. Matches on `title` and `text`. Handles paging, sorting, etc
"""
+ from allura.model import ArtifactReference
+ from allura.lib.security import has_access
+
history = kw.pop('history', None)
if app and kw.pop('project', False):
# Used from app's search controller. If `project` is True, redirect to
@@ -299,13 +302,25 @@ def search_app(q='', fq=None, app=True, **kw):
return doc
def paginate_comment_urls(doc):
- from allura.model import ArtifactReference
-
if doc.get('type_s', '') == 'Post':
- aref = ArtifactReference.query.get(_id=doc.get('id'))
- if aref and aref.artifact:
- doc['url_paginated'] = aref.artifact.url_paginated()
+ artifact = doc['_artifact']
+ if artifact:
+ doc['url_paginated'] = artifact.url_paginated()
return doc
+
+ def filter_unauthorized(doc):
+ aref = ArtifactReference.query.get(_id=doc.get('id'))
+ # cache for paginate_comment_urls to re-use
+ doc['_artifact'] = aref and aref.artifact
+ # .primary() necessary so that a ticket's comment for example is checked with the ticket's perms
+ if doc['_artifact'] and not has_access(doc['_artifact'].primary(), 'read', c.user):
+ return None
+ else:
+ return doc
+
+ filtered_results = filter(None, imap(filter_unauthorized, results))
+ count -= len(results) - len(filtered_results)
+ results = filtered_results
results = imap(historize_urls, results)
results = imap(add_matches, results)
results = imap(paginate_comment_urls, results)
diff --git a/Allura/allura/tests/functional/test_search.py b/Allura/allura/tests/functional/test_search.py
index fe0cb46..3a1a72b 100644
--- a/Allura/allura/tests/functional/test_search.py
+++ b/Allura/allura/tests/functional/test_search.py
@@ -16,9 +16,15 @@
# under the License.
from mock import patch
+from tg import tmpl_context as c
+
+from allura.lib import helpers as h
+import allura.model as M
from allura.tests import TestController
from allura.tests.decorators import with_tool
+from forgewiki.model import Page
+
class TestSearch(TestController):
@@ -31,7 +37,38 @@ class TestSearch(TestController):
# use test2 project since 'test' project has a subproject and MockSOLR can't handle "OR" (caused by subproject)
@with_tool('test2', 'Wiki', 'wiki')
+ # include a wiki on 'test' project too though, for testing that searches are limited to correct project
+ @with_tool('test', 'Wiki', 'wiki')
def test_project_search_controller(self):
self.app.get('/p/test2/search/')
+
+ # add a comment
+ with h.push_context('test2', 'wiki', neighborhood='Projects'):
+ page = Page.find_page('Home')
+ page.discussion_thread.add_post(text='Sample wiki comment')
+ M.MonQTask.run_ready()
+
resp = self.app.get('/p/test2/search/', params=dict(q='wiki'))
resp.mustcontain('Welcome to your wiki! This is the default page')
+ # only from this one project:
+ resp.mustcontain('/test2/')
+ resp.mustcontain(no='/test/')
+ # nice links to comments:
+ resp.mustcontain('Sample wiki comment')
+ resp.mustcontain('/Home/?limit=25#')
+ resp.mustcontain(no='discuss/_thread')
+
+ # make wiki private
+ with h.push_context('test2', 'wiki', neighborhood='Projects'):
+ anon_role = M.ProjectRole.by_name('*anonymous')
+ anon_read_perm = M.ACE.allow(anon_role._id, 'read')
+ acl = c.app.config.acl
+ acl.remove(anon_read_perm)
+
+ resp = self.app.get('/p/test2/search/', params=dict(q='wiki'))
+ resp.mustcontain('Welcome to your wiki! This is the default page')
+ resp.mustcontain('Sample wiki comment')
+
+ resp = self.app.get('/p/test2/search/', params=dict(q='wiki'), extra_environ=dict(username='*anonymous'))
+ resp.mustcontain(no='Welcome to your wiki! This is the default page')
+ resp.mustcontain(no='Sample wiki comment')
diff --git a/Allura/allura/tests/unit/test_solr.py b/Allura/allura/tests/unit/test_solr.py
index af59ae9..5ef3492 100644
--- a/Allura/allura/tests/unit/test_solr.py
+++ b/Allura/allura/tests/unit/test_solr.py
@@ -18,7 +18,7 @@
import unittest
import mock
-from nose.tools import assert_equal
+from datadiff.tools import assert_equal
from markupsafe import Markup
from allura.lib import helpers as h
@@ -191,6 +191,7 @@ class TestSearch_app(unittest.TestCase):
},
)
results.__iter__ = lambda self: iter(results.docs)
+ results.__len__ = lambda self: len(results.docs)
solr_search.return_value = results
with h.push_context('test', 'wiki', neighborhood='Projects'):
resp = search_app(q='foo bar')
@@ -214,12 +215,14 @@ class TestSearch_app(unittest.TestCase):
'title_match': Markup('some <strong>Foo</strong> stuff'),
# HTML in the solr plaintext results get escaped
'text_match': Markup('scary <script>alert(1)</script> bar'),
+ '_artifact': None,
}, {
'id': 321,
'type_s': 'Post',
'title_match': Markup('blah blah'),
# highlighting in text
'text_match': Markup('less scary but still dangerous &lt;script&gt;alert(1)&lt;/script&gt; blah <strong>bar</strong> foo foo'),
+ '_artifact': None,
}]
))
diff --git a/ForgeWiki/forgewiki/model/wiki.py b/ForgeWiki/forgewiki/model/wiki.py
index edd1685..ed0d4de 100644
--- a/ForgeWiki/forgewiki/model/wiki.py
+++ b/ForgeWiki/forgewiki/model/wiki.py
@@ -245,15 +245,6 @@ class Page(VersionedArtifact, ActivityObject):
def attachment_class(cls):
return WikiAttachment
- def reply(self, text):
- Feed.post(self, text)
- # Get thread
- thread = Thread.query.get(artifact_id=self._id)
- return Post(
- discussion_id=thread.discussion_id,
- thread_id=thread._id,
- text=text)
-
@property
def html_text(self):
"""A markdown processed version of the page text"""
[allura] 02/03: [#8335] Strong search test
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8335
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 0a1817f871158426a1936f6aeb89cdf33314660e
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Mon Sep 30 17:27:27 2019 -0400
[#8335] Strong search test
---
Allura/allura/ext/search/search_main.py | 7 ++-----
Allura/allura/lib/solr.py | 10 +++++++++-
Allura/allura/tests/functional/test_search.py | 10 +++++++---
3 files changed, 18 insertions(+), 9 deletions(-)
diff --git a/Allura/allura/ext/search/search_main.py b/Allura/allura/ext/search/search_main.py
index 91bb1da..0c54021 100644
--- a/Allura/allura/ext/search/search_main.py
+++ b/Allura/allura/ext/search/search_main.py
@@ -74,11 +74,8 @@ class SearchController(BaseController):
def index(self, q=None, history=None, **kw):
c.search_results = SearchResults()
c.help_modal = SearchHelp(comments=False)
- pids = [c.project._id] + [
- p._id for p in c.project.subprojects]
- project_match = ' OR '.join(
- 'project_id_s:%s' % pid
- for pid in pids)
+ pids = [c.project._id] + [p._id for p in c.project.subprojects]
+ project_match = ' OR '.join(map(str, pids))
search_params = kw
search_params.update({
'q': q,
diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py
index b74a4ab..222fdef 100644
--- a/Allura/allura/lib/solr.py
+++ b/Allura/allura/lib/solr.py
@@ -165,6 +165,8 @@ class MockSOLR(object):
continue
if ':' in part:
field, value = part.split(':', 1)
+ if value.startswith('(') and value.endswith(')'):
+ value = value[1:-1]
preds.append((field, value))
else:
preds.append(('text', part))
@@ -172,17 +174,23 @@ class MockSOLR(object):
for obj in self.db.values():
for field, value in preds:
neg = False
- if field[0] == '!':
+ if field[0] in ('!', '-'):
neg = True
field = field[1:]
if field == 'text' or field.endswith('_t'):
if (value not in str(obj.get(field, ''))) ^ neg:
break
+ elif field.endswith('_b'):
+ if (asbool(value) != obj.get(field, False)) ^ neg:
+ break
else:
if (value != str(obj.get(field, ''))) ^ neg:
break
else:
result.append(obj)
+
+ if asbool(kw.get('hl')):
+ result.highlighting = {}
return result
def delete(self, *args, **kwargs):
diff --git a/Allura/allura/tests/functional/test_search.py b/Allura/allura/tests/functional/test_search.py
index 118c0f0..fe0cb46 100644
--- a/Allura/allura/tests/functional/test_search.py
+++ b/Allura/allura/tests/functional/test_search.py
@@ -14,9 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
from mock import patch
+
from allura.tests import TestController
+from allura.tests.decorators import with_tool
class TestSearch(TestController):
@@ -28,6 +29,9 @@ class TestSearch(TestController):
self.app.get('/search/', params=dict(q='Root'))
assert search.called, search.called
+ # use test2 project since 'test' project has a subproject and MockSOLR can't handle "OR" (caused by subproject)
+ @with_tool('test2', 'Wiki', 'wiki')
def test_project_search_controller(self):
- self.app.get('/p/test/search/')
- self.app.get('/p/test/search/', params=dict(q='Root'))
+ self.app.get('/p/test2/search/')
+ resp = self.app.get('/p/test2/search/', params=dict(q='wiki'))
+ resp.mustcontain('Welcome to your wiki! This is the default page')