You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by ke...@apache.org on 2019/04/12 18:32:04 UTC
[allura] 02/03: Fix project-wide search with unicode terms
This is an automated email from the ASF dual-hosted git repository.
kentontaylor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 57c715de7b8e96d22230830bbf1978a8b92efec9
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Thu Apr 11 18:08:13 2019 -0400
Fix project-wide search with unicode terms
---
Allura/allura/lib/search.py | 6 +++---
Allura/allura/lib/utils.py | 45 +++++++++++++++++++++++++++++++++++++++++++++
Allura/allura/model/auth.py | 44 --------------------------------------------
3 files changed, 48 insertions(+), 47 deletions(-)
diff --git a/Allura/allura/lib/search.py b/Allura/allura/lib/search.py
index 370cef6..77fc1a7 100644
--- a/Allura/allura/lib/search.py
+++ b/Allura/allura/lib/search.py
@@ -18,7 +18,6 @@
import re
import socket
from logging import getLogger
-from urllib import urlencode
from itertools import imap
import bson
@@ -31,6 +30,7 @@ from pysolr import SolrError
from allura.lib import helpers as h
from allura.lib.solr import escape_solr_arg
+from allura.lib.utils import urlencode
log = getLogger(__name__)
@@ -141,8 +141,8 @@ def search(q, short_timeout=False, ignore_errors=True, **kw):
except (SolrError, socket.error) as e:
log.exception('Error in solr search')
if not ignore_errors:
- match = re.search(r'<pre>(.*)</pre>', str(e))
- raise SearchError('Error running search query: %s' %
+ match = re.search(r'<pre>(.*)</pre>', unicode(e))
+ raise SearchError(u'Error running search query: %s' %
(match.group(1) if match else e))
diff --git a/Allura/allura/lib/utils.py b/Allura/allura/lib/utils.py
index 449a4e7..f1484ff 100644
--- a/Allura/allura/lib/utils.py
+++ b/Allura/allura/lib/utils.py
@@ -31,6 +31,7 @@ from itertools import groupby
import operator as op
import collections
from urlparse import urlparse
+import urllib
import tg
import emoji
@@ -800,3 +801,47 @@ def is_nofollow_url(url):
url_domain = urlparse(url).hostname
ok_domains = re.split(r'\s*,\s*', tg.config.get('nofollow_exempt_domains', '')) + [tg.config['domain']]
return url_domain and url_domain not in ok_domains
+
+
+def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'):
+ """
+ Returns a bytestring version of 's', encoded as specified in 'encoding'.
+
+ If strings_only is True, don't convert (some) non-string-like objects.
+
+ This function was borrowed from Django
+ """
+ if strings_only and isinstance(s, (types.NoneType, int)):
+ return s
+ elif not isinstance(s, basestring):
+ try:
+ return str(s)
+ except UnicodeEncodeError:
+ if isinstance(s, Exception):
+ # An Exception subclass containing non-ASCII data that doesn't
+ # know how to print itself properly. We shouldn't raise a
+ # further exception.
+ return ' '.join([smart_str(arg, encoding, strings_only,
+ errors) for arg in s])
+ return unicode(s).encode(encoding, errors)
+ elif isinstance(s, unicode):
+ r = s.encode(encoding, errors)
+ return r
+ elif s and encoding != 'utf-8':
+ return s.decode('utf-8', errors).encode(encoding, errors)
+ else:
+ return s
+
+
+def generate_smart_str(params):
+ for (key, value) in params.iteritems():
+ yield smart_str(key), smart_str(value)
+
+
+def urlencode(params):
+ """
+ A version of Python's urllib.urlencode() function that can operate on
+ unicode strings. The parameters are first case to UTF-8 encoded strings and
+ then encoded as per normal.
+ """
+ return urllib.urlencode([i for i in generate_smart_str(params)])
diff --git a/Allura/allura/model/auth.py b/Allura/allura/model/auth.py
index 6a2f999..3ca5594 100644
--- a/Allura/allura/model/auth.py
+++ b/Allura/allura/model/auth.py
@@ -53,50 +53,6 @@ from .timeline import ActivityNode, ActivityObject
log = logging.getLogger(__name__)
-def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'):
- """
- Returns a bytestring version of 's', encoded as specified in 'encoding'.
-
- If strings_only is True, don't convert (some) non-string-like objects.
-
- This function was borrowed from Django
- """
- if strings_only and isinstance(s, (types.NoneType, int)):
- return s
- elif not isinstance(s, basestring):
- try:
- return str(s)
- except UnicodeEncodeError:
- if isinstance(s, Exception):
- # An Exception subclass containing non-ASCII data that doesn't
- # know how to print itself properly. We shouldn't raise a
- # further exception.
- return ' '.join([smart_str(arg, encoding, strings_only,
- errors) for arg in s])
- return unicode(s).encode(encoding, errors)
- elif isinstance(s, unicode):
- r = s.encode(encoding, errors)
- return r
- elif s and encoding != 'utf-8':
- return s.decode('utf-8', errors).encode(encoding, errors)
- else:
- return s
-
-
-def generate_smart_str(params):
- for (key, value) in params:
- yield smart_str(key), smart_str(value)
-
-
-def urlencode(params):
- """
- A version of Python's urllib.urlencode() function that can operate on
- unicode strings. The parameters are first case to UTF-8 encoded strings and
- then encoded as per normal.
- """
- return urllib.urlencode([i for i in generate_smart_str(params)])
-
-
class AlluraUserProperty(ForeignIdProperty):
'''
Specialized ForeignIdProperty for users, specifically to set allow_none=True