You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2026/10/12 17:43:14 UTC
(allura) branch db/misc1 created (now f2c0782fd)
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a change to branch db/misc1
in repository https://gitbox.apache.org/repos/asf/allura.git
at f2c0782fd pytest.ini warning cleanup
This branch includes the following new commits:
new f94cdd687 make memoize_cleanup work for instance methods and class methods on different python versions
new 8e2637826 deal with argparse.FileType deprecation
new 4b43ba1ef replace faust-cchardet with turbohtml charset detection, check BOM
new f2c0782fd pytest.ini warning cleanup
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(allura) 02/04: deal with argparse.FileType deprecation
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch db/misc1
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 8e2637826458a6e0a7ea388d037650d52124ade0
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Mon Oct 12 15:27:01 2026 +0000
deal with argparse.FileType deprecation
---
scripts/perf/call_count.py | 12 ++++++------
scripts/perf/parse_timings.py | 5 +++--
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/scripts/perf/call_count.py b/scripts/perf/call_count.py
index 32cf5f3d8..2cc5290f6 100755
--- a/scripts/perf/call_count.py
+++ b/scripts/perf/call_count.py
@@ -49,7 +49,7 @@ def parse_args():
parser.add_argument('--debug-html', action='store_true', default=False,
help='Save HTML responses as local files')
parser.add_argument(
- '--data-file', default='call_counts.csv', type=argparse.FileType('a'),
+ '--data-file', default='call_counts.csv',
help='CSV file that is appended to')
parser.add_argument('--id', default='',
help='An identifier for this run. Examples:\n'
@@ -141,11 +141,11 @@ def write_csv(counts, id, data_file):
if id:
cols = ['id'] + cols
row = dict(counts, id=id)
- csv_out = csv.DictWriter(data_file, cols)
- if data_file.tell() == 0:
- csv_out.writeheader()
- csv_out.writerow(row)
- data_file.close()
+ with open(data_file, 'a', newline='') as f:
+ csv_out = csv.DictWriter(f, cols)
+ if f.tell() == 0:
+ csv_out.writeheader()
+ csv_out.writerow(row)
if __name__ == '__main__':
diff --git a/scripts/perf/parse_timings.py b/scripts/perf/parse_timings.py
index e55dd833a..f0db75db0 100644
--- a/scripts/perf/parse_timings.py
+++ b/scripts/perf/parse_timings.py
@@ -28,12 +28,13 @@
help='e.g. tickets, or discussion, etc')
parser.add_argument('-t', '--timings', type=str, nargs='+', metavar='TIMING', required=True,
help='e.g. total ming mongo sidebar jinja ...')
-parser.add_argument('-i', '--input-file', type=argparse.FileType('r'), nargs='?', default=sys.stdin,
+parser.add_argument('-i', '--input-file', nargs='?', default='-',
help='Filename, or use stdin by default')
args = parser.parse_args()
timings = []
-for line in args.input_file:
+input_file = sys.stdin if args.input_file in (None, '-') else open(args.input_file)
+for line in input_file:
data = json.loads(line)
try:
typ = data['message']['request_category']
(allura) 03/04: replace faust-cchardet with turbohtml charset detection, check BOM
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch db/misc1
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 4b43ba1ef30157e3162f34ba72116913f98068d8
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Mon Oct 12 17:03:03 2026 +0000
replace faust-cchardet with turbohtml charset detection, check BOM
- faust-cchardet sometimes has slowdowns https://github.com/faust-streaming/cChardet/issues/57
- and can be hard to compile if prebuilt wheel isn't available
- turbohtml is just as fast, and we are depending on it now anyway
- adds some tests too
---
Allura/allura/lib/helpers.py | 31 ++++++++++++++++++++++++++++---
Allura/allura/tests/test_helpers.py | 17 +++++++++++++++++
requirements.in | 2 +-
requirements.txt | 6 +++---
4 files changed, 49 insertions(+), 7 deletions(-)
diff --git a/Allura/allura/lib/helpers.py b/Allura/allura/lib/helpers.py
index bd02ba694..bd4c50de1 100644
--- a/Allura/allura/lib/helpers.py
+++ b/Allura/allura/lib/helpers.py
@@ -42,11 +42,12 @@
import http.client
import importlib.resources
import importlib.metadata
+import codecs
import emoji
import tg
import six
-import cchardet as chardet
+from turbohtml.detect import detect as detect_encoding
import webob
from formencode.validators import FancyValidator
from dateutil.parser import parse
@@ -257,6 +258,30 @@ def _attempt_encodings(s, encodings):
return str(repr(str(s)))[1:-1]
+# WHATWG encoding names mapped to the Python codec closest to what they decode (same-named ones are stricter)
+_WHATWG_TO_PYTHON_CODEC = {
+ 'EUC-KR': 'cp949',
+ 'Shift_JIS': 'cp932',
+ 'GBK': 'gb18030',
+ 'Big5': 'big5hkscs',
+ 'x-mac-cyrillic': 'mac-cyrillic',
+}
+
+
+def _guess_encoding(s: bytes) -> str | None:
+ # Returns a strict Python codec name, so bad bytes raise and really_unicode() tries its next guess.
+ # To decode leniently instead (bad bytes become U+FFFD, latin-1 fallback never reached), return match.codec:
+ # turbohtml registers those "whatwg-*" codecs and they never raise.
+ if s.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
+ return 'utf-32' # turbohtml >= 1.14 follows HTML, which has no UTF-32
+ match = detect_encoding(s)
+ if match.encoding is None:
+ return None
+ if match.bom and match.encoding.startswith('UTF-16'):
+ return 'utf-16' # generic codec strips the BOM
+ return _WHATWG_TO_PYTHON_CODEC.get(match.encoding, match.encoding)
+
+
def really_unicode(s):
if isinstance(s, str):
# default case. Also lets Markup() instances be preserved
@@ -266,8 +291,8 @@ def really_unicode(s):
def encodings():
yield None
yield 'utf-8'
- yield chardet.detect(s[:1024])['encoding']
- yield chardet.detect(s)['encoding']
+ yield _guess_encoding(s[:1024])
+ yield _guess_encoding(s)
yield 'latin-1'
return _attempt_encodings(s, encodings())
diff --git a/Allura/allura/tests/test_helpers.py b/Allura/allura/tests/test_helpers.py
index 986c2a40f..9a997e53d 100644
--- a/Allura/allura/tests/test_helpers.py
+++ b/Allura/allura/tests/test_helpers.py
@@ -111,6 +111,23 @@ def test_really_unicode():
assert isinstance(s, str)
# try non-ascii string in legacy 8bit encoding
h.really_unicode('\u0410\u0401'.encode('cp1251'))
+ text = 'The vendor\u2019s \u201cfinal\u201d quote is \u20ac1,250\u2026 we\u2019ll review it Monday. ' * 3
+ assert h.really_unicode(text.encode('cp1252')) == text
+ text = '\ub620\ubc29\uac01\ud558 \ud55c\uad6d\uc5b4 \ud14d\uc2a4\ud2b8\uc785\ub2c8\ub2e4. \uc624\ud508 \uc18c\uc2a4 \uc18c\ud504\ud2b8\uc6e8\uc5b4\ub294 \uc804 \uc138\uacc4 \uc790\uc6d0 \ubd09\uc0ac\uc790\ub4e4\uc774 \uac1c\ubc1c\ud569\ub2c8\ub2e4. ' * 5
+ assert h.really_unicode(text.encode('cp949')) == text
+ text = '\u2460\u756a\u76ee\u306e\u9805\u76ee\u3067\u3059\u3002\u30aa\u30fc\u30d7\u30f3\u30bd\u30fc\u30b9\u30bd\u30d5\u30c8\u30a6\u30a7\u30a2\u306f\u4e16\u754c\u4e2d\u3067\u958b\u767a\u3055\u308c\u3066\u3044\u307e\u3059\u3002' * 5
+ assert h.really_unicode(text.encode('cp932')) == text
+ text = '我哋今日冇時間，佢話啲嘢未搞掂。開源軟體由世界各地的志願者共同開發。歡迎提交錯誤報告。' * 5
+ assert h.really_unicode(text.encode('big5hkscs')) == text
+ # BOM is stripped from utf-16
+ assert h.really_unicode('\ufeffhello'.encode('utf-16-le')) == 'hello'
+ assert h.really_unicode('\ufeffhello'.encode('utf-16-be')) == 'hello'
+ assert h.really_unicode('\ufeffhello'.encode('utf-32-le')) == 'hello'
+ assert h.really_unicode('\ufeffhello'.encode('utf-32-be')) == 'hello'
+ # undecodable bytes fall back to latin-1 rather than being replaced
+ s = h.really_unicode(b'caf\xc3\xa9 \x81 end')
+ assert '\ufffd' not in s
+ assert s.encode('latin-1') == b'caf\xc3\xa9 \x81 end'
# ensure invalid encodings are handled gracefully
s = h._attempt_encodings(b'foo', ['LKDJFLDK'])
assert isinstance(s, str)
diff --git a/requirements.in b/requirements.in
index 43af7b8bf..dee3c5dbb 100644
--- a/requirements.in
+++ b/requirements.in
@@ -9,7 +9,6 @@ decorator
defopt
EasyWidgets
emoji
-faust-cchardet
feedgenerator
feedparser
FormEncode
@@ -53,6 +52,7 @@ six
sxsdiff
TimerMiddleware
TurboGears2
+turbohtml
WebHelpers2
WebOb
werkzeug
diff --git a/requirements.txt b/requirements.txt
index 8842d0a0c..2f25121ae 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -78,8 +78,6 @@ emoji==2.15.0
# via -r requirements.in
execnet==2.1.2
# via pytest-xdist
-faust-cchardet==2.2.1
- # via -r requirements.in
feedgenerator==2.2.1
# via -r requirements.in
feedparser==6.0.12
@@ -312,7 +310,9 @@ translationstring==1.4
turbogears2==2.5.0
# via -r requirements.in
turbohtml==1.13.1
- # via pypeline
+ # via
+ # -r requirements.in
+ # pypeline
typing-extensions==4.16.0
# via beautifulsoup4
tzdata==2025.2 ; sys_platform == 'win32'
(allura) 04/04: pytest.ini warning cleanup
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch db/misc1
in repository https://gitbox.apache.org/repos/asf/allura.git
commit f2c0782fd0c3a2615bb043dec0656e48e5a4fa84
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Mon Oct 12 17:41:53 2026 +0000
pytest.ini warning cleanup
---
pytest.ini | 3 ---
1 file changed, 3 deletions(-)
diff --git a/pytest.ini b/pytest.ini
index 5408aff71..188c342d9 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -34,9 +34,6 @@ filterwarnings =
ignore:'crypt' is deprecated:DeprecationWarning:passlib.utils
ignore:Accessing argon2.__version__ is deprecated:DeprecationWarning:passlib.handlers.argon2
- # py3.12
- ignore::DeprecationWarning:smtpd
-
# py3.13
ignore:'cgi' is deprecated:DeprecationWarning:webob.compat
ignore:'cgi' is deprecated:DeprecationWarning:formencode.validators
(allura) 01/04: make memoize_cleanup work for instance methods and class methods on different python versions
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch db/misc1
in repository https://gitbox.apache.org/repos/asf/allura.git
commit f94cdd687f233823b2807ee06f5fde75ca0a38a8
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Fri Oct 9 16:23:59 2026 +0000
make memoize_cleanup work for instance methods and class methods on different python versions
---
Allura/allura/lib/decorators.py | 8 ++++++++
Allura/allura/tests/test_decorators.py | 33 ++++++++++++++++++++++++++++++++-
2 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/Allura/allura/lib/decorators.py b/Allura/allura/lib/decorators.py
index f6c578168..f5e331134 100644
--- a/Allura/allura/lib/decorators.py
+++ b/Allura/allura/lib/decorators.py
@@ -18,6 +18,7 @@
import inspect
import json
import logging
+import types
from http.cookies import SimpleCookie as Cookie
from collections import defaultdict
from urllib.parse import unquote
@@ -194,6 +195,13 @@ def memoize_cleanup(obj):
"""
Remove any _memoize_dic_* keys that were created by @memoize on methods
"""
+ if isinstance(obj, types.MethodType):
+ # @classmethod on top of @memoize caches on the class before py3.13, on the function after
+ try:
+ delattr(obj.__self__, f"_memoize_dic__{obj.__func__.__name__}")
+ except AttributeError:
+ pass
+ obj = obj.__func__
attrs = obj.keys() if hasattr(obj, 'keys') else obj.__dir__()
for k in list(attrs):
if k.startswith('_memoize_dic'):
diff --git a/Allura/allura/tests/test_decorators.py b/Allura/allura/tests/test_decorators.py
index 5b7ed112f..0cfc5b763 100644
--- a/Allura/allura/tests/test_decorators.py
+++ b/Allura/allura/tests/test_decorators.py
@@ -19,7 +19,7 @@
import random
import gc
-from allura.lib.decorators import task, memoize
+from allura.lib.decorators import task, memoize, memoize_cleanup
from alluratest.controller import setup_basic_test
@@ -121,6 +121,37 @@ def other(self, do_random):
assert r2other1 != other1
assert r2other1 == r2other2
+ def test_cleanup_method(self):
+
+ class Randomy:
+ @memoize
+ def randomy(self):
+ return random.random()
+
+ @memoize
+ def other(self):
+ return random.random()
+
+ r = Randomy()
+ rand1 = r.randomy()
+ other1 = r.other()
+ memoize_cleanup(r.randomy)
+ assert r.randomy() != rand1
+ assert r.other() == other1
+
+ def test_cleanup_classmethod(self):
+
+ class Randomy:
+ @classmethod
+ @memoize
+ def randomy(cls):
+ return random.random()
+
+ rand1 = Randomy.randomy()
+ assert Randomy.randomy() == rand1
+ memoize_cleanup(Randomy.randomy)
+ assert Randomy.randomy() != rand1
+
def test_methods_garbage_collection(self):
class Randomy: