You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by fu...@apache.org on 2021/01/18 16:30:27 UTC
svn commit: r1885656 - /subversion/trunk/tools/hook-scripts/mailer/mailer.py
Author: futatuki
Date: Mon Jan 18 16:30:27 2021
New Revision: 1885656
URL: http://svn.apache.org/viewvc?rev=1885656&view=rev
Log:
mailer.py: Restore Python 2 support.
* tools/hook-scripts/mailer/mailer.py
():
- Absorb difference of import module name.
- import codecs, to examine equivalence of codecs.
- Don't import locale.
(to_bytes): New function to absorb differnce between Python 2 and Python 3.
Replace occurence of .encode('utf-8') with this whole in this file.
(to_str): New function to absorb difference between Python 2 and Python 3.
Replace occurence of .decode('utf-8') with this whole in this file.
(_stdin): New variable to hold bytes I/O object for stdin
(_stdout): New variable to hold bytes I/O object for stdout
(OutputBase.make_subject):
- Truncate subject by number of bytes, for compatibility before r1884427.
- Truncate subject on character boundary (don't truncate in the middle of
multi-byte sequence of a UTF-8 character).
(StandardOutput.__init__): Use bytes output interface _stdout for
StandardOutput.write_binary.
(StandardOutput.wirte):
- Override this method only if on Python 3 and encoding of stdout is not
'utf-8'
- Use sys.stdout.encoding instead of locale.getpreferredencoding().
(PropChange.generate): Use bytes input interface _stdin.
(Lock.__init__): Use bytes input interface _stdin.
(DiffURLSelections._get_url): Remove extra trailing spaces.
(DiffGenerator.__getitem__): Prepare bytes and str representation of
base_path, 'base_path_bytes' and 'base_path', then use them properly.
Modified:
subversion/trunk/tools/hook-scripts/mailer/mailer.py
Modified: subversion/trunk/tools/hook-scripts/mailer/mailer.py
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/hook-scripts/mailer/mailer.py?rev=1885656&r1=1885655&r2=1885656&view=diff
==============================================================================
--- subversion/trunk/tools/hook-scripts/mailer/mailer.py (original)
+++ subversion/trunk/tools/hook-scripts/mailer/mailer.py Mon Jan 18 16:30:27 2021
@@ -46,15 +46,21 @@
import os
import sys
-import configparser
-from urllib.parse import quote as _url_quote
+if sys.hexversion >= 0x3000000:
+ PY3 = True
+ import configparser
+ from urllib.parse import quote as _url_quote
+else:
+ PY3 = False
+ import ConfigParser as configparser
+ from urllib import quote as _url_quote
import time
import subprocess
from io import BytesIO
import smtplib
import re
import tempfile
-import locale
+import codecs
# Minimal version of Subversion's bindings required
_MIN_SVN_VERSION = [1, 5, 0]
@@ -73,6 +79,28 @@ if _MIN_SVN_VERSION > [svn.core.SVN_VER_
% ".".join([str(x) for x in _MIN_SVN_VERSION]))
sys.exit(1)
+# Absorb difference between Python 2 and Python >= 3
+if PY3:
+ def to_bytes(x):
+ return x.encode('utf-8')
+
+ def to_str(x):
+ return x.decode('utf-8')
+
+ # We never use sys.stdin nor sys.stdout TextIOwrapper.
+ _stdin = sys.stdin.buffer
+ _stdout = sys.stdout.buffer
+else:
+ # Python 2
+ def to_bytes(x):
+ return x
+
+ def to_str(x):
+ return x
+
+ _stdin = sys.stdin
+ _stdout = sys.stdout
+
SEPARATOR = '=' * 78
@@ -151,8 +179,16 @@ class OutputBase:
except ValueError:
truncate_subject = 0
- if truncate_subject and len(subject) > truncate_subject:
- subject = subject[:(truncate_subject - 3)] + "..."
+ # truncate subject as UTF-8 string.
+ # Note: there still exists an issue on combining characters.
+ if truncate_subject:
+ bsubject = to_bytes(subject)
+ if len(bsubject) > truncate_subject:
+ idx = truncate_subject - 2
+ while b'\x80' <= bsubject[idx-1:idx] <= b'\xbf':
+ idx -= 1
+ subject = to_str(bsubject[:idx-1]) + "..."
+
return subject
def start(self, group, params):
@@ -177,7 +213,7 @@ class OutputBase:
def write(self, output):
"""Append the literal text string OUTPUT to the output representation."""
- return self.write_binary(output.encode('utf-8'))
+ return self.write_binary(to_bytes(output))
def run(self, cmd):
"""Override this method, if the default implementation is not sufficient.
@@ -356,7 +392,7 @@ class StandardOutput(OutputBase):
def __init__(self, cfg, repos, prefix_param):
OutputBase.__init__(self, cfg, repos, prefix_param)
- self.write_binary = sys.stdout.buffer.write
+ self.write_binary = _stdout.write
def start(self, group, params):
self.write("Group: " + (group or "defaults") + "\n")
@@ -365,10 +401,11 @@ class StandardOutput(OutputBase):
def finish(self):
pass
- def write(self, output):
- """Write text as *default* encoding string"""
- return self.write_binary(output.encode(locale.getpreferredencoding(),
- 'backslashreplace'))
+ if (PY3 and (codecs.lookup(sys.stdout.encoding) != codecs.lookup('utf-8'))):
+ def write(self, output):
+ """Write text as *default* encoding string"""
+ return self.write_binary(output.encode(sys.stdout.encoding,
+ 'backslashreplace'))
class PipeOutput(MailedOutput):
@@ -431,8 +468,7 @@ class Commit(Messenger):
self.changelist = sorted(editor.get_changes().items())
- log = (repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG)
- or b'').decode('utf-8')
+ log = to_str(repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG) or b'')
# collect the set of groups and the unique sets of params for the options
self.groups = { }
@@ -451,7 +487,7 @@ class Commit(Messenger):
# figure out the changed directories
dirs = { }
for path, change in self.changelist:
- path = path.decode('utf-8')
+ path = to_str(path)
if change.item_kind == svn.core.svn_node_dir:
dirs[path] = None
else:
@@ -542,7 +578,7 @@ class PropChange(Messenger):
elif self.action == 'M':
self.output.write('Property diff:\n')
tempfile1 = tempfile.NamedTemporaryFile()
- tempfile1.write(sys.stdin.buffer.read())
+ tempfile1.write(_stdin.read())
tempfile1.flush()
tempfile2 = tempfile.NamedTemporaryFile()
tempfile2.write(self.repos.get_rev_prop(self.propname))
@@ -604,8 +640,7 @@ class Lock(Messenger):
or 'unlock_subject_prefix'))
# read all the locked paths from STDIN and strip off the trailing newlines
- self.dirlist = [x.decode('utf-8').rstrip()
- for x in sys.stdin.buffer.readlines()]
+ self.dirlist = [to_str(x).rstrip() for x in _stdin.readlines()]
# collect the set of groups and the unique sets of params for the options
self.groups = { }
@@ -634,7 +669,7 @@ class Lock(Messenger):
# The lock comment is the same for all paths, so we can just pull
# the comment for the first path in the dirlist and cache it.
self.lock = svn.fs.svn_fs_get_lock(self.repos.fs_ptr,
- self.dirlist[0].encode('utf-8'),
+ to_bytes(self.dirlist[0]),
self.pool)
def generate(self):
@@ -709,7 +744,7 @@ class DiffURLSelections:
# KeyError exceptions.
params = self.params.copy()
params['path'] = _url_quote(change.path) if change.path else None
- params['base_path'] = (_url_quote(change.base_path)
+ params['base_path'] = (_url_quote(change.base_path)
if change.base_path else None)
params['rev'] = repos_rev
params['base_rev'] = change.base_rev
@@ -764,8 +799,7 @@ def generate_content(renderer, cfg, repo
author=repos.author,
date=date,
rev=repos.rev,
- log=(repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG)
- or b'').decode('utf-8'),
+ log=to_str(repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG) or b''),
commit_url=commit_url,
added_data=generate_list('A', changelist, paths, True),
replaced_data=generate_list('R', changelist, paths, True),
@@ -873,7 +907,9 @@ class DiffGenerator:
# figure out if/how to generate a diff
- base_path = remove_leading_slashes(change.base_path)
+ base_path_bytes = remove_leading_slashes(change.base_path)
+ base_path = (to_str(base_path_bytes)
+ if base_path_bytes is not None else None)
if change.action == svn.repos.CHANGE_ACTION_DELETE:
# it was delete.
kind = 'D'
@@ -884,10 +920,9 @@ class DiffGenerator:
# show the diff?
if self.diffsels.delete:
diff = svn.fs.FileDiff(self.repos.get_root(change.base_rev),
- base_path, None, None, self.pool)
+ base_path_bytes, None, None, self.pool)
- label1 = '%s\t%s\t(r%s)' % (base_path.decode('utf-8'), self.date,
- change.base_rev)
+ label1 = '%s\t%s\t(r%s)' % (base_path, self.date, change.base_rev)
label2 = '/dev/null\t00:00:00 1970\t(deleted)'
singular = True
@@ -906,14 +941,13 @@ class DiffGenerator:
# show the diff?
if self.diffsels.modify:
diff = svn.fs.FileDiff(self.repos.get_root(change.base_rev),
- base_path,
+ base_path_bytes,
self.repos.root_this, change.path,
self.pool)
- label1 = '%s\t%s\t(r%s, copy source)' \
- % (base_path.decode('utf-8'), base_date, change.base_rev)
- label2 = '%s\t%s\t(r%s)' \
- % (change.path.decode('utf-8'), self.date, \
- self.repos.rev)
+ label1 = ('%s\t%s\t(r%s, copy source)'
+ % (base_path, base_date, change.base_rev))
+ label2 = ('%s\t%s\t(r%s)'
+ % (to_str(change.path), self.date, self.repos.rev))
singular = False
else:
# this file was copied.
@@ -921,12 +955,12 @@ class DiffGenerator:
if self.diffsels.copy:
diff = svn.fs.FileDiff(None, None, self.repos.root_this,
change.path, self.pool)
- label1 = '/dev/null\t00:00:00 1970\t' \
- '(empty, because file is newly added)'
- label2 = '%s\t%s\t(r%s, copy of r%s, %s)' \
- % (change.path.decode('utf-8'), self.date,
- self.repos.rev, change.base_rev,
- base_path.decode('utf-8'))
+ label1 = ('/dev/null\t00:00:00 1970\t'
+ '(empty, because file is newly added)')
+ label2 = ('%s\t%s\t(r%s, copy of r%s, %s)'
+ % (to_str(change.path),
+ self.date, self.repos.rev, change.base_rev,
+ base_path))
singular = False
else:
# the file was added.
@@ -942,7 +976,7 @@ class DiffGenerator:
label1 = '/dev/null\t00:00:00 1970\t' \
'(empty, because file is newly added)'
label2 = '%s\t%s\t(r%s)' \
- % (change.path.decode('utf-8'), self.date, self.repos.rev)
+ % (to_str(change.path), self.date, self.repos.rev)
singular = True
elif not change.text_changed:
@@ -962,9 +996,9 @@ class DiffGenerator:
self.repos.root_this, change.path,
self.pool)
label1 = '%s\t%s\t(r%s)' \
- % (base_path.decode('utf-8'), base_date, change.base_rev)
+ % (base_path, base_date, change.base_rev)
label2 = '%s\t%s\t(r%s)' \
- % (change.path.decode('utf-8'), self.date, self.repos.rev)
+ % (to_str(change.path), self.date, self.repos.rev)
singular = False
if diff:
@@ -1154,7 +1188,7 @@ class TextCommitRenderer:
props = ' (props changed)'
else:
props = ''
- w(' %s%s%s\n' % (d.path.decode('utf-8'), is_dir, props))
+ w(' %s%s%s\n' % (to_str(d.path), is_dir, props))
if d.copied:
if is_dir:
text = ''
@@ -1163,7 +1197,7 @@ class TextCommitRenderer:
else:
text = ' unchanged'
w(' - copied%s from r%d, %s%s\n'
- % (text, d.base_rev, d.base_path.decode('utf-8'), is_dir))
+ % (text, d.base_rev, to_str(d.base_path), is_dir))
def _render_diffs(self, diffs, section_header):
"""Render diffs. Write the SECTION_HEADER if there are actually
@@ -1180,20 +1214,20 @@ class TextCommitRenderer:
w(section_header)
section_header_printed = True
if diff.kind == 'D':
- w('\nDeleted: %s\n' % diff.base_path.decode('utf-8'))
+ w('\nDeleted: %s\n' % to_str(diff.base_path))
elif diff.kind == 'A':
- w('\nAdded: %s\n' % diff.path.decode('utf-8'))
+ w('\nAdded: %s\n' % to_str(diff.path))
elif diff.kind == 'C':
w('\nCopied: %s (from r%d, %s)\n'
- % (diff.path.decode('utf-8'), diff.base_rev,
- diff.base_path.decode('utf-8')))
+ % (to_str(diff.path), diff.base_rev,
+ to_str(diff.base_path)))
elif diff.kind == 'W':
w('\nCopied and modified: %s (from r%d, %s)\n'
- % (diff.path.decode('utf-8'), diff.base_rev,
- diff.base_path.decode('utf-8')))
+ % (to_str(diff.path), diff.base_rev,
+ to_str(diff.base_path)))
else:
# kind == 'M'
- w('\nModified: %s\n' % diff.path.decode('utf-8'))
+ w('\nModified: %s\n' % to_str(diff.path))
if diff.diff_url:
w('URL: %s\n' % diff.diff_url)
@@ -1232,7 +1266,7 @@ class Repository:
self.author = self.get_rev_prop(svn.core.SVN_PROP_REVISION_AUTHOR)
if self.author is not None:
- self.author = self.author.decode('utf-8')
+ self.author = to_str(self.author)
def get_rev_prop(self, propname, rev = None):
if not rev:
@@ -1441,9 +1475,9 @@ class Config:
"Return the path's associated groups."
groups = []
for group, pattern, exclude_pattern, repos_params, search_logmsg_re in self._group_re:
- match = pattern.match(path.decode('utf-8'))
+ match = pattern.match(to_str(path))
if match:
- if exclude_pattern and exclude_pattern.match(path.decode('utf-8')):
+ if exclude_pattern and exclude_pattern.match(to_str(path)):
continue
params = repos_params.copy()
params.update(match.groupdict())
@@ -1522,8 +1556,7 @@ if the property was added, modified or d
usage()
cmd = sys.argv[1]
- repos_dir = svn.core.svn_path_canonicalize(sys.argv[2].encode('utf-8'))
- repos_dir = repos_dir.decode('utf-8')
+ repos_dir = to_str(svn.core.svn_path_canonicalize(to_bytes(sys.argv[2])))
try:
expected_args = cmd_list[cmd]
except KeyError:
Re: svn commit: r1885656 -
/subversion/trunk/tools/hook-scripts/mailer/mailer.py
Posted by Stefan Sperling <st...@apache.org>.
On Fri, Jan 22, 2021 at 11:58:40PM +0900, Yasuhito FUTATSUKI wrote:
> On 2021/01/22 21:05, Stefan Sperling wrote:
> > On Fri, Jan 22, 2021 at 12:48:24PM +0100, Stefan Sperling wrote:
> >> On Mon, Jan 18, 2021 at 04:30:27PM -0000, futatuki@apache.org wrote:
> >>> Author: futatuki
> >>> Date: Mon Jan 18 16:30:27 2021
> >>> New Revision: 1885656
> >>>
> >>> URL: http://svn.apache.org/viewvc?rev=1885656&view=rev
> >>> Log:
> >>> mailer.py: Restore Python 2 support.
> >>
> >> mailer.py's test suite starts failing with Python3 after this change.
> >> I don't know yet how this could be fixed. I will try to find out.
> >
> > It seems this was fairly easy to fix. See http://svn.apache.org/r1885784
>
> Thank you very much, and I'm sorry bother you. Actuall I didn't mailer.py's
> test suite in source tree but used another scripts for check that I wrote
> and isn't completed as tests yet, and my script didn't hit the error
> pattern (perhaps my test case is always base_path is None). I'm still working
> to add my script as additional tests, because current tests also don't
> covered SMTPOutput and PipeOutput for Output, and propchange, propchange2,
> lock and unlock commands, although my check script don't propchange
> commnand yet.
Nice, that is great news! Improvements to the script's test suite will
be very useful indeed.
Thank you,
Stefan
Re: svn commit: r1885656 -
/subversion/trunk/tools/hook-scripts/mailer/mailer.py
Posted by Yasuhito FUTATSUKI <fu...@yf.bsdclub.org>.
On 2021/01/22 21:05, Stefan Sperling wrote:
> On Fri, Jan 22, 2021 at 12:48:24PM +0100, Stefan Sperling wrote:
>> On Mon, Jan 18, 2021 at 04:30:27PM -0000, futatuki@apache.org wrote:
>>> Author: futatuki
>>> Date: Mon Jan 18 16:30:27 2021
>>> New Revision: 1885656
>>>
>>> URL: http://svn.apache.org/viewvc?rev=1885656&view=rev
>>> Log:
>>> mailer.py: Restore Python 2 support.
>>
>> mailer.py's test suite starts failing with Python3 after this change.
>> I don't know yet how this could be fixed. I will try to find out.
>
> It seems this was fairly easy to fix. See http://svn.apache.org/r1885784
Thank you very much, and I'm sorry bother you. Actuall I didn't mailer.py's
test suite in source tree but used another scripts for check that I wrote
and isn't completed as tests yet, and my script didn't hit the error
pattern (perhaps my test case is always base_path is None). I'm still working
to add my script as additional tests, because current tests also don't
covered SMTPOutput and PipeOutput for Output, and propchange, propchange2,
lock and unlock commands, although my check script don't propchange
commnand yet.
Cheers,
--
Yasuhito FUTATSUKI <fu...@yf.bsclub.org>
Re: svn commit: r1885656 -
/subversion/trunk/tools/hook-scripts/mailer/mailer.py
Posted by Stefan Sperling <st...@apache.org>.
On Fri, Jan 22, 2021 at 12:48:24PM +0100, Stefan Sperling wrote:
> On Mon, Jan 18, 2021 at 04:30:27PM -0000, futatuki@apache.org wrote:
> > Author: futatuki
> > Date: Mon Jan 18 16:30:27 2021
> > New Revision: 1885656
> >
> > URL: http://svn.apache.org/viewvc?rev=1885656&view=rev
> > Log:
> > mailer.py: Restore Python 2 support.
>
> mailer.py's test suite starts failing with Python3 after this change.
> I don't know yet how this could be fixed. I will try to find out.
It seems this was fairly easy to fix. See http://svn.apache.org/r1885784
Re: svn commit: r1885656 -
/subversion/trunk/tools/hook-scripts/mailer/mailer.py
Posted by Stefan Sperling <st...@apache.org>.
On Mon, Jan 18, 2021 at 04:30:27PM -0000, futatuki@apache.org wrote:
> Author: futatuki
> Date: Mon Jan 18 16:30:27 2021
> New Revision: 1885656
>
> URL: http://svn.apache.org/viewvc?rev=1885656&view=rev
> Log:
> mailer.py: Restore Python 2 support.
mailer.py's test suite starts failing with Python3 after this change.
I don't know yet how this could be fixed. I will try to find out.
$ pwd
/home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests
$ svn diff ~/svn/svn-trunk
Index: /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/mailer.py
===================================================================
--- /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/mailer.py (revision 1885780)
+++ /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/mailer.py (working copy)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/home/stsp/svn/prefix/python/bin/python3
# -*- coding: utf-8 -*-
#
#
$ ./mailer-t1.sh mailer-init.33014/repos ../mailer.py
Traceback (most recent call last):
File "../mailer.py", line 1583, in <module>
sys.argv[3:3+expected_args])
File "/home/stsp/svn/prefix/svn-trunk/lib/svn-python/svn/core.py", line 324, in run_app
return func(application_pool, *args, **kw)
File "../mailer.py", line 147, in main
return messenger.generate()
File "../mailer.py", line 533, in generate
group, params, paths, subpool)
File "../mailer.py", line 817, in generate_content
renderer.render(data)
File "../mailer.py", line 1167, in render
self._render_diffs(data.diffs, '')
File "../mailer.py", line 1223, in _render_diffs
to_str(diff.base_path)))
File "../mailer.py", line 88, in to_str
return x.decode('utf-8')
AttributeError: 'str' object has no attribute 'decode'
Traceback (most recent call last):
File "../mailer.py", line 1583, in <module>
sys.argv[3:3+expected_args])
File "/home/stsp/svn/prefix/svn-trunk/lib/svn-python/svn/core.py", line 324, in run_app
return func(application_pool, *args, **kw)
File "../mailer.py", line 147, in main
return messenger.generate()
File "../mailer.py", line 533, in generate
group, params, paths, subpool)
File "../mailer.py", line 817, in generate_content
renderer.render(data)
File "../mailer.py", line 1167, in render
self._render_diffs(data.diffs, '')
File "../mailer.py", line 1227, in _render_diffs
to_str(diff.base_path)))
File "../mailer.py", line 88, in to_str
return x.decode('utf-8')
AttributeError: 'str' object has no attribute 'decode'
Traceback (most recent call last):
File "../mailer.py", line 1583, in <module>
sys.argv[3:3+expected_args])
File "/home/stsp/svn/prefix/svn-trunk/lib/svn-python/svn/core.py", line 324, in run_app
return func(application_pool, *args, **kw)
File "../mailer.py", line 147, in main
return messenger.generate()
File "../mailer.py", line 533, in generate
group, params, paths, subpool)
File "../mailer.py", line 817, in generate_content
renderer.render(data)
File "../mailer.py", line 1167, in render
self._render_diffs(data.diffs, '')
File "../mailer.py", line 1217, in _render_diffs
w('\nDeleted: %s\n' % to_str(diff.base_path))
File "../mailer.py", line 88, in to_str
return x.decode('utf-8')
AttributeError: 'str' object has no attribute 'decode'
Traceback (most recent call last):
File "../mailer.py", line 1583, in <module>
sys.argv[3:3+expected_args])
File "/home/stsp/svn/prefix/svn-trunk/lib/svn-python/svn/core.py", line 324, in run_app
return func(application_pool, *args, **kw)
File "../mailer.py", line 147, in main
return messenger.generate()
File "../mailer.py", line 533, in generate
group, params, paths, subpool)
File "../mailer.py", line 817, in generate_content
renderer.render(data)
File "../mailer.py", line 1167, in render
self._render_diffs(data.diffs, '')
File "../mailer.py", line 1223, in _render_diffs
to_str(diff.base_path)))
File "../mailer.py", line 88, in to_str
return x.decode('utf-8')
AttributeError: 'str' object has no attribute 'decode'
current mailer.py output in: /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.current
dos2unix: converting file /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.current to Unix format...
diff -q /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.output /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.current
Files /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.output and /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.current differ
$
> * tools/hook-scripts/mailer/mailer.py
> ():
> - Absorb difference of import module name.
> - import codecs, to examine equivalence of codecs.
> - Don't import locale.
> (to_bytes): New function to absorb differnce between Python 2 and Python 3.
> Replace occurence of .encode('utf-8') with this whole in this file.
> (to_str): New function to absorb difference between Python 2 and Python 3.
> Replace occurence of .decode('utf-8') with this whole in this file.
> (_stdin): New variable to hold bytes I/O object for stdin
> (_stdout): New variable to hold bytes I/O object for stdout
> (OutputBase.make_subject):
> - Truncate subject by number of bytes, for compatibility before r1884427.
> - Truncate subject on character boundary (don't truncate in the middle of
> multi-byte sequence of a UTF-8 character).
> (StandardOutput.__init__): Use bytes output interface _stdout for
> StandardOutput.write_binary.
> (StandardOutput.wirte):
> - Override this method only if on Python 3 and encoding of stdout is not
> 'utf-8'
> - Use sys.stdout.encoding instead of locale.getpreferredencoding().
> (PropChange.generate): Use bytes input interface _stdin.
> (Lock.__init__): Use bytes input interface _stdin.
> (DiffURLSelections._get_url): Remove extra trailing spaces.
> (DiffGenerator.__getitem__): Prepare bytes and str representation of
> base_path, 'base_path_bytes' and 'base_path', then use them properly.
>
> Modified:
> subversion/trunk/tools/hook-scripts/mailer/mailer.py
>
> Modified: subversion/trunk/tools/hook-scripts/mailer/mailer.py
> URL: http://svn.apache.org/viewvc/subversion/trunk/tools/hook-scripts/mailer/mailer.py?rev=1885656&r1=1885655&r2=1885656&view=diff
> ==============================================================================
> --- subversion/trunk/tools/hook-scripts/mailer/mailer.py (original)
> +++ subversion/trunk/tools/hook-scripts/mailer/mailer.py Mon Jan 18 16:30:27 2021
> @@ -46,15 +46,21 @@
>
> import os
> import sys
> -import configparser
> -from urllib.parse import quote as _url_quote
> +if sys.hexversion >= 0x3000000:
> + PY3 = True
> + import configparser
> + from urllib.parse import quote as _url_quote
> +else:
> + PY3 = False
> + import ConfigParser as configparser
> + from urllib import quote as _url_quote
> import time
> import subprocess
> from io import BytesIO
> import smtplib
> import re
> import tempfile
> -import locale
> +import codecs
>
> # Minimal version of Subversion's bindings required
> _MIN_SVN_VERSION = [1, 5, 0]
> @@ -73,6 +79,28 @@ if _MIN_SVN_VERSION > [svn.core.SVN_VER_
> % ".".join([str(x) for x in _MIN_SVN_VERSION]))
> sys.exit(1)
>
> +# Absorb difference between Python 2 and Python >= 3
> +if PY3:
> + def to_bytes(x):
> + return x.encode('utf-8')
> +
> + def to_str(x):
> + return x.decode('utf-8')
> +
> + # We never use sys.stdin nor sys.stdout TextIOwrapper.
> + _stdin = sys.stdin.buffer
> + _stdout = sys.stdout.buffer
> +else:
> + # Python 2
> + def to_bytes(x):
> + return x
> +
> + def to_str(x):
> + return x
> +
> + _stdin = sys.stdin
> + _stdout = sys.stdout
> +
>
> SEPARATOR = '=' * 78
>
> @@ -151,8 +179,16 @@ class OutputBase:
> except ValueError:
> truncate_subject = 0
>
> - if truncate_subject and len(subject) > truncate_subject:
> - subject = subject[:(truncate_subject - 3)] + "..."
> + # truncate subject as UTF-8 string.
> + # Note: there still exists an issue on combining characters.
> + if truncate_subject:
> + bsubject = to_bytes(subject)
> + if len(bsubject) > truncate_subject:
> + idx = truncate_subject - 2
> + while b'\x80' <= bsubject[idx-1:idx] <= b'\xbf':
> + idx -= 1
> + subject = to_str(bsubject[:idx-1]) + "..."
> +
> return subject
>
> def start(self, group, params):
> @@ -177,7 +213,7 @@ class OutputBase:
>
> def write(self, output):
> """Append the literal text string OUTPUT to the output representation."""
> - return self.write_binary(output.encode('utf-8'))
> + return self.write_binary(to_bytes(output))
>
> def run(self, cmd):
> """Override this method, if the default implementation is not sufficient.
> @@ -356,7 +392,7 @@ class StandardOutput(OutputBase):
>
> def __init__(self, cfg, repos, prefix_param):
> OutputBase.__init__(self, cfg, repos, prefix_param)
> - self.write_binary = sys.stdout.buffer.write
> + self.write_binary = _stdout.write
>
> def start(self, group, params):
> self.write("Group: " + (group or "defaults") + "\n")
> @@ -365,10 +401,11 @@ class StandardOutput(OutputBase):
> def finish(self):
> pass
>
> - def write(self, output):
> - """Write text as *default* encoding string"""
> - return self.write_binary(output.encode(locale.getpreferredencoding(),
> - 'backslashreplace'))
> + if (PY3 and (codecs.lookup(sys.stdout.encoding) != codecs.lookup('utf-8'))):
> + def write(self, output):
> + """Write text as *default* encoding string"""
> + return self.write_binary(output.encode(sys.stdout.encoding,
> + 'backslashreplace'))
>
>
> class PipeOutput(MailedOutput):
> @@ -431,8 +468,7 @@ class Commit(Messenger):
>
> self.changelist = sorted(editor.get_changes().items())
>
> - log = (repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG)
> - or b'').decode('utf-8')
> + log = to_str(repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG) or b'')
>
> # collect the set of groups and the unique sets of params for the options
> self.groups = { }
> @@ -451,7 +487,7 @@ class Commit(Messenger):
> # figure out the changed directories
> dirs = { }
> for path, change in self.changelist:
> - path = path.decode('utf-8')
> + path = to_str(path)
> if change.item_kind == svn.core.svn_node_dir:
> dirs[path] = None
> else:
> @@ -542,7 +578,7 @@ class PropChange(Messenger):
> elif self.action == 'M':
> self.output.write('Property diff:\n')
> tempfile1 = tempfile.NamedTemporaryFile()
> - tempfile1.write(sys.stdin.buffer.read())
> + tempfile1.write(_stdin.read())
> tempfile1.flush()
> tempfile2 = tempfile.NamedTemporaryFile()
> tempfile2.write(self.repos.get_rev_prop(self.propname))
> @@ -604,8 +640,7 @@ class Lock(Messenger):
> or 'unlock_subject_prefix'))
>
> # read all the locked paths from STDIN and strip off the trailing newlines
> - self.dirlist = [x.decode('utf-8').rstrip()
> - for x in sys.stdin.buffer.readlines()]
> + self.dirlist = [to_str(x).rstrip() for x in _stdin.readlines()]
>
> # collect the set of groups and the unique sets of params for the options
> self.groups = { }
> @@ -634,7 +669,7 @@ class Lock(Messenger):
> # The lock comment is the same for all paths, so we can just pull
> # the comment for the first path in the dirlist and cache it.
> self.lock = svn.fs.svn_fs_get_lock(self.repos.fs_ptr,
> - self.dirlist[0].encode('utf-8'),
> + to_bytes(self.dirlist[0]),
> self.pool)
>
> def generate(self):
> @@ -709,7 +744,7 @@ class DiffURLSelections:
> # KeyError exceptions.
> params = self.params.copy()
> params['path'] = _url_quote(change.path) if change.path else None
> - params['base_path'] = (_url_quote(change.base_path)
> + params['base_path'] = (_url_quote(change.base_path)
> if change.base_path else None)
> params['rev'] = repos_rev
> params['base_rev'] = change.base_rev
> @@ -764,8 +799,7 @@ def generate_content(renderer, cfg, repo
> author=repos.author,
> date=date,
> rev=repos.rev,
> - log=(repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG)
> - or b'').decode('utf-8'),
> + log=to_str(repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG) or b''),
> commit_url=commit_url,
> added_data=generate_list('A', changelist, paths, True),
> replaced_data=generate_list('R', changelist, paths, True),
> @@ -873,7 +907,9 @@ class DiffGenerator:
>
> # figure out if/how to generate a diff
>
> - base_path = remove_leading_slashes(change.base_path)
> + base_path_bytes = remove_leading_slashes(change.base_path)
> + base_path = (to_str(base_path_bytes)
> + if base_path_bytes is not None else None)
> if change.action == svn.repos.CHANGE_ACTION_DELETE:
> # it was delete.
> kind = 'D'
> @@ -884,10 +920,9 @@ class DiffGenerator:
> # show the diff?
> if self.diffsels.delete:
> diff = svn.fs.FileDiff(self.repos.get_root(change.base_rev),
> - base_path, None, None, self.pool)
> + base_path_bytes, None, None, self.pool)
>
> - label1 = '%s\t%s\t(r%s)' % (base_path.decode('utf-8'), self.date,
> - change.base_rev)
> + label1 = '%s\t%s\t(r%s)' % (base_path, self.date, change.base_rev)
> label2 = '/dev/null\t00:00:00 1970\t(deleted)'
> singular = True
>
> @@ -906,14 +941,13 @@ class DiffGenerator:
> # show the diff?
> if self.diffsels.modify:
> diff = svn.fs.FileDiff(self.repos.get_root(change.base_rev),
> - base_path,
> + base_path_bytes,
> self.repos.root_this, change.path,
> self.pool)
> - label1 = '%s\t%s\t(r%s, copy source)' \
> - % (base_path.decode('utf-8'), base_date, change.base_rev)
> - label2 = '%s\t%s\t(r%s)' \
> - % (change.path.decode('utf-8'), self.date, \
> - self.repos.rev)
> + label1 = ('%s\t%s\t(r%s, copy source)'
> + % (base_path, base_date, change.base_rev))
> + label2 = ('%s\t%s\t(r%s)'
> + % (to_str(change.path), self.date, self.repos.rev))
> singular = False
> else:
> # this file was copied.
> @@ -921,12 +955,12 @@ class DiffGenerator:
> if self.diffsels.copy:
> diff = svn.fs.FileDiff(None, None, self.repos.root_this,
> change.path, self.pool)
> - label1 = '/dev/null\t00:00:00 1970\t' \
> - '(empty, because file is newly added)'
> - label2 = '%s\t%s\t(r%s, copy of r%s, %s)' \
> - % (change.path.decode('utf-8'), self.date,
> - self.repos.rev, change.base_rev,
> - base_path.decode('utf-8'))
> + label1 = ('/dev/null\t00:00:00 1970\t'
> + '(empty, because file is newly added)')
> + label2 = ('%s\t%s\t(r%s, copy of r%s, %s)'
> + % (to_str(change.path),
> + self.date, self.repos.rev, change.base_rev,
> + base_path))
> singular = False
> else:
> # the file was added.
> @@ -942,7 +976,7 @@ class DiffGenerator:
> label1 = '/dev/null\t00:00:00 1970\t' \
> '(empty, because file is newly added)'
> label2 = '%s\t%s\t(r%s)' \
> - % (change.path.decode('utf-8'), self.date, self.repos.rev)
> + % (to_str(change.path), self.date, self.repos.rev)
> singular = True
>
> elif not change.text_changed:
> @@ -962,9 +996,9 @@ class DiffGenerator:
> self.repos.root_this, change.path,
> self.pool)
> label1 = '%s\t%s\t(r%s)' \
> - % (base_path.decode('utf-8'), base_date, change.base_rev)
> + % (base_path, base_date, change.base_rev)
> label2 = '%s\t%s\t(r%s)' \
> - % (change.path.decode('utf-8'), self.date, self.repos.rev)
> + % (to_str(change.path), self.date, self.repos.rev)
> singular = False
>
> if diff:
> @@ -1154,7 +1188,7 @@ class TextCommitRenderer:
> props = ' (props changed)'
> else:
> props = ''
> - w(' %s%s%s\n' % (d.path.decode('utf-8'), is_dir, props))
> + w(' %s%s%s\n' % (to_str(d.path), is_dir, props))
> if d.copied:
> if is_dir:
> text = ''
> @@ -1163,7 +1197,7 @@ class TextCommitRenderer:
> else:
> text = ' unchanged'
> w(' - copied%s from r%d, %s%s\n'
> - % (text, d.base_rev, d.base_path.decode('utf-8'), is_dir))
> + % (text, d.base_rev, to_str(d.base_path), is_dir))
>
> def _render_diffs(self, diffs, section_header):
> """Render diffs. Write the SECTION_HEADER if there are actually
> @@ -1180,20 +1214,20 @@ class TextCommitRenderer:
> w(section_header)
> section_header_printed = True
> if diff.kind == 'D':
> - w('\nDeleted: %s\n' % diff.base_path.decode('utf-8'))
> + w('\nDeleted: %s\n' % to_str(diff.base_path))
> elif diff.kind == 'A':
> - w('\nAdded: %s\n' % diff.path.decode('utf-8'))
> + w('\nAdded: %s\n' % to_str(diff.path))
> elif diff.kind == 'C':
> w('\nCopied: %s (from r%d, %s)\n'
> - % (diff.path.decode('utf-8'), diff.base_rev,
> - diff.base_path.decode('utf-8')))
> + % (to_str(diff.path), diff.base_rev,
> + to_str(diff.base_path)))
> elif diff.kind == 'W':
> w('\nCopied and modified: %s (from r%d, %s)\n'
> - % (diff.path.decode('utf-8'), diff.base_rev,
> - diff.base_path.decode('utf-8')))
> + % (to_str(diff.path), diff.base_rev,
> + to_str(diff.base_path)))
> else:
> # kind == 'M'
> - w('\nModified: %s\n' % diff.path.decode('utf-8'))
> + w('\nModified: %s\n' % to_str(diff.path))
>
> if diff.diff_url:
> w('URL: %s\n' % diff.diff_url)
> @@ -1232,7 +1266,7 @@ class Repository:
>
> self.author = self.get_rev_prop(svn.core.SVN_PROP_REVISION_AUTHOR)
> if self.author is not None:
> - self.author = self.author.decode('utf-8')
> + self.author = to_str(self.author)
>
> def get_rev_prop(self, propname, rev = None):
> if not rev:
> @@ -1441,9 +1475,9 @@ class Config:
> "Return the path's associated groups."
> groups = []
> for group, pattern, exclude_pattern, repos_params, search_logmsg_re in self._group_re:
> - match = pattern.match(path.decode('utf-8'))
> + match = pattern.match(to_str(path))
> if match:
> - if exclude_pattern and exclude_pattern.match(path.decode('utf-8')):
> + if exclude_pattern and exclude_pattern.match(to_str(path)):
> continue
> params = repos_params.copy()
> params.update(match.groupdict())
> @@ -1522,8 +1556,7 @@ if the property was added, modified or d
> usage()
>
> cmd = sys.argv[1]
> - repos_dir = svn.core.svn_path_canonicalize(sys.argv[2].encode('utf-8'))
> - repos_dir = repos_dir.decode('utf-8')
> + repos_dir = to_str(svn.core.svn_path_canonicalize(to_bytes(sys.argv[2])))
> try:
> expected_args = cmd_list[cmd]
> except KeyError:
>
>
>