You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by br...@apache.org on 2015/08/15 07:19:54 UTC
svn commit: r1696016 - in /subversion/trunk/tools/dist/security: mailer.py
parser.py
Author: brane
Date: Sat Aug 15 05:19:53 2015
New Revision: 1696016
URL: http://svn.apache.org/r1696016
Log:
Finish up message generation and signing for security notifications.
* tools/dist/security/parser.py: Import re.
(Notification.Metadata.__CULPRITS): Renamed from __culprits; all uses updated.
(Notification.Metadata.__init__): Sort patches using the new key member, see below.
(Notification.base_version_keys): New.
(__Part.__init__, __Part.__load_file): Support construction from a string.
(Patch.base_version_key): New property; returns a sortable key for the
base version of the patch.
(Patch.split_version, Patch.join_version): New.
* tools/dist/security/mailer.py: Import re, uuid, hashlib, smtplib, textwrap
and security.parser.
(Mailer.__init__): Accept additional parameters for constructing the
notification message from a message template.
(Mailer.__message_content): New.
(Mailer.__versions, Mailer.__culprits): New; helpers for __message_content.
(Mailer.generate_message): New.
(Mailer.send_mail): New.
(SignedMessage.__init__): Extend comment.
(SignedMessage.__signature): Make sure that the signed content conforms
to requiremens set down in RFC3156 section 5. Whith this change, mail clients
that understand PGP/MIME can actually verify the signed mail.
Modified:
subversion/trunk/tools/dist/security/mailer.py
subversion/trunk/tools/dist/security/parser.py
Modified: subversion/trunk/tools/dist/security/mailer.py
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/dist/security/mailer.py?rev=1696016&r1=1696015&r2=1696016&view=diff
==============================================================================
--- subversion/trunk/tools/dist/security/mailer.py (original)
+++ subversion/trunk/tools/dist/security/mailer.py Sat Aug 15 05:19:53 2015
@@ -23,6 +23,11 @@ Generator of signed advisory mails
from __future__ import absolute_import
+import re
+import uuid
+import hashlib
+import smtplib
+import textwrap
import email.utils
from email.mime.multipart import MIMEMultipart
@@ -33,15 +38,21 @@ try:
except ImportError:
import security._gnupg as gnupg
+import security.parser
+
class Mailer(object):
"""
Constructs signed PGP/MIME advisory mails.
"""
- def __init__(self, notification):
+ def __init__(self, notification, sender, message_template,
+ release_date, dist_revision, *release_versions):
assert len(notification) > 0
+ self.__sender = sender
self.__notification = notification
+ self.__message_content = self.__message_content(
+ message_template, release_date, dist_revision, release_versions)
def __subject(self):
"""
@@ -78,6 +89,79 @@ class Mailer(object):
return template.format(**kwargs)
+ def __message_content(self, message_template,
+ release_date, dist_revision, release_versions):
+ """
+ Construct the message from the notification mail template.
+ """
+
+ # Construct the replacement arguments for the notification template
+ culprits = set()
+ advisories = []
+ base_version_keys = self.__notification.base_version_keys()
+ for metadata in self.__notification:
+ culprits |= metadata.culprit
+ advisories.append(
+ ' * {}\n {}'.format(metadata.tracking_id, metadata.title))
+ release_version_keys = set(security.parser.Patch.split_version(n)
+ for n in release_versions)
+
+ multi = (len(self.__notification) > 1)
+ kwargs = dict(multiple=(multi and 'multiple ' or 'a '),
+ alert=(multi and 'alerts' or 'alert'),
+ culprits=self.__culprits(culprits),
+ advisories='\n'.join(advisories),
+ release_date=release_date.strftime('%d %B %Y'),
+ release_day=release_date.strftime('%d %B'),
+ base_versions = self.__versions(base_version_keys),
+ release_versions = self.__versions(release_version_keys),
+ dist_revision=str(dist_revision))
+
+ # Parse, interpolate and rewrap the notification template
+ wrapped = []
+ content = security.parser.Text(message_template)
+ for line in content.text.format(**kwargs).split('\n'):
+ if len(line) > 0 and not line[0].isspace():
+ for part in textwrap.wrap(line,
+ break_long_words=False,
+ break_on_hyphens=False):
+ wrapped.append(part)
+ else:
+ wrapped.append(line)
+ return security.parser.Text(None, '\n'.join(wrapped).encode('utf-8'))
+
+ def __versions(self, versions):
+ """
+ Return a textual representation of the set of VERSIONS
+ suitable for inclusion in a notification mail.
+ """
+
+ text = tuple(security.parser.Patch.join_version(n)
+ for n in sorted(versions))
+ assert len(text) > 0
+ if len(text) == 1:
+ return text[0]
+ elif len(text) == 2:
+ return ' and '.join(text)
+ else:
+ return ', '.join(text[:-1]) + ' and ' + text[-1]
+
+ def __culprits(self, culprits):
+ """
+ Return a textual representation of the set of CULPRITS
+ suitable for inclusion in a notification mail.
+ """
+
+ if self.__notification.Metadata.CULPRIT_CLIENT in culprits:
+ if self.__notification.Metadata.CULPRIT_SERVER in culprits:
+ return 'clients and servers'
+ else:
+ return 'clients'
+ elif self.__notification.Metadata.CULPRIT_SERVER in culprits:
+ return 'servers'
+ else:
+ raise ValueError('Unknown culprit ' + repr(culprits))
+
def __attachments(self):
filenames = set()
@@ -109,6 +193,60 @@ class Mailer(object):
+ ' Patch for Subversion ' + patch.base_version)
yield attachment(filename, description, 'base64', patch.base64)
+ def generate_message(self):
+ message = SignedMessage(
+ self.__message_content,
+ self.__attachments())
+ message['From'] = self.__sender
+ message['Reply-To'] = self.__sender
+ message['To'] = self.__sender # Will be replaced later
+ message['Subject'] = self.__subject()
+ message['Date'] = email.utils.formatdate()
+
+ # Try to make the message-id refer to the sender's domain
+ address = email.utils.parseaddr(self.__sender)[1]
+ if not address:
+ domain = None
+ else:
+ domain = address.split('@')[1]
+ if not domain:
+ domain = None
+
+ idstring = uuid.uuid1().hex
+ try:
+ msgid = email.utils.make_msgid(idstring, domain=domain)
+ except TypeError:
+ # The domain keyword was added in Python 3.2
+ msgid = email.utils.make_msgid(idstring)
+ message["Message-ID"] = msgid
+ return message
+
+ def send_mail(self, message, username, password, recipients=None,
+ host='mail-relay.apache.org', starttls=True, port=None):
+ if not port and starttls:
+ port = 587
+ server = smtplib.SMTP(host, port)
+ if starttls:
+ server.starttls()
+ if username and password:
+ server.login(username, password)
+
+ def send(message):
+ server.sendmail("From: " + message['From'],
+ "To: " + message['To'],
+ message.as_string())
+
+ if recipients is None:
+ # Test mode, send message back to originator to checck
+ # that contents and signature are OK.
+ message.replace_header('To', message['From'])
+ send(message)
+ else:
+ for recipient in recipients:
+ message.replace_header('To', recipient)
+ send(message)
+ server.quit()
+
class SignedMessage(MIMEMultipart):
"""
@@ -132,7 +270,7 @@ class SignedMessage(MIMEMultipart):
payload, gpgbinary, gnupghome, use_agent, keyring, keyid)
self.set_param('protocol', 'application/pgp-signature')
- self.set_param('micalg', 'pgp-sha512') ####!!!
+ self.set_param('micalg', 'pgp-sha512') ####!!! GET THIS FROM KEY!
self.preamble = 'This is an OpenPGP/MIME signed message.'
self.attach(payload)
self.attach(signature)
@@ -162,9 +300,13 @@ class SignedMessage(MIMEMultipart):
a MIME attachment.
"""
+ # RFC3156 section 5 says line endings in the signed message
+ # must be canonical <CR><LF>.
+ cleartext = re.sub(r'\r?\n', '\r\n', payload.as_string())
+
gpg = gnupg.GPG(gpgbinary=gpgbinary, gnupghome=gnupghome,
use_agent=use_agent, keyring=keyring)
- signature = gpg.sign(payload.as_string(),
+ signature = gpg.sign(cleartext,
keyid=keyid, detach=True, clearsign=False)
sig = MIMEText('')
sig.set_type('application/pgp-signature')
Modified: subversion/trunk/tools/dist/security/parser.py
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/dist/security/parser.py?rev=1696016&r1=1696015&r2=1696016&view=diff
==============================================================================
--- subversion/trunk/tools/dist/security/parser.py (original)
+++ subversion/trunk/tools/dist/security/parser.py Sat Aug 15 05:19:53 2015
@@ -25,6 +25,7 @@ from __future__ import absolute_import
import os
+import re
import ast
import base64
import quopri
@@ -49,15 +50,15 @@ class Notification(object):
CULPRIT_SERVER = 'server'
CULPRIT_CLIENT = 'client'
- __culprits = ((CULPRIT_SERVER, CULPRIT_CLIENT,
+ __CULPRITS = ((CULPRIT_SERVER, CULPRIT_CLIENT,
(CULPRIT_SERVER, CULPRIT_CLIENT),
(CULPRIT_CLIENT, CULPRIT_SERVER)))
def __init__(self, basedir, tracking_id,
title, culprit, advisory, patches):
- if culprit not in self.__culprits:
+ if culprit not in self.__CULPRITS:
raise ValueError('Culprit should be one of: '
- + ', '.join(repr(x) for x in self.__culprits))
+ + ', '.join(repr(x) for x in self.__CULPRITS))
if not isinstance(culprit, tuple):
culprit = (culprit,)
@@ -69,9 +70,7 @@ class Notification(object):
for base_version, patchfile in patches.items():
patch = Patch(base_version, os.path.join(basedir, patchfile))
self.__patches.append(patch)
- self.__patches.sort(reverse=True,
- key=lambda x: tuple(
- int(q) for q in x.base_version.split('.')))
+ self.__patches.sort(reverse=True, key=lambda x: x.base_version_key)
@property
def tracking_id(self):
@@ -99,8 +98,13 @@ class Notification(object):
Create the security notification for all TRACKING_IDS.
The advisories and patches for each tracking ID must be
in the appropreiately named subdirectory of ROOTDIR.
+
+ The notification text assumes that RELEASE_VERSIONS will
+ be published on RELEASE_DATE and that the tarballs are
+ available in DIST_REVISION of the dist repository.
"""
+ assert(len(tracking_ids) > 0)
self.__advisories = []
for tid in tracking_ids:
self.__advisories.append(self.__parse_advisory(rootdir, tid))
@@ -112,6 +116,10 @@ class Notification(object):
return len(self.__advisories)
def __parse_advisory(self, rootdir, tracking_id):
+ """
+ Parse a single advisory named TRACKING_ID in ROOTDIR.
+ """
+
basedir = os.path.join(rootdir, tracking_id)
with open(os.path.join(basedir, 'metadata'), 'rt') as md:
metadata = ast.literal_eval(md.read())
@@ -122,31 +130,48 @@ class Notification(object):
metadata['advisory'],
metadata['patches'])
+ def base_version_keys(self):
+ """
+ Return the set of base-version keys of all the patches.
+ """
+
+ base_version_keys = set()
+ for metadata in self:
+ for patch in metadata.patches:
+ base_version_keys.add(patch.base_version_key)
+ return base_version_keys
-class __Part(object):
- def __init__(self, path):
- self.__text = self.__load_file(path)
- def __load_file(self, path):
+class __Part(object):
+ def __init__(self, path, text=None):
"""
- Load a file at PATH into memory as an array of lines.
- if self.TEXTMODE is True, strip whitespace from the end of
+ Create a text object with contents from the file at PATH.
+ If self.TEXTMODE is True, strip whitespace from the end of
all lines and strip empty lines from the end of the file.
+
+ Alternatively, if PATH is None, set the contents to TEXT,
+ which must be convertible to bytes.
"""
- text = []
+ assert (path is None) is not (text is None)
+ if path:
+ self.__text = self.__load_file(path)
+ else:
+ self.__text = bytes(text)
+
+ def __load_file(self, path):
with open(path, 'rb') as src:
+ if not self.TEXTMODE:
+ return src.read()
+
+ text = []
for line in src:
- if self.TEXTMODE:
- line = line.rstrip() + b'\n'
- text.append(line)
+ text.append(line.rstrip() + b'\n')
- # Strip trailing empty lines in text mode
- if self.TEXTMODE:
+ # Strip trailing empty lines in text mode
while len(text) and not text[-1]:
del text[-1]
-
- return b''.join(text)
+ return b''.join(text)
@property
def text(self):
@@ -204,11 +229,52 @@ class Patch(__Part):
def __init__(self, base_version, path):
super(Patch, self).__init__(path)
self.__base_version = base_version
+ self.__base_version_key = self.split_version(base_version)
@property
def base_version(self):
return self.__base_version
@property
+ def base_version_key(self):
+ return self.__base_version_key
+
+ @property
def quoted_printable(self):
raise NotImplementedError('Quoted-printable patches? Really?')
+
+
+ __SPLIT_VERSION_RX = re.compile(r'^(\d+)(?:\.(\d+))?(?:\.(\d+))?(.+)?$')
+
+ @classmethod
+ def split_version(cls, version):
+ """
+ Splits a version number in the form n.n.n-tag into a tuple
+ of its components.
+ """
+ def splitv(version):
+ for s in cls.__SPLIT_VERSION_RX.match(version).groups():
+ if s is None:
+ continue
+ try:
+ n = int(s)
+ except ValueError:
+ n = s
+ yield n
+ return tuple(splitv(version))
+
+ @classmethod
+ def join_version(cls, version_tuple):
+ """
+ Joins a version number tuple returned by Patch.split_version
+ into a string.
+ """
+
+ def joinv(version_tuple):
+ prev = None
+ for n in version_tuple:
+ if isinstance(n, int) and prev is not None:
+ yield '.'
+ prev = n
+ yield str(n)
+ return ''.join(joinv(version_tuple))