You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by hu...@apache.org on 2016/06/01 12:41:03 UTC
[01/11] incubator-ponymail git commit: add a synch ponymail with imap
tool
Repository: incubator-ponymail
Updated Branches:
refs/heads/master 3acd303a4 -> 43d70910c
add a synch ponymail with imap tool
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/cefa48d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/cefa48d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/cefa48d0
Branch: refs/heads/master
Commit: cefa48d0baef7a9cd37c53f9dcbc3e52670c329a
Parents: a383086
Author: Sam Ruby <ru...@intertwingly.net>
Authored: Tue May 3 14:21:49 2016 -0400
Committer: Sam Ruby <ru...@intertwingly.net>
Committed: Tue May 3 14:21:49 2016 -0400
----------------------------------------------------------------------
tools/archiver.py | 25 +++++--
tools/sync-imap.py | 176 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 195 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/cefa48d0/tools/archiver.py
----------------------------------------------------------------------
diff --git a/tools/archiver.py b/tools/archiver.py
index ec1fef7..8cdbcbf 100644
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -75,6 +75,7 @@ def parse_attachment(part):
dispositions = cd.strip().split(";")
if dispositions[0].lower() == "attachment":
fd = part.get_payload(decode=True)
+ if not fd: return None, None
attachment = {}
attachment['content_type'] = part.get_content_type()
attachment['size'] = len(fd)
@@ -83,7 +84,8 @@ def parse_attachment(part):
b64 = codecs.encode(fd, "base64").decode('ascii', 'ignore')
attachment['hash'] = h
for param in dispositions[1:]:
- key,val = param.split("=")
+ if not '=' in param: continue
+ key,val = param.split("=", 1)
if key.lower().strip() == "filename":
val = val.strip(' "')
print("Found attachment: %s" % val)
@@ -282,8 +284,9 @@ class Archiver(object):
body = body.encode('utf-8')
except:
body = None
- if body:
- attachments, contents = self.msgfiles(msg)
+
+ attachments, contents = self.msgfiles(msg)
+ if body or attachments:
private = False
if hasattr(mlist, 'archive_public') and mlist.archive_public == True:
private = False
@@ -293,7 +296,8 @@ class Archiver(object):
private = True
pmid = mid
try:
- mid = "%s@%s@%s" % (hashlib.sha224(body if type(body) is bytes else body.encode('ascii', 'ignore')).hexdigest(), uid_mdate, lid)
+ mid = "%s@%s" % (hashlib.sha224(msg.as_bytes()).hexdigest(), lid)
+ print(mid)
except Exception as err:
if logger:
logger.warn("Could not generate MID: %s" % err)
@@ -475,6 +479,8 @@ if __name__ == '__main__':
help='Use the archive timestamp as the email date instead of the Date header')
parser.add_argument('--quiet', dest='quiet', action='store_true',
help='Do not exit -1 if the email could not be parsed')
+ parser.add_argument('--verbose', dest='verbose', action='store_true',
+ help='Output additional log messages')
parser.add_argument('--html2text', dest='html2text', action='store_true',
help='Try to convert HTML to text if no text/plain message is found')
args = parser.parse_args()
@@ -482,6 +488,10 @@ if __name__ == '__main__':
if args.html2text:
import html2text
parseHTML = True
+
+ if args.verbose:
+ import logging
+ logging.basicConfig(stream=sys.stdout, level=logging.INFO)
foo = Archiver()
input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8', errors="ignore")
@@ -512,7 +522,7 @@ if __name__ == '__main__':
msg.replace_header('List-ID', msg.get(altheader))
except:
msg.add_header('list-id', msg.get(altheader))
-
+
# Set specific LID?
if args.lid and len(args.lid[0]) > 3:
try:
@@ -562,6 +572,9 @@ if __name__ == '__main__':
lid = foo.archive_message(msg_metadata, msg)
print("%s: Done archiving to %s!" % (email.utils.formatdate(), lid))
except Exception as err:
+ if args.verbose:
+ import traceback
+ traceback.print_exc()
print("Archiving failed!: %s" % err)
raise Exception("Archiving to ES failed")
else:
@@ -572,4 +585,4 @@ if __name__ == '__main__':
else:
print("Could not parse email: %s" % err)
sys.exit(-1)
-
\ No newline at end of file
+
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/cefa48d0/tools/sync-imap.py
----------------------------------------------------------------------
diff --git a/tools/sync-imap.py b/tools/sync-imap.py
new file mode 100755
index 0000000..43d457d
--- /dev/null
+++ b/tools/sync-imap.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python3.4
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+ #the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Syncronize ponymail with an imap server.
+
+Fetches message-ids from both ponymail and an imap server, and adds or
+deletes whatever is necessary from ponymail to make it match.
+
+See usage for instructions.
+
+"""
+
+import argparse
+import configparser
+import elasticsearch
+import imaplib
+import os
+import pwd
+import subprocess
+import sys
+import re
+
+# change working directory to location of this script
+
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+# global defaults
+
+es_list = None
+imap_host = 'localhost'
+imap_port = 993
+imap_user = pwd.getpwuid(os.getuid()).pw_name
+imap_password = None
+imap_folder = 'INBOX'
+html2text = False
+verbose = False
+
+# fetch config overrides
+
+config = configparser.RawConfigParser()
+config.read('ponymail.cfg')
+iname = config.get("elasticsearch", "dbname")
+if config.has_option('imap', 'host'):
+ imap_host = config.get('imap', 'host')
+if config.has_option('imap', 'port'):
+ imap_port = config.getint('imap', 'port')
+if config.has_option('imap', 'user'):
+ imap_user = config.getint('imap', 'user')
+if config.has_option('imap', 'password'):
+ imap_password = config.getint('imap', 'password')
+
+# fetch command line argument overrides
+
+parser = argparse.ArgumentParser(description='Command line options.')
+parser.add_argument('--list', dest='list', type=str, nargs=1,
+ help='ElasticSearch list')
+parser.add_argument('--host', dest='host', type=str, nargs=1,
+ help='IMAP host')
+parser.add_argument('--port', dest='port', type=int, nargs=1,
+ help='IMAP port')
+parser.add_argument('--user', dest='user', type=str, nargs=1,
+ help='IMAP user')
+parser.add_argument('--password', dest='password', type=str, nargs=1,
+ help='IMAP password')
+parser.add_argument('--folder', dest='folder', type=str, nargs=1,
+ help='IMAP folder')
+parser.add_argument('--html2text', dest='html2text', action='store_true',
+ help='Try to convert HTML to text if no text/plain message is found')
+parser.add_argument('--verbose', dest='verbose', action='store_true',
+ help='Output additional log messages')
+
+args = parser.parse_args()
+
+if args.list:
+ es_list = args.list[0]
+if args.host:
+ imap_host = args.host[0]
+if args.port:
+ imap_port = args.port[0]
+if args.user:
+ imap_user = args.user[0]
+if args.password:
+ imap_password = args.password[0]
+if args.folder:
+ imap_folder = args.folder[0]
+if args.html2text:
+ html2text = True
+if args.verbose:
+ verbose = True
+
+if not es_list or not imap_password:
+ parser.print_help()
+ sys.exit(-1)
+
+es_list = "<%s>" % es_list.strip("<>") # We need <> around it!
+
+# fetch message-id => _id pairs from elasticsearch
+
+es = elasticsearch.Elasticsearch()
+result = es.search(scroll = '5m',
+ body = {
+ 'size': 1024,
+ 'fields': ['message-id'],
+ 'query': {'match': {'list': es_list}}
+ }
+)
+
+db = {}
+while len(result['hits']['hits']) > 0:
+ for hit in result['hits']['hits']:
+ db[hit['fields']['message-id'][0]] = hit['_id']
+ result = es.scroll(scroll='5m', scroll_id=result['_scroll_id'])
+
+# fetch message-id => uid pairs from imap
+
+imap = imaplib.IMAP4_SSL(imap_host, imap_port)
+imap.login(imap_user, imap_password)
+imap.select(imap_folder, readonly=True)
+results = imap.uid('search', None, 'ALL')
+uids = b','.join(results[1][0].split())
+results = imap.uid('fetch', uids, '(BODY[HEADER.FIELDS (MESSAGE-ID)])')
+
+mail = {}
+uid_re = re.compile(b'^\d+ \(UID (\d+) BODY\[')
+mid_re = re.compile(b'^Message-ID:\s*(.*?)\s*$', re.I)
+uid = None
+for result in results[1]:
+ for line in result:
+ if isinstance(line, bytes):
+ match = uid_re.match(line)
+ if match:
+ uid = match.group(1)
+ else:
+ match = mid_re.match(line)
+ if match:
+ try:
+ mail[match.group(1).decode('utf-8')] = uid
+ uid = None
+ except ValueError:
+ pass
+
+# delete items from elasticsearch that are not present in imap
+
+for mid, _id in db.items():
+ if not mid in mail:
+ es.delete(index=iname, id=_id, doc_type='mbox')
+ es.delete(index=iname, id=_id, doc_type='mbox_source')
+ print("deleted: " + mid)
+
+# add new items to elasticsearch from imap
+
+for mid, uid in mail.items():
+ if not mid in db:
+ argv = [sys.executable, 'archiver.py', '--lid=%s' % es_list]
+ if verbose: argv.append('--verbose')
+ if html2text: argv.append('--html2text')
+ child = subprocess.Popen(argv, stdin=subprocess.PIPE)
+ child.stdin.write(imap.uid('fetch', uid, '(RFC822)')[1][0][1])
+ child.stdin.close()
+ rc = child.wait()
+ print("inserted: %s, rc = %d" % (mid, rc))
+
[03/11] incubator-ponymail git commit: bulk deletes
Posted by hu...@apache.org.
bulk deletes
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/6bcb4912
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/6bcb4912
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/6bcb4912
Branch: refs/heads/master
Commit: 6bcb491206bd0b22218505fc96a80057b6166294
Parents: 9571469
Author: Sam Ruby <ru...@intertwingly.net>
Authored: Wed May 4 07:32:14 2016 -0400
Committer: Sam Ruby <ru...@intertwingly.net>
Committed: Wed May 4 07:32:14 2016 -0400
----------------------------------------------------------------------
tools/sync-imap.py | 33 +++++++++++++++++++++++++++------
1 file changed, 27 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/6bcb4912/tools/sync-imap.py
----------------------------------------------------------------------
diff --git a/tools/sync-imap.py b/tools/sync-imap.py
index 43d457d..14d2711 100755
--- a/tools/sync-imap.py
+++ b/tools/sync-imap.py
@@ -27,6 +27,7 @@ See usage for instructions.
import argparse
import configparser
import elasticsearch
+from elasticsearch import helpers as eshelper
import imaplib
import os
import pwd
@@ -155,16 +156,37 @@ for result in results[1]:
# delete items from elasticsearch that are not present in imap
+queue1 = []
+queue2 = []
for mid, _id in db.items():
- if not mid in mail:
- es.delete(index=iname, id=_id, doc_type='mbox')
- es.delete(index=iname, id=_id, doc_type='mbox_source')
- print("deleted: " + mid)
+ if True: # not mid in mail:
+ queue1.append({
+ '_op_type': 'delete',
+ '_index': iname,
+ '_type': 'mbox',
+ '_id': _id
+ })
+ queue2.append({
+ '_op_type': 'delete',
+ '_index': iname,
+ '_type': 'mbox_source',
+ '_id': _id
+ })
+ print("deleting: " + mid)
+
+while len(queue1) > 0:
+ eshelper.bulk(es, queue1[0:1024])
+ del queue1[0:1024]
+
+while len(queue2) > 0:
+ eshelper.bulk(es, queue2[0:1024])
+ del queue2[0:1024]
# add new items to elasticsearch from imap
for mid, uid in mail.items():
if not mid in db:
+ print("indexing %s" % mid)
argv = [sys.executable, 'archiver.py', '--lid=%s' % es_list]
if verbose: argv.append('--verbose')
if html2text: argv.append('--html2text')
@@ -172,5 +194,4 @@ for mid, uid in mail.items():
child.stdin.write(imap.uid('fetch', uid, '(RFC822)')[1][0][1])
child.stdin.close()
rc = child.wait()
- print("inserted: %s, rc = %d" % (mid, rc))
-
+ if rc != 0: print("rc %d" % rc)
[10/11] incubator-ponymail git commit: Merge branch 'rubys-sync-imap'
Posted by hu...@apache.org.
Merge branch 'rubys-sync-imap'
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/f5074232
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/f5074232
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/f5074232
Branch: refs/heads/master
Commit: f50742322a3ba6198338dfbdc4dce94d0e6abc2c
Parents: 3acd303 d1d6c7f
Author: humbedooh <hu...@apache.org>
Authored: Wed Jun 1 14:25:33 2016 +0200
Committer: humbedooh <hu...@apache.org>
Committed: Wed Jun 1 14:25:33 2016 +0200
----------------------------------------------------------------------
tools/archiver.py | 336 +++++++++++++++++++---------------
tools/import-mbox.py | 447 +++++++++++++++++-----------------------------
2 files changed, 351 insertions(+), 432 deletions(-)
----------------------------------------------------------------------
[06/11] incubator-ponymail git commit: eliminate some redundant code
Posted by hu...@apache.org.
eliminate some redundant code
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/1ed519e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/1ed519e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/1ed519e5
Branch: refs/heads/master
Commit: 1ed519e5b152e3c92d07722fbc2329ba13e48697
Parents: 6b42d4d
Author: Sam Ruby <ru...@intertwingly.net>
Authored: Wed May 4 20:58:28 2016 -0400
Committer: Sam Ruby <ru...@intertwingly.net>
Committed: Wed May 4 20:58:28 2016 -0400
----------------------------------------------------------------------
tools/archiver.py | 14 ++-
tools/import-mbox.py | 311 +++++++---------------------------------------
2 files changed, 51 insertions(+), 274 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/1ed519e5/tools/archiver.py
----------------------------------------------------------------------
diff --git a/tools/archiver.py b/tools/archiver.py
index 31f790a..991b6d6 100644
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -129,12 +129,15 @@ class Archiver(object):
"x-mailman-rule-misses",
]
- def __init__(self):
+ def __init__(self, parseHTML=False):
""" Just initialize ES. """
- global config, auth, parseHTML
+ global config, auth
ssl = False
self.cropout = None
self.html = parseHTML
+ if parseHTML:
+ import html2text
+ self.html2text = html2text.html2text
self.dbname = config.get("elasticsearch", "dbname")
self.consistency = 'quorum'
if config.has_option("elasticsearch", "ssl") and config.get("elasticsearch", "ssl").lower() == 'true':
@@ -209,7 +212,7 @@ class Archiver(object):
# this requires a GPL lib, user will have to install it themselves
if firstHTML and (not body or len(body) <= 1):
- body = html2text.html2text(firstHTML.decode("utf-8", 'ignore') if type(firstHTML) is bytes else firstHTML)
+ body = self.html2text(firstHTML.decode("utf-8", 'ignore') if type(firstHTML) is bytes else firstHTML)
for charset in pm_charsets(msg):
try:
@@ -287,6 +290,7 @@ class Archiver(object):
body = None
attachments, contents = self.msgfiles(msg)
+ irt = ""
if body or attachments:
pmid = mid
try:
@@ -295,7 +299,6 @@ class Archiver(object):
if logger:
logger.warn("Could not generate MID: %s" % err)
mid = pmid
- irt = ""
if 'in-reply-to' in msg_metadata:
try:
try:
@@ -511,14 +514,13 @@ if __name__ == '__main__':
args = parser.parse_args()
if args.html2text:
- import html2text
parseHTML = True
if args.verbose:
import logging
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
- foo = Archiver()
+ foo = Archiver(parseHTML = parseHTML)
input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8', errors="ignore")
try:
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/1ed519e5/tools/import-mbox.py
----------------------------------------------------------------------
diff --git a/tools/import-mbox.py b/tools/import-mbox.py
index 3d24036..399afe6 100644
--- a/tools/import-mbox.py
+++ b/tools/import-mbox.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python3.4
+
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -35,6 +36,7 @@ import codecs
import multiprocessing
import tempfile
import gzip
+import archiver
try:
from elasticsearch import Elasticsearch, helpers
@@ -49,7 +51,6 @@ block = Lock()
lists = []
start = time.time()
quickmode = False
-lid_override = None
private = False
appender = "apache.org"
@@ -105,100 +106,6 @@ es = Elasticsearch([
rootURL = ""
-def parse_attachment(part):
- cd = part.get("Content-Disposition", None)
- if cd:
- dispositions = cd.strip().split(";")
- if dispositions[0].lower() == "attachment":
- try:
- fd = part.get_payload(decode=True)
- if fd:
- attachment = {}
- attachment['content_type'] = part.get_content_type()
- attachment['size'] = len(fd)
- attachment['filename'] = None
- h = hashlib.sha256(fd).hexdigest()
- b64 = codecs.encode(fd, "base64").decode('ascii')
- attachment['hash'] = h
- for param in dispositions[1:]:
- key,val = param.split("=", 1)
- if key.lower().strip() == "filename":
- val = val.strip(' "')
- print("Found attachment: %s" % val)
- attachment['filename'] = val
- if attachment['filename']:
- return attachment, b64 # Return meta data and contents separately
- except:
- pass
- return None, None
-
-def msgfiles(msg):
- attachments = []
- contents = {}
- if msg.is_multipart():
- for part in msg.walk():
- part_meta, part_file = parse_attachment(part)
- if part_meta:
- attachments.append(part_meta)
- contents[part_meta['hash']] = part_file
- return attachments, contents
-
-def pm_charsets(msg):
- charsets = set({})
- for c in msg.get_charsets():
- if c is not None:
- charsets.update([c])
- return charsets
-
-def msgbody(msg):
- global parseHTML, iBody
- firstHTML = None
- body = None
- if msg.is_multipart():
- for part in msg.walk():
- try:
- if part.is_multipart():
- for subpart in part.walk():
- if subpart.get_content_type() == 'text/plain' and not body:
- body = subpart.get_payload(decode=True)
- elif subpart.get_content_type() == 'text/html' and parseHTML and not firstHTML:
- firstHTML = subpart.get_payload(decode=True)
-
- elif part.get_content_type() == 'text/plain' and not body:
- body = part.get_payload(decode=True)
- elif part.get_content_type() == 'text/html' and parseHTML and not firstHTML:
- firstHTML = part.get_payload(decode=True)
- except Exception as err:
- print("Body parser error: %s" % err)
- elif msg.get_content_type() == 'text/plain':
- body = msg.get_payload(decode=True)
- elif msg.get_content_type() == 'text/html' and parseHTML and not firstHTML:
- firstHTML = msg.get_payload(decode=True)
-
- # this requires a GPL lib, user will have to install it themselves
- if firstHTML and (not body or len(body) <= 1 or (iBody and str(body).find(str(iBody)) != -1)):
- print("No body found as text/plain, converting HTML to text")
- body = html2text.html2text(firstHTML.decode("utf-8", errors='replace') if type(firstHTML) is bytes else firstHTML)
-
- for charset in pm_charsets(msg):
- try:
- body = body.decode(charset) if type(body) is bytes else body
- except:
- body = body.decode('utf-8', errors='replace') if type(body) is bytes else body
-
- return body
-
-
-def msgfactory(fp):
- try:
- return email.message_from_file(fp)
- except Exception as err:
- # Don't return None since that will
- # stop the mailbox iterator
- print("hmm: %s" % err)
- return None
-
-
class BulkThread(Thread):
def assign(self, json, xes, dtype = 'mbox', wc = 'quorum'):
self.json = json
@@ -247,6 +154,8 @@ class SlurpThread(Thread):
mboxfile = ""
filename = ""
xlist_override = None
+
+ foo = archiver.Archiver(parseHTML = parseHTML)
while len(lists) > 0:
print("%u elements left to slurp" % len(lists))
@@ -334,180 +243,46 @@ class SlurpThread(Thread):
if (time.time() - stime > timeout): # break out after N seconds, it shouldn't take this long..!
print("Whoa, this is taking way too long, ignoring %s for now" % tmpname)
break
- if 'subject' in message:
- subject = message['subject'] # Could possibly be None.
- mid = message['message-id']
-
- lid = message['list-id']
- if args.requirelid and (not lid or lid == ""):
- continue
- if not lid or lid == "": # Guess list name in absence
- lid = '.'.join(reversed(ml.split("-"))) + "." + appender
-
- # Compact LID to <fo...@domain>, discard rest
- m = re.search(r"(<.+>)", lid)
- if m:
- lid = m.group(1)
- if xlist_override and len(xlist_override) > 3:
- lid = xlist_override
- lid = lid.replace("@",".") # we want foo.bar.org, not foo@bar.org
- lid = "<%s>" % lid.strip("<>") # We need <> around it!
- if cropout:
- crops = cropout.split(" ")
- # Regex replace?
- if len(crops) == 2:
- lid = re.sub(crops[0], crops[1], lid)
- # Standard crop out?
- else:
- lid = lid.replace(cropout, "")
-
- date = message['date']
- fro = message['from']
- to = message['to']
- body = msgbody(message)
- try:
- if 'content-type' in message and message['content-type'].find("flowed") != -1:
- body = convertToWrapped(body, character_set="utf-8")
- if isinstance(body, str):
- body = body.encode('utf-8')
- except Exception as err:
- try:
- body = body.decode(chardet.detect(body)['encoding'])
- except Exception as err:
- try:
- body = body.decode('latin-1')
- except:
- try:
- if isinstance(body, str):
- body = body.encode('utf-8')
- except:
- body = None
-
- okay = True
- dheader = {}
- for key in ['to','from','subject','message-id']:
- try:
- hval = ""
- if message.get(key):
- for t in email.header.decode_header(message[key]):
- if t[1] == None or t[1].find("8bit") != -1:
- hval += t[0].decode('utf-8', errors='replace') if type(t[0]) is bytes else t[0]
- else:
- hval += t[0].decode(t[1],errors='ignore')
- dheader[key] = hval
- else:
- dheader[key] = "(Unknown)"
- except Exception as err:
- print("Could not decode headers, ignoring..: %s" % err)
- okay = False
- mdt = ""
- if not 'date' in message and 'received' in message:
- print("No Date header found, resorting to Received")
- m = re.search(r"(\d+ \S+ \d{4} \d\d:\d\d:\d\d ([-+]\d{4})?)", message['received'])
- if m:
- mdt = m.group(1)
- else:
- mdt = message['date']
- mdate = None
- uid_mdate = 0
- try:
- mdate = email.utils.parsedate_tz(mdt)
- uid_mdate = email.utils.mktime_tz(mdate)
- except:
- pass
- if not mdate or mdate[0] < (LEY-1):
- print("Date is wrong or missing here, setting to %s" % ( LEY))
- mdate = datetime.datetime(LEY, EM, 1).timetuple()
- else:
- LEY = mdate[0] # Gather evidence 'n'stuff!
- mdatestring = ""
- try:
- mdatestring = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(email.utils.mktime_tz(mdate)))
- except:
- okay = False
- if body and okay and mdate and {'from','subject'} <= set(dheader):
- # Pipermail transforms from: to something weird - reset that!
- if piperWeirdness:
- m = re.match(r"(.+) at ([^(]+) \((.+)\)$", dheader['from'])
- # Try just 'foo at bar.tld' if 'foo at bar.tld (foo bar)' isn't working
- if not m:
- m = re.match(r"(.+) at ([^(]+)$", dheader['from'])
- if m:
- dheader['from'] = "%s <%s@%s>" % (m.group(3), m.group(1), m.group(2))
-
- attachments, contents = msgfiles(message)
- if mid == None or not mid:
- try:
- mid = hashlib.sha256(body if type(body) is bytes else body.encode('ascii', errors='ignore')).hexdigest() + "@" + lid + "@" + appender
- except:
- if filebased:
- mid = hashlib.sha256("%f-%f-%s" % (random.random(), time.time(), filename) ).hexdigest()+ "@" + appender
- else:
- mid = hashlib.sha256("%f-%f-%s-%s" % (random.random(), time.time(), ml, mboxfile) ).hexdigest()+ "@" + appender
- print("No MID found, setting to %s" % mid)
- mid2 = "%s@%s@%s" % (hashlib.sha224(body if type(body) is bytes else body.encode('ascii', 'ignore')).hexdigest(), uid_mdate, lid)
- count += 1
- mr = ""
- if 'references' in message:
- mr = message['references']
- irt = ""
- if 'in-reply-to' in message:
- try:
- irt = "".join(message['in-reply-to'])
- except:
- irt = message.get('in-reply-to').__str__()
-
- json = {
- 'from_raw': dheader['from'],
- 'from': dheader['from'],
- 'to': dheader['to'],
- 'subject': dheader['subject'],
- 'cc': message.get('cc'),
- 'message-id': mid,
- 'mid': mid2,
- 'epoch': email.utils.mktime_tz(mdate),
- 'list': lid,
- 'list_raw': lid,
- 'date': mdatestring,
- 'private': private,
- 'references': mr,
- 'in-reply-to': irt,
- 'body': body.decode('utf-8', errors='replace') if type(body) is bytes else body,
- 'attachments': attachments
- }
- json_source = {
- 'mid': mid2,
- 'message-id': mid,
- 'source': message.as_bytes().decode('utf-8', errors='replace')
- }
- ja.append(json)
- jas.append(json_source)
- if contents:
- iname = config.get("elasticsearch", "dbname")
- if not args.dry:
- for key in contents:
- es.index(
- index=iname,
- doc_type="attachment",
- id=key,
- body = {
- 'source': contents[key]
- }
- )
- if len(ja) >= 40:
- if not args.dry:
- bulk = BulkThread()
- bulk.assign(ja, es, 'mbox')
- bulk.insert()
- ja = []
-
- if not args.dry:
- bulks = BulkThread()
- bulks.assign(jas, es, 'mbox_source')
- bulks.insert()
- jas = []
+
+ json, contents = foo.compute_updates(list_override, private, message)
+
+ if json:
+ json_source = {
+ 'mid': json['mid'],
+ 'message-id': json['message-id'],
+ 'source': message.as_bytes().decode('utf-8', errors='replace')
+ }
+
+ count += 1
+ ja.append(json)
+ jas.append(json_source)
+ if contents:
+ iname = config.get("elasticsearch", "dbname")
+ if not args.dry:
+ for key in contents:
+ es.index(
+ index=iname,
+ doc_type="attachment",
+ id=key,
+ body = {
+ 'source': contents[key]
+ }
+ )
+ if len(ja) >= 40:
+ if not args.dry:
+ bulk = BulkThread()
+ bulk.assign(ja, es, 'mbox')
+ bulk.insert()
+ ja = []
+
+ if not args.dry:
+ bulks = BulkThread()
+ bulks.assign(jas, es, 'mbox_source')
+ bulks.insert()
+ jas = []
else:
baddies += 1
+
if filebased:
print("Parsed %u records from %s" % (count, filename))
if dFile:
@@ -585,7 +360,7 @@ if args.source:
if args.dir:
maildir = args.dir
if args.listid:
- list_override = args.listid[0]
+ list_override = '<%s>' % args.listid[0].strip('<>')
if args.project:
project = args.project[0]
if args.domain:
[08/11] incubator-ponymail git commit: merge sync-imap into
import-mbox
Posted by hu...@apache.org.
merge sync-imap into import-mbox
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/6127911a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/6127911a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/6127911a
Branch: refs/heads/master
Commit: 6127911a367a527423495c98f551d3939359eae9
Parents: 7135cd4
Author: Sam Ruby <ru...@intertwingly.net>
Authored: Fri May 6 12:46:31 2016 -0400
Committer: Sam Ruby <ru...@intertwingly.net>
Committed: Fri May 6 12:46:31 2016 -0400
----------------------------------------------------------------------
tools/import-mbox.py | 124 +++++++++++++++++++++++++++--
tools/sync-imap.py | 197 ----------------------------------------------
2 files changed, 118 insertions(+), 203 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/6127911a/tools/import-mbox.py
----------------------------------------------------------------------
diff --git a/tools/import-mbox.py b/tools/import-mbox.py
index 2220f72..0b59d1f 100644
--- a/tools/import-mbox.py
+++ b/tools/import-mbox.py
@@ -45,6 +45,10 @@ except:
print("Sorry, you need to install the elasticsearch and formatflowed modules from pip first.")
sys.exit(-1)
+# change working directory to location of this script
+
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
y = 0
baddies = 0
block = Lock()
@@ -57,6 +61,7 @@ appender = "apache.org"
source = "./"
maildir = False
+imap = False
list_override = None
project = ""
filebased = False
@@ -174,7 +179,18 @@ class SlurpThread(Thread):
EM = 1
stime = time.time()
dFile = False
- if filebased:
+ if maildir:
+ messages = mailbox.Maildir(tmpname)
+ elif imap:
+ y -= 1 # TODO don't understand the increment above
+ imap4 = mla[2]
+ def mailgen(list):
+ for uid in list:
+ msgbytes = imap4.uid('fetch', uid, '(RFC822)')[1][0][1]
+ yield email.message_from_bytes(msgbytes)
+ messages = mailgen(mla[0])
+ xlist_override = mla[1]
+ elif filebased:
tmpname = mla[0]
filename = mla[0]
@@ -197,6 +213,8 @@ class SlurpThread(Thread):
except Exception as err:
print("This wasn't a gzip file: %s" % err )
print("Slurping %s" % filename)
+ messages = mailbox.mbox(tmpname)
+
else:
ml = mla[0]
mboxfile = mla[1]
@@ -215,15 +233,11 @@ class SlurpThread(Thread):
with open(tmpname, "w") as f:
f.write(inp)
f.close()
+ messages = mailbox.mbox(tmpname)
count = 0
LEY = EY
- if maildir:
- messages = mailbox.Maildir(tmpname)
- else:
- messages = mailbox.mbox(tmpname)
-
for message in messages:
if resendTo:
print("Delivering message %s via MTA" % message['message-id'] if 'message-id' in message else '??')
@@ -285,6 +299,8 @@ class SlurpThread(Thread):
print("Parsed %u records from %s" % (count, filename))
if dFile:
os.unlink(tmpname)
+ elif imap:
+ print("Parsed %u records from imap" % count)
else:
print("Parsed %s/%s: %u records from %s" % (ml, mboxfile, count, tmpname))
os.unlink(tmpname)
@@ -460,6 +476,102 @@ elif source[0] == "h":
if quickmode and qn >= 2:
break
+# IMAP(S) based import?
+elif source[0] == "i":
+ imap = True
+ import urllib, getpass, imaplib
+ url = urllib.parse.urlparse(source)
+
+ port = url.port or (143 if url.scheme == 'imap' else 993)
+ user = url.username or getpass.getuser()
+ password = url.password or getpass.getpass('IMAP Password: ')
+ folder = url.path.strip('/') or 'INBOX'
+ listname = list_override or "<%s/%s.%s>" % (user, folder, url.hostname)
+
+ # fetch message-id => _id pairs from elasticsearch
+
+ es = Elasticsearch()
+ result = es.search(scroll = '5m',
+ body = {
+ 'size': 1024,
+ 'fields': ['message-id'],
+ 'query': {'match': {'list': listname}}
+ }
+ )
+
+ db = {}
+ while len(result['hits']['hits']) > 0:
+ for hit in result['hits']['hits']:
+ db[hit['fields']['message-id'][0]] = hit['_id']
+ result = es.scroll(scroll='5m', scroll_id=result['_scroll_id'])
+
+ # fetch message-id => uid pairs from imap
+
+ if url.scheme == 'imaps':
+ imap4 = imaplib.IMAP4_SSL(url.hostname, port)
+ else:
+ imap4 = imaplib.IMAP4(url.hostname, port)
+ imap4.login(user, password)
+ imap4.select(folder, readonly=True)
+ results = imap4.uid('search', None, 'ALL')
+ uids = b','.join(results[1][0].split())
+ results = imap4.uid('fetch', uids, '(BODY[HEADER.FIELDS (MESSAGE-ID)])')
+
+ mail = {}
+ uid_re = re.compile(b'^\d+ \(UID (\d+) BODY\[')
+ mid_re = re.compile(b'^Message-ID:\s*(.*?)\s*$', re.I)
+ uid = None
+ for result in results[1]:
+ for line in result:
+ if isinstance(line, bytes):
+ match = uid_re.match(line)
+ if match:
+ uid = match.group(1)
+ else:
+ match = mid_re.match(line)
+ if match:
+ try:
+ mail[match.group(1).decode('utf-8')] = uid
+ uid = None
+ except ValueError:
+ pass
+
+ # delete items from elasticsearch that are not present in imap
+
+ queue1 = []
+ queue2 = []
+ for mid, _id in db.items():
+ if not mid in mail:
+ queue1.append({
+ '_op_type': 'delete',
+ '_index': dbname,
+ '_type': 'mbox',
+ '_id': _id
+ })
+ queue2.append({
+ '_op_type': 'delete',
+ '_index': dbname,
+ '_type': 'mbox_source',
+ '_id': _id
+ })
+ print("deleting: " + mid)
+
+ while len(queue1) > 0:
+ eshelper.bulk(es, queue1[0:1024])
+ del queue1[0:1024]
+
+ while len(queue2) > 0:
+ eshelper.bulk(es, queue2[0:1024])
+ del queue2[0:1024]
+
+ # add new items to elasticsearch from imap
+
+ uids = []
+ for mid, uid in mail.items():
+ if not mid in db:
+ uids.append(uid)
+ lists.append([uids, listname, imap4])
+
threads = []
cc = int( multiprocessing.cpu_count() / 2) + 1
print("Starting up to %u threads to fetch the %u %s lists" % (cc, len(lists), project))
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/6127911a/tools/sync-imap.py
----------------------------------------------------------------------
diff --git a/tools/sync-imap.py b/tools/sync-imap.py
deleted file mode 100755
index 14d2711..0000000
--- a/tools/sync-imap.py
+++ /dev/null
@@ -1,197 +0,0 @@
-#!/usr/bin/env python3.4
-# -*- coding: utf-8 -*-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
- #the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-""" Syncronize ponymail with an imap server.
-
-Fetches message-ids from both ponymail and an imap server, and adds or
-deletes whatever is necessary from ponymail to make it match.
-
-See usage for instructions.
-
-"""
-
-import argparse
-import configparser
-import elasticsearch
-from elasticsearch import helpers as eshelper
-import imaplib
-import os
-import pwd
-import subprocess
-import sys
-import re
-
-# change working directory to location of this script
-
-os.chdir(os.path.dirname(os.path.abspath(__file__)))
-
-# global defaults
-
-es_list = None
-imap_host = 'localhost'
-imap_port = 993
-imap_user = pwd.getpwuid(os.getuid()).pw_name
-imap_password = None
-imap_folder = 'INBOX'
-html2text = False
-verbose = False
-
-# fetch config overrides
-
-config = configparser.RawConfigParser()
-config.read('ponymail.cfg')
-iname = config.get("elasticsearch", "dbname")
-if config.has_option('imap', 'host'):
- imap_host = config.get('imap', 'host')
-if config.has_option('imap', 'port'):
- imap_port = config.getint('imap', 'port')
-if config.has_option('imap', 'user'):
- imap_user = config.getint('imap', 'user')
-if config.has_option('imap', 'password'):
- imap_password = config.getint('imap', 'password')
-
-# fetch command line argument overrides
-
-parser = argparse.ArgumentParser(description='Command line options.')
-parser.add_argument('--list', dest='list', type=str, nargs=1,
- help='ElasticSearch list')
-parser.add_argument('--host', dest='host', type=str, nargs=1,
- help='IMAP host')
-parser.add_argument('--port', dest='port', type=int, nargs=1,
- help='IMAP port')
-parser.add_argument('--user', dest='user', type=str, nargs=1,
- help='IMAP user')
-parser.add_argument('--password', dest='password', type=str, nargs=1,
- help='IMAP password')
-parser.add_argument('--folder', dest='folder', type=str, nargs=1,
- help='IMAP folder')
-parser.add_argument('--html2text', dest='html2text', action='store_true',
- help='Try to convert HTML to text if no text/plain message is found')
-parser.add_argument('--verbose', dest='verbose', action='store_true',
- help='Output additional log messages')
-
-args = parser.parse_args()
-
-if args.list:
- es_list = args.list[0]
-if args.host:
- imap_host = args.host[0]
-if args.port:
- imap_port = args.port[0]
-if args.user:
- imap_user = args.user[0]
-if args.password:
- imap_password = args.password[0]
-if args.folder:
- imap_folder = args.folder[0]
-if args.html2text:
- html2text = True
-if args.verbose:
- verbose = True
-
-if not es_list or not imap_password:
- parser.print_help()
- sys.exit(-1)
-
-es_list = "<%s>" % es_list.strip("<>") # We need <> around it!
-
-# fetch message-id => _id pairs from elasticsearch
-
-es = elasticsearch.Elasticsearch()
-result = es.search(scroll = '5m',
- body = {
- 'size': 1024,
- 'fields': ['message-id'],
- 'query': {'match': {'list': es_list}}
- }
-)
-
-db = {}
-while len(result['hits']['hits']) > 0:
- for hit in result['hits']['hits']:
- db[hit['fields']['message-id'][0]] = hit['_id']
- result = es.scroll(scroll='5m', scroll_id=result['_scroll_id'])
-
-# fetch message-id => uid pairs from imap
-
-imap = imaplib.IMAP4_SSL(imap_host, imap_port)
-imap.login(imap_user, imap_password)
-imap.select(imap_folder, readonly=True)
-results = imap.uid('search', None, 'ALL')
-uids = b','.join(results[1][0].split())
-results = imap.uid('fetch', uids, '(BODY[HEADER.FIELDS (MESSAGE-ID)])')
-
-mail = {}
-uid_re = re.compile(b'^\d+ \(UID (\d+) BODY\[')
-mid_re = re.compile(b'^Message-ID:\s*(.*?)\s*$', re.I)
-uid = None
-for result in results[1]:
- for line in result:
- if isinstance(line, bytes):
- match = uid_re.match(line)
- if match:
- uid = match.group(1)
- else:
- match = mid_re.match(line)
- if match:
- try:
- mail[match.group(1).decode('utf-8')] = uid
- uid = None
- except ValueError:
- pass
-
-# delete items from elasticsearch that are not present in imap
-
-queue1 = []
-queue2 = []
-for mid, _id in db.items():
- if True: # not mid in mail:
- queue1.append({
- '_op_type': 'delete',
- '_index': iname,
- '_type': 'mbox',
- '_id': _id
- })
- queue2.append({
- '_op_type': 'delete',
- '_index': iname,
- '_type': 'mbox_source',
- '_id': _id
- })
- print("deleting: " + mid)
-
-while len(queue1) > 0:
- eshelper.bulk(es, queue1[0:1024])
- del queue1[0:1024]
-
-while len(queue2) > 0:
- eshelper.bulk(es, queue2[0:1024])
- del queue2[0:1024]
-
-# add new items to elasticsearch from imap
-
-for mid, uid in mail.items():
- if not mid in db:
- print("indexing %s" % mid)
- argv = [sys.executable, 'archiver.py', '--lid=%s' % es_list]
- if verbose: argv.append('--verbose')
- if html2text: argv.append('--html2text')
- child = subprocess.Popen(argv, stdin=subprocess.PIPE)
- child.stdin.write(imap.uid('fetch', uid, '(RFC822)')[1][0][1])
- child.stdin.close()
- rc = child.wait()
- if rc != 0: print("rc %d" % rc)
[05/11] incubator-ponymail git commit: switch to a mailman section
Posted by hu...@apache.org.
switch to a mailman section
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/6b42d4dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/6b42d4dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/6b42d4dd
Branch: refs/heads/master
Commit: 6b42d4dd5fa22fd85284205c98828c2348268df6
Parents: 2bf40b5
Author: Sam Ruby <ru...@intertwingly.net>
Authored: Wed May 4 08:38:57 2016 -0400
Committer: Sam Ruby <ru...@intertwingly.net>
Committed: Wed May 4 08:38:57 2016 -0400
----------------------------------------------------------------------
tools/archiver.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/6b42d4dd/tools/archiver.py
----------------------------------------------------------------------
diff --git a/tools/archiver.py b/tools/archiver.py
index cdd52e1..31f790a 100644
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -26,6 +26,11 @@ Enable the module by adding the following to your mailman.cfg file::
class: mailman_ponymail_plugin.Archiver
enable: yes
+and by adding the following to ponymail.cfg:
+
+[mailman]
+plugin: true
+
OR, to use the STDIN version (non-MM3 mailing list managers),
sub someone to the list(s) and add this to their .forward file:
"|/usr/bin/env python3.4 /path/to/archiver.py"
@@ -56,7 +61,7 @@ config.read("%s/ponymail.cfg" % path)
auth = None
parseHTML = False
-if config.has_section('archiver.ponymail'):
+if config.has_section('mailman') and config.has_option('mailman', 'plugin'):
from zope.interface import implementer
from mailman.interfaces.archiver import IArchiver
from mailman.interfaces.archiver import ArchivePolicy
@@ -103,7 +108,7 @@ def pm_charsets(msg):
class Archiver(object):
""" A mailman 3 archiver that forwards messages to pony mail. """
- if config.has_section('archiver.ponymail'):
+ if config.has_section('mailman') and config.has_option('mailman', 'plugin'):
implementer(IArchiver)
name = "ponymail"
[02/11] incubator-ponymail git commit: start to split out compute
updates
Posted by hu...@apache.org.
start to split out compute updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/9571469a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/9571469a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/9571469a
Branch: refs/heads/master
Commit: 9571469a7bd7cb0c06eded35c232f54ed89e7d37
Parents: cefa48d
Author: Sam Ruby <ru...@intertwingly.net>
Authored: Wed May 4 07:00:29 2016 -0400
Committer: Sam Ruby <ru...@intertwingly.net>
Committed: Wed May 4 07:00:29 2016 -0400
----------------------------------------------------------------------
tools/archiver.py | 270 ++++++++++++++++++++++++++-----------------------
1 file changed, 145 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/9571469a/tools/archiver.py
----------------------------------------------------------------------
diff --git a/tools/archiver.py b/tools/archiver.py
index 8cdbcbf..4ba0c5d 100644
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -214,19 +214,15 @@ class Archiver(object):
return body
- def archive_message(self, mlist, msg):
- """Send the message to the archiver.
+ def compute_updates(self, lid, private, msg):
+ """Determine what needs to be sent to the archiver.
- :param mlist: The IMailingList object.
+ :param lid: The list id
:param msg: The message object.
"""
- lid = None
- m = re.search(r"(<.+>)", mlist.list_id.replace("@", "."))
- if m:
- lid = m.group(1)
- else:
- lid = "<%s>" % mlist.list_id.strip("<>").replace("@", ".")
+ ojson = None
+
if self.cropout:
crops = self.cropout.split(" ")
# Regex replace?
@@ -287,17 +283,9 @@ class Archiver(object):
attachments, contents = self.msgfiles(msg)
if body or attachments:
- private = False
- if hasattr(mlist, 'archive_public') and mlist.archive_public == True:
- private = False
- elif hasattr(mlist, 'archive_public') and mlist.archive_public == False:
- private = True
- elif hasattr(mlist, 'archive_policy') and mlist.archive_policy is not ArchivePolicy.public:
- private = True
pmid = mid
try:
mid = "%s@%s" % (hashlib.sha224(msg.as_bytes()).hexdigest(), lid)
- print(mid)
except Exception as err:
if logger:
logger.warn("Could not generate MID: %s" % err)
@@ -329,122 +317,154 @@ class Archiver(object):
'body': body.decode('utf-8', 'replace') if type(body) is bytes else body,
'attachments': attachments
}
+
+ self.msg_metadata = msg_metadata
+ self.irt = irt
+
+ return ojson, contents
- if contents:
- for key in contents:
- self.es.index(
- index=self.dbname,
- doc_type="attachment",
- id=key,
- body = {
- 'source': contents[key]
- }
- )
+ def archive_message(self, mlist, msg):
+ """Send the message to the archiver.
+
+ :param mlist: The IMailingList object.
+ :param msg: The message object.
+ """
+
+ lid = None
+ m = re.search(r"(<.+>)", mlist.list_id.replace("@", "."))
+ if m:
+ lid = m.group(1)
+ else:
+ lid = "<%s>" % mlist.list_id.strip("<>").replace("@", ".")
+
+ private = False
+ if hasattr(mlist, 'archive_public') and mlist.archive_public == True:
+ private = False
+ elif hasattr(mlist, 'archive_public') and mlist.archive_public == False:
+ private = True
+ elif hasattr(mlist, 'archive_policy') and mlist.archive_policy is not ArchivePolicy.public:
+ private = True
+
+ ojson, contents = self.compute_updates(lid, private, msg)
+
+ msg_metadata = self.msg_metadata
+ irt = self.irt
+
+ if contents:
+ for key in contents:
+ self.es.index(
+ index=self.dbname,
+ doc_type="attachment",
+ id=key,
+ body = {
+ 'source': contents[key]
+ }
+ )
+
+ self.es.index(
+ index=self.dbname,
+ doc_type="mbox",
+ id=ojson['mid'],
+ consistency = self.consistency,
+ body = ojson
+ )
+ self.es.index(
+ index=self.dbname,
+ doc_type="mbox_source",
+ id=ojson['mid'],
+ consistency = self.consistency,
+ body = {
+ "message-id": msg_metadata['message-id'],
+ "source": msg.as_string()
+ }
+ )
+
+ # If MailMan and list info is present, save/update it in ES:
+ if hasattr(mlist, 'description') and hasattr(mlist, 'list_name') and mlist.description and mlist.list_name:
self.es.index(
index=self.dbname,
- doc_type="mbox",
- id=mid,
- consistency = self.consistency,
- body = ojson
- )
-
- self.es.index(
- index=self.dbname,
- doc_type="mbox_source",
- id=mid,
+ doc_type="mailinglists",
+ id=lid,
consistency = self.consistency,
body = {
- "message-id": msg_metadata['message-id'],
- "source": msg.as_string()
+ 'list': lid,
+ 'name': mlist.list_name,
+ 'description': mlist.description,
+ 'private': private
}
)
-
- # If MailMan and list info is present, save/update it in ES:
- if hasattr(mlist, 'description') and hasattr(mlist, 'list_name') and mlist.description and mlist.list_name:
- self.es.index(
- index=self.dbname,
- doc_type="mailinglists",
- id=lid,
- consistency = self.consistency,
- body = {
- 'list': lid,
- 'name': mlist.list_name,
- 'description': mlist.description,
- 'private': private
- }
- )
-
- if logger:
- logger.info("Pony Mail archived message %s successfully" % mid)
- oldrefs = []
-
- # Is this a direct reply to a pony mail email?
- if irt != "":
- dm = re.search(r"pony-([a-f0-9]+)-([a-f0-9]+)@", irt)
- if dm:
- cid = dm.group(1)
- mid = dm.group(2)
- if self.es.exists(index = self.dbname, doc_type = 'account', id = cid):
- doc = self.es.get(index = self.dbname, doc_type = 'account', id = cid)
- if doc:
- oldrefs.append(cid)
- self.es.index(
- index=self.dbname,
- doc_type="notifications",
- consistency = self.consistency,
- body = {
- 'type': 'direct',
- 'recipient': cid,
- 'list': lid,
- 'private': private,
- 'date': msg_metadata['date'],
- 'from': msg_metadata['from'],
- 'to': msg_metadata['to'],
- 'subject': msg_metadata['subject'],
- 'message-id': msg_metadata['message-id'],
- 'in-reply-to': irt,
- 'epoch': email.utils.mktime_tz(mdate),
- 'mid': mid,
- 'seen': 0
- }
- )
- if logger:
- logger.info("Notification sent to %s for %s" % (cid, mid))
-
- # Are there indirect replies to pony emails?
- if msg_metadata.get('references'):
- for im in re.finditer(r"pony-([a-f0-9]+)-([a-f0-9]+)@", msg_metadata.get('references')):
- cid = im.group(1)
- mid = im.group(2)
- if self.es.exists(index = self.dbname, doc_type = 'account', id = cid):
- doc = self.es.get(index = self.dbname, doc_type = 'account', id = cid)
-
- # does the user want to be notified of indirect replies?
- if doc and 'preferences' in doc['_source'] and doc['_source']['preferences'].get('notifications') == 'indirect' and not cid in oldrefs:
- oldrefs.append(cid)
- self.es.index(
- index=self.dbname,
- consistency = self.consistency,
- doc_type="notifications",
- body = {
- 'type': 'indirect',
- 'recipient': cid,
- 'list': lid,
- 'private': private,
- 'date': msg_metadata['date'],
- 'from': msg_metadata['from'],
- 'to': msg_metadata['to'],
- 'subject': msg_metadata['subject'],
- 'message-id': msg_metadata['message-id'],
- 'in-reply-to': mirt,
- 'epoch': email.utils.mktime_tz(mdate),
- 'mid': mid,
- 'seen': 0
- }
- )
- if logger:
- logger.info("Notification sent to %s for %s" % (cid, mid))
+
+ if logger:
+ logger.info("Pony Mail archived message %s successfully" % mid)
+ oldrefs = []
+
+ # Is this a direct reply to a pony mail email?
+ if irt != "":
+ dm = re.search(r"pony-([a-f0-9]+)-([a-f0-9]+)@", irt)
+ if dm:
+ cid = dm.group(1)
+ mid = dm.group(2)
+ if self.es.exists(index = self.dbname, doc_type = 'account', id = cid):
+ doc = self.es.get(index = self.dbname, doc_type = 'account', id = cid)
+ if doc:
+ oldrefs.append(cid)
+ self.es.index(
+ index=self.dbname,
+ doc_type="notifications",
+ consistency = self.consistency,
+ body = {
+ 'type': 'direct',
+ 'recipient': cid,
+ 'list': lid,
+ 'private': private,
+ 'date': msg_metadata['date'],
+ 'from': msg_metadata['from'],
+ 'to': msg_metadata['to'],
+ 'subject': msg_metadata['subject'],
+ 'message-id': msg_metadata['message-id'],
+ 'in-reply-to': irt,
+ 'epoch': email.utils.mktime_tz(mdate),
+ 'mid': mid,
+ 'seen': 0
+ }
+ )
+ if logger:
+ logger.info("Notification sent to %s for %s" % (cid, mid))
+
+ # Are there indirect replies to pony emails?
+ if msg_metadata.get('references'):
+ for im in re.finditer(r"pony-([a-f0-9]+)-([a-f0-9]+)@", msg_metadata.get('references')):
+ cid = im.group(1)
+ mid = im.group(2)
+ if self.es.exists(index = self.dbname, doc_type = 'account', id = cid):
+ doc = self.es.get(index = self.dbname, doc_type = 'account', id = cid)
+
+ # does the user want to be notified of indirect replies?
+ if doc and 'preferences' in doc['_source'] and doc['_source']['preferences'].get('notifications') == 'indirect' and not cid in oldrefs:
+ oldrefs.append(cid)
+ self.es.index(
+ index=self.dbname,
+ consistency = self.consistency,
+ doc_type="notifications",
+ body = {
+ 'type': 'indirect',
+ 'recipient': cid,
+ 'list': lid,
+ 'private': private,
+ 'date': msg_metadata['date'],
+ 'from': msg_metadata['from'],
+ 'to': msg_metadata['to'],
+ 'subject': msg_metadata['subject'],
+ 'message-id': msg_metadata['message-id'],
+ 'in-reply-to': mirt,
+ 'epoch': email.utils.mktime_tz(mdate),
+ 'mid': mid,
+ 'seen': 0
+ }
+ )
+ if logger:
+ logger.info("Notification sent to %s for %s" % (cid, mid))
return lid
def list_url(self, mlist):
[11/11] incubator-ponymail git commit: make new MID generation
optional, enable by default
Posted by hu...@apache.org.
make new MID generation optional, enable by default
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/43d70910
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/43d70910
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/43d70910
Branch: refs/heads/master
Commit: 43d70910c09bcfb0aca94bf0e71050e60d039bb9
Parents: f507423
Author: humbedooh <hu...@apache.org>
Authored: Wed Jun 1 14:39:38 2016 +0200
Committer: humbedooh <hu...@apache.org>
Committed: Wed Jun 1 14:39:38 2016 +0200
----------------------------------------------------------------------
tools/archiver.py | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/43d70910/tools/archiver.py
----------------------------------------------------------------------
diff --git a/tools/archiver.py b/tools/archiver.py
index b49472e..010e88e 100644
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -296,7 +296,12 @@ class Archiver(object):
if body or attachments:
pmid = mid
try:
- mid = "%s@%s" % (hashlib.sha224(msg.as_bytes()).hexdigest(), lid)
+ # Use full message as bytes for mid?
+ if config.has_section('archiver') and config.has_option("archiver", "generator") and config.get("archiver", "generator") == "full":
+ mid = "%s@%s" % (hashlib.sha224(msg.as_bytes()).hexdigest(), lid)
+ else:
+ # Or revert to the old way?
+ mid = "%s@%s@%s" % (hashlib.sha224(body if type(body) is bytes else body.encode('ascii', 'ignore')).hexdigest(), uid_mdate, lid)
except Exception as err:
if logger:
logger.warn("Could not generate MID: %s" % err)
@@ -475,7 +480,7 @@ class Archiver(object):
)
if logger:
logger.info("Notification sent to %s for %s" % (cid, mid))
- return lid
+ return lid, ojson['mid']
def list_url(self, mlist):
""" Gots
@@ -598,8 +603,8 @@ if __name__ == '__main__':
msg_metadata = namedtuple('importmsg', ['list_id', 'archive_public'])(list_id = msg.get('list-id'), archive_public=ispublic)
try:
- lid = foo.archive_message(msg_metadata, msg)
- print("%s: Done archiving to %s!" % (email.utils.formatdate(), lid))
+ lid, mid = foo.archive_message(msg_metadata, msg)
+ print("%s: Done archiving to %s as %s!" % (email.utils.formatdate(), lid, mid))
except Exception as err:
if args.verbose:
import traceback
Re: [01/11] incubator-ponymail git commit: add a synch ponymail with imap tool
Posted by Daniel Gruno <hu...@apache.org>.
Should have squashed this, apologies!
On 2016-06-01 14:41, humbedooh@apache.org wrote:
> Repository: incubator-ponymail
> Updated Branches:
> refs/heads/master 3acd303a4 -> 43d70910c
>
>
> add a synch ponymail with imap tool
>
>
> Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
> Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/cefa48d0
> Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/cefa48d0
> Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/cefa48d0
>
> Branch: refs/heads/master
> Commit: cefa48d0baef7a9cd37c53f9dcbc3e52670c329a
> Parents: a383086
> Author: Sam Ruby <ru...@intertwingly.net>
> Authored: Tue May 3 14:21:49 2016 -0400
> Committer: Sam Ruby <ru...@intertwingly.net>
> Committed: Tue May 3 14:21:49 2016 -0400
>
> ----------------------------------------------------------------------
> tools/archiver.py | 25 +++++--
> tools/sync-imap.py | 176 ++++++++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 195 insertions(+), 6 deletions(-)
> ----------------------------------------------------------------------
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/cefa48d0/tools/archiver.py
> ----------------------------------------------------------------------
> diff --git a/tools/archiver.py b/tools/archiver.py
> index ec1fef7..8cdbcbf 100644
> --- a/tools/archiver.py
> +++ b/tools/archiver.py
> @@ -75,6 +75,7 @@ def parse_attachment(part):
> dispositions = cd.strip().split(";")
> if dispositions[0].lower() == "attachment":
> fd = part.get_payload(decode=True)
> + if not fd: return None, None
> attachment = {}
> attachment['content_type'] = part.get_content_type()
> attachment['size'] = len(fd)
> @@ -83,7 +84,8 @@ def parse_attachment(part):
> b64 = codecs.encode(fd, "base64").decode('ascii', 'ignore')
> attachment['hash'] = h
> for param in dispositions[1:]:
> - key,val = param.split("=")
> + if not '=' in param: continue
> + key,val = param.split("=", 1)
> if key.lower().strip() == "filename":
> val = val.strip(' "')
> print("Found attachment: %s" % val)
> @@ -282,8 +284,9 @@ class Archiver(object):
> body = body.encode('utf-8')
> except:
> body = None
> - if body:
> - attachments, contents = self.msgfiles(msg)
> +
> + attachments, contents = self.msgfiles(msg)
> + if body or attachments:
> private = False
> if hasattr(mlist, 'archive_public') and mlist.archive_public == True:
> private = False
> @@ -293,7 +296,8 @@ class Archiver(object):
> private = True
> pmid = mid
> try:
> - mid = "%s@%s@%s" % (hashlib.sha224(body if type(body) is bytes else body.encode('ascii', 'ignore')).hexdigest(), uid_mdate, lid)
> + mid = "%s@%s" % (hashlib.sha224(msg.as_bytes()).hexdigest(), lid)
> + print(mid)
> except Exception as err:
> if logger:
> logger.warn("Could not generate MID: %s" % err)
> @@ -475,6 +479,8 @@ if __name__ == '__main__':
> help='Use the archive timestamp as the email date instead of the Date header')
> parser.add_argument('--quiet', dest='quiet', action='store_true',
> help='Do not exit -1 if the email could not be parsed')
> + parser.add_argument('--verbose', dest='verbose', action='store_true',
> + help='Output additional log messages')
> parser.add_argument('--html2text', dest='html2text', action='store_true',
> help='Try to convert HTML to text if no text/plain message is found')
> args = parser.parse_args()
> @@ -482,6 +488,10 @@ if __name__ == '__main__':
> if args.html2text:
> import html2text
> parseHTML = True
> +
> + if args.verbose:
> + import logging
> + logging.basicConfig(stream=sys.stdout, level=logging.INFO)
>
> foo = Archiver()
> input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8', errors="ignore")
> @@ -512,7 +522,7 @@ if __name__ == '__main__':
> msg.replace_header('List-ID', msg.get(altheader))
> except:
> msg.add_header('list-id', msg.get(altheader))
> -
> +
> # Set specific LID?
> if args.lid and len(args.lid[0]) > 3:
> try:
> @@ -562,6 +572,9 @@ if __name__ == '__main__':
> lid = foo.archive_message(msg_metadata, msg)
> print("%s: Done archiving to %s!" % (email.utils.formatdate(), lid))
> except Exception as err:
> + if args.verbose:
> + import traceback
> + traceback.print_exc()
> print("Archiving failed!: %s" % err)
> raise Exception("Archiving to ES failed")
> else:
> @@ -572,4 +585,4 @@ if __name__ == '__main__':
> else:
> print("Could not parse email: %s" % err)
> sys.exit(-1)
> -
> \ No newline at end of file
> +
>
> http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/cefa48d0/tools/sync-imap.py
> ----------------------------------------------------------------------
> diff --git a/tools/sync-imap.py b/tools/sync-imap.py
> new file mode 100755
> index 0000000..43d457d
> --- /dev/null
> +++ b/tools/sync-imap.py
> @@ -0,0 +1,176 @@
> +#!/usr/bin/env python3.4
> +# -*- coding: utf-8 -*-
> +# Licensed to the Apache Software Foundation (ASF) under one or more
> +# contributor license agreements. See the NOTICE file distributed with
> +# this work for additional information regarding copyright ownership.
> +# The ASF licenses this file to You under the Apache License, Version 2.0
> +# (the "License"); you may not use this file except in compliance with
> + #the License. You may obtain a copy of the License at
> +#
> +# http://www.apache.org/licenses/LICENSE-2.0
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS,
> +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +# See the License for the specific language governing permissions and
> +# limitations under the License.
> +
> +""" Syncronize ponymail with an imap server.
> +
> +Fetches message-ids from both ponymail and an imap server, and adds or
> +deletes whatever is necessary from ponymail to make it match.
> +
> +See usage for instructions.
> +
> +"""
> +
> +import argparse
> +import configparser
> +import elasticsearch
> +import imaplib
> +import os
> +import pwd
> +import subprocess
> +import sys
> +import re
> +
> +# change working directory to location of this script
> +
> +os.chdir(os.path.dirname(os.path.abspath(__file__)))
> +
> +# global defaults
> +
> +es_list = None
> +imap_host = 'localhost'
> +imap_port = 993
> +imap_user = pwd.getpwuid(os.getuid()).pw_name
> +imap_password = None
> +imap_folder = 'INBOX'
> +html2text = False
> +verbose = False
> +
> +# fetch config overrides
> +
> +config = configparser.RawConfigParser()
> +config.read('ponymail.cfg')
> +iname = config.get("elasticsearch", "dbname")
> +if config.has_option('imap', 'host'):
> + imap_host = config.get('imap', 'host')
> +if config.has_option('imap', 'port'):
> + imap_port = config.getint('imap', 'port')
> +if config.has_option('imap', 'user'):
> + imap_user = config.getint('imap', 'user')
> +if config.has_option('imap', 'password'):
> + imap_password = config.getint('imap', 'password')
> +
> +# fetch command line argument overrides
> +
> +parser = argparse.ArgumentParser(description='Command line options.')
> +parser.add_argument('--list', dest='list', type=str, nargs=1,
> + help='ElasticSearch list')
> +parser.add_argument('--host', dest='host', type=str, nargs=1,
> + help='IMAP host')
> +parser.add_argument('--port', dest='port', type=int, nargs=1,
> + help='IMAP port')
> +parser.add_argument('--user', dest='user', type=str, nargs=1,
> + help='IMAP user')
> +parser.add_argument('--password', dest='password', type=str, nargs=1,
> + help='IMAP password')
> +parser.add_argument('--folder', dest='folder', type=str, nargs=1,
> + help='IMAP folder')
> +parser.add_argument('--html2text', dest='html2text', action='store_true',
> + help='Try to convert HTML to text if no text/plain message is found')
> +parser.add_argument('--verbose', dest='verbose', action='store_true',
> + help='Output additional log messages')
> +
> +args = parser.parse_args()
> +
> +if args.list:
> + es_list = args.list[0]
> +if args.host:
> + imap_host = args.host[0]
> +if args.port:
> + imap_port = args.port[0]
> +if args.user:
> + imap_user = args.user[0]
> +if args.password:
> + imap_password = args.password[0]
> +if args.folder:
> + imap_folder = args.folder[0]
> +if args.html2text:
> + html2text = True
> +if args.verbose:
> + verbose = True
> +
> +if not es_list or not imap_password:
> + parser.print_help()
> + sys.exit(-1)
> +
> +es_list = "<%s>" % es_list.strip("<>") # We need <> around it!
> +
> +# fetch message-id => _id pairs from elasticsearch
> +
> +es = elasticsearch.Elasticsearch()
> +result = es.search(scroll = '5m',
> + body = {
> + 'size': 1024,
> + 'fields': ['message-id'],
> + 'query': {'match': {'list': es_list}}
> + }
> +)
> +
> +db = {}
> +while len(result['hits']['hits']) > 0:
> + for hit in result['hits']['hits']:
> + db[hit['fields']['message-id'][0]] = hit['_id']
> + result = es.scroll(scroll='5m', scroll_id=result['_scroll_id'])
> +
> +# fetch message-id => uid pairs from imap
> +
> +imap = imaplib.IMAP4_SSL(imap_host, imap_port)
> +imap.login(imap_user, imap_password)
> +imap.select(imap_folder, readonly=True)
> +results = imap.uid('search', None, 'ALL')
> +uids = b','.join(results[1][0].split())
> +results = imap.uid('fetch', uids, '(BODY[HEADER.FIELDS (MESSAGE-ID)])')
> +
> +mail = {}
> +uid_re = re.compile(b'^\d+ \(UID (\d+) BODY\[')
> +mid_re = re.compile(b'^Message-ID:\s*(.*?)\s*$', re.I)
> +uid = None
> +for result in results[1]:
> + for line in result:
> + if isinstance(line, bytes):
> + match = uid_re.match(line)
> + if match:
> + uid = match.group(1)
> + else:
> + match = mid_re.match(line)
> + if match:
> + try:
> + mail[match.group(1).decode('utf-8')] = uid
> + uid = None
> + except ValueError:
> + pass
> +
> +# delete items from elasticsearch that are not present in imap
> +
> +for mid, _id in db.items():
> + if not mid in mail:
> + es.delete(index=iname, id=_id, doc_type='mbox')
> + es.delete(index=iname, id=_id, doc_type='mbox_source')
> + print("deleted: " + mid)
> +
> +# add new items to elasticsearch from imap
> +
> +for mid, uid in mail.items():
> + if not mid in db:
> + argv = [sys.executable, 'archiver.py', '--lid=%s' % es_list]
> + if verbose: argv.append('--verbose')
> + if html2text: argv.append('--html2text')
> + child = subprocess.Popen(argv, stdin=subprocess.PIPE)
> + child.stdin.write(imap.uid('fetch', uid, '(RFC822)')[1][0][1])
> + child.stdin.close()
> + rc = child.wait()
> + print("inserted: %s, rc = %d" % (mid, rc))
> +
>
>
[09/11] incubator-ponymail git commit: Merge branch 'sync-imap' of
https://github.com/rubys/ponymail into rubys-sync-imap
Posted by hu...@apache.org.
Merge branch 'sync-imap' of https://github.com/rubys/ponymail into rubys-sync-imap
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/d1d6c7f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/d1d6c7f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/d1d6c7f5
Branch: refs/heads/master
Commit: d1d6c7f5823806badf8b413e2be2e62d147b6461
Parents: 3acd303 6127911
Author: humbedooh <hu...@apache.org>
Authored: Wed Jun 1 14:25:15 2016 +0200
Committer: humbedooh <hu...@apache.org>
Committed: Wed Jun 1 14:25:15 2016 +0200
----------------------------------------------------------------------
tools/archiver.py | 336 +++++++++++++++++++---------------
tools/import-mbox.py | 447 +++++++++++++++++-----------------------------
2 files changed, 351 insertions(+), 432 deletions(-)
----------------------------------------------------------------------
[07/11] incubator-ponymail git commit: remove defunct arguments;
wire up iBody argument
Posted by hu...@apache.org.
remove defunct arguments; wire up iBody argument
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/7135cd4a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/7135cd4a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/7135cd4a
Branch: refs/heads/master
Commit: 7135cd4a58a7d75ecdd9ba797479476b995326bd
Parents: 1ed519e
Author: Sam Ruby <ru...@intertwingly.net>
Authored: Thu May 5 07:56:44 2016 -0400
Committer: Sam Ruby <ru...@intertwingly.net>
Committed: Thu May 5 07:56:44 2016 -0400
----------------------------------------------------------------------
tools/archiver.py | 4 +++-
tools/import-mbox.py | 12 +-----------
2 files changed, 4 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/7135cd4a/tools/archiver.py
----------------------------------------------------------------------
diff --git a/tools/archiver.py b/tools/archiver.py
index 991b6d6..b49472e 100644
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -60,6 +60,7 @@ config = configparser.RawConfigParser()
config.read("%s/ponymail.cfg" % path)
auth = None
parseHTML = False
+iBody = None
if config.has_section('mailman') and config.has_option('mailman', 'plugin'):
from zope.interface import implementer
@@ -187,6 +188,7 @@ class Archiver(object):
def msgbody(self, msg):
+ global iBody
body = None
firstHTML = None
if msg.is_multipart():
@@ -211,7 +213,7 @@ class Archiver(object):
firstHTML = msg.get_payload(decode=True)
# this requires a GPL lib, user will have to install it themselves
- if firstHTML and (not body or len(body) <= 1):
+ if firstHTML and (not body or len(body) <= 1 or (iBody and str(body).find(str(iBody)) != -1)):
body = self.html2text(firstHTML.decode("utf-8", 'ignore') if type(firstHTML) is bytes else firstHTML)
for charset in pm_charsets(msg):
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/7135cd4a/tools/import-mbox.py
----------------------------------------------------------------------
diff --git a/tools/import-mbox.py b/tools/import-mbox.py
index 399afe6..2220f72 100644
--- a/tools/import-mbox.py
+++ b/tools/import-mbox.py
@@ -59,12 +59,10 @@ source = "./"
maildir = False
list_override = None
project = ""
-recursive = False
filebased = False
fileToLID = {}
interactive = False
extension = "*.mbox"
-attachments = False
piperWeirdness = False
parseHTML = False
iBody = None
@@ -312,8 +310,6 @@ parser.add_argument('--source', dest='source', type=str, nargs=1,
help='Source to scan (either http(s):// or file path)')
parser.add_argument('--dir', dest='dir', action='store_true',
help='Input is in Maildir format')
-parser.add_argument('--recursive', dest='recursive', action='store_true',
- help='Do a recursive scan (sub dirs etc)')
parser.add_argument('--interactive', dest='interactive', action='store_true',
help='Ask for help when possible')
parser.add_argument('--quick', dest='quick', action='store_true',
@@ -332,8 +328,6 @@ parser.add_argument('--domain', dest='domain', type=str, nargs=1,
help='Optional domain extension for MIDs and List ID reconstruction)')
parser.add_argument('--private', dest='private', action='store_true',
help='This is a privately archived list. Filter through auth proxy.')
-parser.add_argument('--attachments', dest='attachments', action='store_true',
- help='Also iport attached files in emails')
parser.add_argument('--dry', dest='dry', action='store_true',
help='Do not save emails to elasticsearch, only test importing')
parser.add_argument('--html2text', dest='html2text', action='store_true',
@@ -365,23 +359,19 @@ if args.project:
project = args.project[0]
if args.domain:
appender = args.domain[0]
-if args.recursive:
- recursive = args.recursive
if args.interactive:
interactive = args.interactive
if args.quick:
quickmode = args.quick
if args.private:
private = args.private
-if args.attachments:
- attachments = args.attachments
if args.ext:
extension = args.ext[0]
if args.html2text:
import html2text
parseHTML = True
if args.ibody:
- iBody = args.ibody[0]
+ archiver.iBody = args.ibody[0]
if args.resend:
resendTo = args.resend[0]
from smtplib import SMTP
[04/11] incubator-ponymail git commit: make archiver import-able
outside of mailman
Posted by hu...@apache.org.
make archiver import-able outside of mailman
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/2bf40b56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/2bf40b56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/2bf40b56
Branch: refs/heads/master
Commit: 2bf40b5654522dc6b8d64043f86ef40c60a748e2
Parents: 6bcb491
Author: Sam Ruby <ru...@intertwingly.net>
Authored: Wed May 4 07:50:30 2016 -0400
Committer: Sam Ruby <ru...@intertwingly.net>
Committed: Wed May 4 07:50:30 2016 -0400
----------------------------------------------------------------------
tools/archiver.py | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/2bf40b56/tools/archiver.py
----------------------------------------------------------------------
diff --git a/tools/archiver.py b/tools/archiver.py
index 4ba0c5d..cdd52e1 100644
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -35,15 +35,6 @@ sub someone to the list(s) and add this to their .forward file:
# Change this index name to whatever you picked!!
indexname = "ponymail_alpha"
logger = None
-if __name__ != '__main__':
- from zope.interface import implementer
- from mailman.interfaces.archiver import IArchiver
- from mailman.interfaces.archiver import ArchivePolicy
- import logging
- logger = logging.getLogger("mailman.archiver")
-else:
- import sys
- import argparse
from elasticsearch import Elasticsearch
import hashlib
@@ -65,6 +56,15 @@ config.read("%s/ponymail.cfg" % path)
auth = None
parseHTML = False
+if config.has_section('archiver.ponymail'):
+ from zope.interface import implementer
+ from mailman.interfaces.archiver import IArchiver
+ from mailman.interfaces.archiver import ArchivePolicy
+ import logging
+ logger = logging.getLogger("mailman.archiver")
+elif __name__ == '__main__':
+ import sys
+ import argparse
if config.has_option('elasticsearch', 'user'):
auth = (config.get('elasticsearch','user'), config.get('elasticsearch','password'))
@@ -103,7 +103,7 @@ def pm_charsets(msg):
class Archiver(object):
""" A mailman 3 archiver that forwards messages to pony mail. """
- if __name__ != '__main__':
+ if config.has_section('archiver.ponymail'):
implementer(IArchiver)
name = "ponymail"