You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by hu...@apache.org on 2016/07/10 11:09:20 UTC
[4/7] incubator-ponymail git commit: add a few new features to list
editor
add a few new features to list editor
- Allow a single message to be edited, by ID
- Allow obfuscation of specific parts of a message
- Allow for test runs where no changes are applied
Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/75feb484
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/75feb484
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/75feb484
Branch: refs/heads/master
Commit: 75feb484af84d689d71542c380b5c5b7e919b6e4
Parents: 0e86b6d
Author: humbedooh <hu...@apache.org>
Authored: Mon Jul 4 15:35:46 2016 +0200
Committer: humbedooh <hu...@apache.org>
Committed: Mon Jul 4 15:35:46 2016 +0200
----------------------------------------------------------------------
tools/edit-list.py | 79 +++++++++++++++++++++++++++++++++++--------------
1 file changed, 56 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/75feb484/tools/edit-list.py
----------------------------------------------------------------------
diff --git a/tools/edit-list.py b/tools/edit-list.py
index c5a562a..b416834 100644
--- a/tools/edit-list.py
+++ b/tools/edit-list.py
@@ -46,6 +46,9 @@ wildcard = None
debug = False
notag = False
desc = None
+mid = None
+dryrun = False
+obfuscate = None
ssl = False
dbname = config.get("elasticsearch", "dbname")
@@ -70,10 +73,14 @@ rootURL = ""
parser = argparse.ArgumentParser(description='Command line options.')
parser.add_argument('--source', dest='source', type=str, nargs=1,
help='Source list to edit')
+parser.add_argument('--mid', dest='mid', type=str, nargs=1,
+ help='Source Message-ID to edit')
parser.add_argument('--rename', dest='target', type=str, nargs=1,
help='(optional) new list ID')
parser.add_argument('--desc', dest='desc', type=str, nargs=1,
help='(optional) new list description')
+parser.add_argument('--obfuscate', dest='obfuscate', type=str, nargs=1,
+ help='Things to obfuscate in body, if any')
parser.add_argument('--private', dest='private', action='store_true',
help='Make all emails in list private')
parser.add_argument('--public', dest='public', action='store_true',
@@ -86,6 +93,8 @@ parser.add_argument('--debug', dest='debug', action='store_true',
help='Debug output - very noisy!')
parser.add_argument('--notag', dest='notag', action='store_true',
help='List IDs do not have <> in them')
+parser.add_argument('--test', dest='test', action='store_true',
+ help='Only test for occurrences, do not run the chosen action (dry run)')
args = parser.parse_args()
@@ -107,13 +116,19 @@ if args.debug:
debug = args.debug
if args.notag:
notag = args.notag
-
-
-if not sourceLID:
+if args.mid:
+ mid = args.mid[0]
+if args.obfuscate:
+ obfuscate = args.obfuscate[0]
+if args.test:
+ dryrun = args.test
+
+
+if not sourceLID and not mid:
print("No source list ID specified!")
parser.print_help()
sys.exit(-1)
-if not (targetLID or makePrivate or makePublic or deleteEmails or desc):
+if not (targetLID or makePrivate or makePublic or deleteEmails or desc or obfuscate):
print("Nothing to do! No target list ID or action specified")
parser.print_help()
sys.exit(-1)
@@ -122,12 +137,13 @@ if makePublic and makePrivate:
parser.print_help()
sys.exit(-1)
-sourceLID = ("%s" if notag else "<%s>") % sourceLID.replace("@", ".").strip("<>")
+if sourceLID:
+ sourceLID = ("%s" if notag else "<%s>") % sourceLID.replace("@", ".").strip("<>")
if targetLID:
targetLID = "<%s>" % targetLID.replace("@", ".").strip("<>")
print("Beginning list edit:")
-print(" - List ID: %s" % sourceLID)
+print(" - List ID: %s" % (sourceLID if sourceLID else mid))
if targetLID:
print(" - Target ID: %s" % targetLID)
if makePublic:
@@ -136,7 +152,8 @@ if makePrivate:
print(" - Action: Mark all emails private")
if deleteEmails:
print(" - Action: Delete emails (sources will be kept!)")
-
+if obfuscate:
+ print(" - Action: Obfuscate parts of email containing: %s" % obfuscate)
count = 0
if desc:
@@ -154,9 +171,22 @@ if desc:
}
)
-if targetLID or makePrivate or makePublic or deleteEmails:
+if targetLID or makePrivate or makePublic or deleteEmails or mid:
+ if dryrun:
+ print("DRY RUN - NO CHANGES WILL BE MADE")
print("Updating docs...")
then = time.time()
+ terms = {
+ 'wildcard' if wildcard else 'term': {
+ 'list_raw': sourceLID
+ }
+ }
+ if mid:
+ terms = {
+ 'term': {
+ 'mid': mid
+ }
+ }
page = es.search(
index=dbname,
doc_type="mbox",
@@ -167,11 +197,7 @@ if targetLID or makePrivate or makePublic or deleteEmails:
'query': {
'bool': {
'must': [
- {
- 'wildcard' if wildcard else 'term': {
- 'list_raw': sourceLID
- }
- }
+ terms
]
}
}
@@ -191,6 +217,10 @@ if targetLID or makePrivate or makePublic or deleteEmails:
for hit in page['hits']['hits']:
doc = hit['_id']
body = {}
+ if obfuscate:
+ body['body'] = hit['_source']['body'].replace(obfuscate, "...")
+ body['subject'] = hit['_source']['subject'].replace(obfuscate, "...")
+ body['from'] = hit['_source']['from'].replace(obfuscate, "...")
if targetLID:
body['list_raw'] = targetLID
body['list'] = targetLID
@@ -198,21 +228,24 @@ if targetLID or makePrivate or makePublic or deleteEmails:
body['private'] = True
if makePublic:
body['private'] = False
- js_arr.append({
- '_op_type': 'delete' if deleteEmails else 'update',
- '_index': dbname,
- '_type': 'mbox',
- '_id': doc,
- 'doc': body
- })
+ if not dryrun:
+ js_arr.append({
+ '_op_type': 'delete' if deleteEmails else 'update',
+ '_index': dbname,
+ '_type': 'mbox',
+ '_id': doc,
+ 'doc': body
+ })
count += 1
if (count % 500 == 0):
print("Processed %u emails..." % count)
- helpers.bulk(es, js_arr)
- js_arr = []
+ if not dryrun:
+ helpers.bulk(es, js_arr)
+ js_arr = []
if len(js_arr) > 0:
- helpers.bulk(es, js_arr)
+ if not dryrun:
+ helpers.bulk(es, js_arr)
print("All done, processed %u docs in %u seconds" % (count, time.time() - then))