You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by se...@apache.org on 2018/05/27 00:28:42 UTC
[incubator-ponymail] branch master updated (0e7e6b1 -> b21c3be)
This is an automated email from the ASF dual-hosted git repository.
sebb pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail.git.
from 0e7e6b1 Simplify args processing
new 94cbe51 Simplify args processing
new 2de2c6d Unnecessary check - always true
new b21c3be Mid is not yet implemented
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
tools/missing.py | 40 +++++++++++++++++++++-------------------
1 file changed, 21 insertions(+), 19 deletions(-)
--
To stop receiving notification emails like this one, please contact
sebb@apache.org.
[incubator-ponymail] 03/03: Mid is not yet implemented
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail.git
commit b21c3befe36b95c2791061b842572f9d98d39c33
Author: Sebb <se...@apache.org>
AuthorDate: Sun May 27 01:28:37 2018 +0100
Mid is not yet implemented
---
tools/missing.py | 104 +++++++++++++++++++++++++++++--------------------------
1 file changed, 54 insertions(+), 50 deletions(-)
diff --git a/tools/missing.py b/tools/missing.py
index 6e74b44..88588b0 100755
--- a/tools/missing.py
+++ b/tools/missing.py
@@ -77,65 +77,69 @@ count = 0
scroll = '30m'
then = time.time()
elastic = Elastic()
-sourceLID = ("%s" if args.notag else "<%s>") % args.source.replace("@", ".").strip("<>")
-page = elastic.scan(# defaults to mbox
- scroll = scroll,
- body = {
- "_source" : ['subject','message-id'],
- "query" : {
- "bool" : {
- "must" : {
- 'wildcard' if args.wildcard else 'term': {
- 'list_raw': sourceLID
+if args.source:
+ sourceLID = ("%s" if args.notag else "<%s>") % args.source.replace("@", ".").strip("<>")
+ page = elastic.scan(# defaults to mbox
+ scroll = scroll,
+ body = {
+ "_source" : ['subject','message-id'],
+ "query" : {
+ "bool" : {
+ "must" : {
+ 'wildcard' if args.wildcard else 'term': {
+ 'list_raw': sourceLID
+ }
+ },
+ # missing is not supported in ES 5.x
+ "must_not": {
+ "exists" : {
+ "field" : field
}
- },
- # missing is not supported in ES 5.x
- "must_not": {
- "exists" : {
- "field" : field
}
}
}
}
- }
- )
-print(page)
-sid = page['_scroll_id']
-scroll_size = page['hits']['total']
-print("Found %d matches" % scroll_size)
-if args.debug:
+ )
print(page)
-js_arr = []
-while (scroll_size > 0):
- page = elastic.scroll(scroll_id = sid, scroll = scroll)
+ sid = page['_scroll_id']
+ scroll_size = page['hits']['total']
+ print("Found %d matches" % scroll_size)
if args.debug:
print(page)
- sid = page['_scroll_id']
- scroll_size = len(page['hits']['hits'])
- for hit in page['hits']['hits']:
- doc = hit['_id']
- body = {}
- if setField:
- body[field] = value
- js_arr.append({
- '_op_type': 'update',
- '_index': elastic.dbname,
- '_type': 'mbox',
- '_id': doc,
- 'doc': body
- })
- count += 1
- source = hit['_source']
- print("Id: %s Msg-id: %s Subject: %s" %(doc, getField(source, 'message-id'), getField(source,'subject')))
- if (count % 500 == 0):
- print("Processed %u emails..." % count)
+ js_arr = []
+ while (scroll_size > 0):
+ page = elastic.scroll(scroll_id = sid, scroll = scroll)
+ if args.debug:
+ print(page)
+ sid = page['_scroll_id']
+ scroll_size = len(page['hits']['hits'])
+ for hit in page['hits']['hits']:
+ doc = hit['_id']
+ body = {}
if setField:
- update(elastic, js_arr)
- js_arr = []
+ body[field] = value
+ js_arr.append({
+ '_op_type': 'update',
+ '_index': elastic.dbname,
+ '_type': 'mbox',
+ '_id': doc,
+ 'doc': body
+ })
+ count += 1
+ source = hit['_source']
+ print("Id: %s Msg-id: %s Subject: %s" %(doc, getField(source, 'message-id'), getField(source,'subject')))
+ if (count % 500 == 0):
+ print("Processed %u emails..." % count)
+ if setField:
+ update(elastic, js_arr)
+ js_arr = []
+
+ print("Processed %u emails." % count)
+ if len(js_arr) > 0:
+ if setField:
+ update(elastic, js_arr)
-print("Processed %u emails." % count)
-if len(js_arr) > 0:
- if setField:
- update(elastic, js_arr)
+if args.mid:
+ parser.error("--mid: not yet implemented")
print("All done, processed %u docs in %u seconds" % (count, time.time() - then))
--
To stop receiving notification emails like this one, please contact
sebb@apache.org.
[incubator-ponymail] 01/03: Simplify args processing
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail.git
commit 94cbe515bba25857a549d681e4674a52698d1ce2
Author: Sebb <se...@apache.org>
AuthorDate: Sun May 27 01:18:55 2018 +0100
Simplify args processing
Don't use nargs=1
Assign values directly
---
tools/missing.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/tools/missing.py b/tools/missing.py
index dfaa274..8c55f5c 100755
--- a/tools/missing.py
+++ b/tools/missing.py
@@ -26,9 +26,9 @@ from elastic import Elastic
parser = argparse.ArgumentParser(description='Command line options.')
# Cannot have both source and mid as input
source_group = parser.add_mutually_exclusive_group(required=True)
-source_group.add_argument('--source', dest='source', type=str, nargs=1, metavar='list-name',
+source_group.add_argument('--source', dest='source', type=str, metavar='list-name',
help='Source list to edit')
-source_group.add_argument('--mid', dest='mid', type=str, nargs=1, metavar='message-id',
+source_group.add_argument('--mid', dest='mid', type=str, metavar='message-id',
help='Source Message-ID to edit')
action_group = parser.add_mutually_exclusive_group(required=True)
@@ -79,7 +79,7 @@ if args.missing:
scroll = '30m'
then = time.time()
elastic = Elastic()
- sourceLID = ("%s" if args.notag else "<%s>") % args.source[0].replace("@", ".").strip("<>")
+ sourceLID = ("%s" if args.notag else "<%s>") % args.source.replace("@", ".").strip("<>")
page = elastic.scan(# defaults to mbox
scroll = scroll,
body = {
--
To stop receiving notification emails like this one, please contact
sebb@apache.org.
[incubator-ponymail] 02/03: Unnecessary check - always true
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail.git
commit 2de2c6d9ac9e01830a22c5950e6b8b2109fad49b
Author: Sebb <se...@apache.org>
AuthorDate: Sun May 27 01:21:02 2018 +0100
Unnecessary check - always true
---
tools/missing.py | 128 +++++++++++++++++++++++++++----------------------------
1 file changed, 63 insertions(+), 65 deletions(-)
diff --git a/tools/missing.py b/tools/missing.py
index 8c55f5c..6e74b44 100755
--- a/tools/missing.py
+++ b/tools/missing.py
@@ -65,79 +65,77 @@ def update(elastic, js_arr):
if not args.test:
elastic.bulk(js_arr)
-if args.missing:
- setField = len(args.missing) > 1
- field = args.missing[0]
- value = None
- if setField:
- value = args.missing[1]
- if setField:
- print("Set missing/null field %s to '%s'" %(field, value))
- else:
- print("List missing/null field %s" % field)
- count = 0
- scroll = '30m'
- then = time.time()
- elastic = Elastic()
- sourceLID = ("%s" if args.notag else "<%s>") % args.source.replace("@", ".").strip("<>")
- page = elastic.scan(# defaults to mbox
- scroll = scroll,
- body = {
- "_source" : ['subject','message-id'],
- "query" : {
- "bool" : {
- "must" : {
- 'wildcard' if args.wildcard else 'term': {
- 'list_raw': sourceLID
- }
- },
- # missing is not supported in ES 5.x
- "must_not": {
- "exists" : {
- "field" : field
+setField = len(args.missing) > 1
+field = args.missing[0]
+value = None
+if setField:
+ value = args.missing[1]
+ print("Set missing/null field %s to '%s'" %(field, value))
+else:
+ print("List missing/null field %s" % field)
+count = 0
+scroll = '30m'
+then = time.time()
+elastic = Elastic()
+sourceLID = ("%s" if args.notag else "<%s>") % args.source.replace("@", ".").strip("<>")
+page = elastic.scan(# defaults to mbox
+ scroll = scroll,
+ body = {
+ "_source" : ['subject','message-id'],
+ "query" : {
+ "bool" : {
+ "must" : {
+ 'wildcard' if args.wildcard else 'term': {
+ 'list_raw': sourceLID
}
+ },
+ # missing is not supported in ES 5.x
+ "must_not": {
+ "exists" : {
+ "field" : field
}
}
}
}
- )
+ }
+ )
+print(page)
+sid = page['_scroll_id']
+scroll_size = page['hits']['total']
+print("Found %d matches" % scroll_size)
+if args.debug:
print(page)
- sid = page['_scroll_id']
- scroll_size = page['hits']['total']
- print("Found %d matches" % scroll_size)
+js_arr = []
+while (scroll_size > 0):
+ page = elastic.scroll(scroll_id = sid, scroll = scroll)
if args.debug:
print(page)
- js_arr = []
- while (scroll_size > 0):
- page = elastic.scroll(scroll_id = sid, scroll = scroll)
- if args.debug:
- print(page)
- sid = page['_scroll_id']
- scroll_size = len(page['hits']['hits'])
- for hit in page['hits']['hits']:
- doc = hit['_id']
- body = {}
+ sid = page['_scroll_id']
+ scroll_size = len(page['hits']['hits'])
+ for hit in page['hits']['hits']:
+ doc = hit['_id']
+ body = {}
+ if setField:
+ body[field] = value
+ js_arr.append({
+ '_op_type': 'update',
+ '_index': elastic.dbname,
+ '_type': 'mbox',
+ '_id': doc,
+ 'doc': body
+ })
+ count += 1
+ source = hit['_source']
+ print("Id: %s Msg-id: %s Subject: %s" %(doc, getField(source, 'message-id'), getField(source,'subject')))
+ if (count % 500 == 0):
+ print("Processed %u emails..." % count)
if setField:
- body[field] = value
- js_arr.append({
- '_op_type': 'update',
- '_index': elastic.dbname,
- '_type': 'mbox',
- '_id': doc,
- 'doc': body
- })
- count += 1
- source = hit['_source']
- print("Id: %s Msg-id: %s Subject: %s" %(doc, getField(source, 'message-id'), getField(source,'subject')))
- if (count % 500 == 0):
- print("Processed %u emails..." % count)
- if setField:
- update(elastic, js_arr)
- js_arr = []
+ update(elastic, js_arr)
+ js_arr = []
- print("Processed %u emails." % count)
- if len(js_arr) > 0:
- if setField:
- update(elastic, js_arr)
+print("Processed %u emails." % count)
+if len(js_arr) > 0:
+ if setField:
+ update(elastic, js_arr)
- print("All done, processed %u docs in %u seconds" % (count, time.time() - then))
+print("All done, processed %u docs in %u seconds" % (count, time.time() - then))
--
To stop receiving notification emails like this one, please contact
sebb@apache.org.