You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by hu...@apache.org on 2020/09/10 12:45:18 UTC
[incubator-ponymail-foal] branch master updated (19cf33d -> 988e5ea)
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git.
from 19cf33d Add some notes to email docs when migrating
new 3a3d372 simplify
new dc4a620 lint
new 988e5ea forgot to add magic number
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
tools/migrate.py | 55 +++++++++++++++++++++++++++++++------------------------
1 file changed, 31 insertions(+), 24 deletions(-)
[incubator-ponymail-foal] 03/03: forgot to add magic number
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 988e5ea31a533504dc6318a9ddac05262b30fd7a
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Thu Sep 10 14:45:05 2020 +0200
forgot to add magic number
---
tools/migrate.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/tools/migrate.py b/tools/migrate.py
index edab6aa..3968709 100644
--- a/tools/migrate.py
+++ b/tools/migrate.py
@@ -7,6 +7,9 @@ import time
import base64
import hashlib
+# Increment this number whenever breaking changes happen in the migration workflow:
+MIGRATION_MAGIC_NUMBER = "1"
+
# ES connections
es = None
new_es = None
[incubator-ponymail-foal] 02/03: lint
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit dc4a620372c00a541831fbd6f35790ab66d673a8
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Thu Sep 10 14:44:23 2020 +0200
lint
---
tools/migrate.py | 41 ++++++++++++++++++++++-------------------
1 file changed, 22 insertions(+), 19 deletions(-)
diff --git a/tools/migrate.py b/tools/migrate.py
index 7a9e22f..edab6aa 100644
--- a/tools/migrate.py
+++ b/tools/migrate.py
@@ -17,11 +17,11 @@ async def bulk_push(json):
js_arr = []
for entry in json:
bulk_op = {
- "_op_type": "index",
- "_index": entry['index'],
- "_id": entry['id'],
- "_source": entry['body'],
- }
+ "_op_type": "index",
+ "_index": entry['index'],
+ "_id": entry['id'],
+ "_source": entry['body'],
+ }
js_arr.append(
bulk_op
)
@@ -68,10 +68,10 @@ async def main():
bulk_array = []
async for doc in async_scan(
- client=es,
- query={"query": {"match_all": {}}},
- doc_type="mbox",
- index=old_dbname,
+ client=es,
+ query={"query": {"match_all": {}}},
+ doc_type="mbox",
+ index=old_dbname,
):
list_id = doc['_source']['list_raw'].strip("<>")
try:
@@ -111,11 +111,11 @@ async def main():
if isinstance(notes, str):
notes = list(notes)
notes.append("MIGRATE: Document migrated from Pony Mail to Pony Mail Foal at %u, "
- "using foal migrator v/%s" % (now, MIGRATION_MAGIC_NUMBER))
+ "using foal migrator v/%s" % (now, MIGRATION_MAGIC_NUMBER))
# If we re-indexed the document, make a note of that as well.
if do_dkim:
notes.append("REINDEX: Document re-indexed with DKIM_ID at %u, "
- "from %s to %s" % (now, dkim_id, old_id))
+ "from %s to %s" % (now, dkim_id, old_id))
doc['_source']['notes'] = notes
# Copy to new DB
@@ -144,9 +144,10 @@ async def main():
# stringify time left
time_left_str = "%u seconds" % time_left
if time_left > 60:
- time_left_str = "%u minute(s), %u second(s)" % ( int(time_left/60), time_left % 60)
+ time_left_str = "%u minute(s), %u second(s)" % (int(time_left / 60), time_left % 60)
if time_left > 3600:
- time_left_str = "%u hour(s), %u minute(s), %u second(s)" % ( int(time_left/3600), int(time_left%3600/60), time_left % 60)
+ time_left_str = "%u hour(s), %u minute(s), %u second(s)" % (
+ int(time_left / 3600), int(time_left % 3600 / 60), time_left % 60)
print("Processed %u emails, %u remain. ETA: %s (at %u emails per second)" %
(processed, (no_emails - processed), time_left_str, docs_per_second)
@@ -158,10 +159,10 @@ async def main():
no_att = count['count']
print("Transferring %u attachments..." % no_att)
async for doc in async_scan(
- client=es,
- query={"query": {"match_all": {}}},
- doc_type="attachment",
- index=old_dbname,
+ client=es,
+ query={"query": {"match_all": {}}},
+ doc_type="attachment",
+ index=old_dbname,
):
# Copy to new DB
await new_es.index(index=dbname_attachment, doc_type='_doc', id=doc['_id'], body=doc['_source'])
@@ -176,9 +177,10 @@ async def main():
# stringify time left
time_left_str = "%u seconds" % time_left
if time_left > 60:
- time_left_str = "%u minute(s), %u second(s)" % ( int(time_left/60), time_left % 60)
+ time_left_str = "%u minute(s), %u second(s)" % (int(time_left / 60), time_left % 60)
if time_left > 3600:
- time_left_str = "%u hour(s), %u minute(s), %u second(s)" % ( int(time_left/3600), int(time_left%3600/60), time_left % 60)
+ time_left_str = "%u hour(s), %u minute(s), %u second(s)" % (
+ int(time_left / 3600), int(time_left % 3600 / 60), time_left % 60)
print("Processed %u emails, %u remain. ETA: %s (at %u attachments per second)" %
(processed, (no_att - processed), time_left_str, docs_per_second)
@@ -188,5 +190,6 @@ async def main():
await new_es.close()
print("All done, enjoy!")
+
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
[incubator-ponymail-foal] 01/03: simplify
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 3a3d37250f7c197343ac6a56c3e9103a44b64070
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Thu Sep 10 14:44:11 2020 +0200
simplify
---
tools/migrate.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/tools/migrate.py b/tools/migrate.py
index a6620ee..7a9e22f 100644
--- a/tools/migrate.py
+++ b/tools/migrate.py
@@ -106,16 +106,17 @@ async def main():
doc['_source']['dbid'] = hashlib.sha3_256(source_text).hexdigest()
# Append migration details to notes field in doc
- doc['_source']['notes'] = doc['_source'].get('notes', [])
+ notes = doc['_source'].get('notes', [])
# We want a list, not a single string
- if isinstance(doc['_source']['notes'], str):
- doc['_source']['notes'] = [doc['_source']['notes']]
- doc['_source']['notes'].append("MIGRATE: Document migrated from Pony Mail to Pony Mail Foal at %u, "
+ if isinstance(notes, str):
+ notes = list(notes)
+ notes.append("MIGRATE: Document migrated from Pony Mail to Pony Mail Foal at %u, "
"using foal migrator v/%s" % (now, MIGRATION_MAGIC_NUMBER))
# If we re-indexed the document, make a note of that as well.
if do_dkim:
- doc['_source']['notes'].append("REINDEX: Document re-indexed with DKIM_ID at %u, "
+ notes.append("REINDEX: Document re-indexed with DKIM_ID at %u, "
"from %s to %s" % (now, dkim_id, old_id))
+ doc['_source']['notes'] = notes
# Copy to new DB
bulk_array.append({