[lucene-solr] branch master updated: LUCENE-8827: Speed up poll-mirrors.py

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

janhoy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/master by this push:
     new 6b70bdb  LUCENE-8827: Speed up poll-mirrors.py
6b70bdb is described below

commit 6b70bdb3c0d7331bfe4dfe92b9eb1a71791ab1f4
Author: Jan Høydahl <ja...@apache.org>
AuthorDate: Wed Jun 5 21:22:29 2019 +0200

    LUCENE-8827: Speed up poll-mirrors.py
---
 dev-tools/scripts/poll-mirrors.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/dev-tools/scripts/poll-mirrors.py b/dev-tools/scripts/poll-mirrors.py
old mode 100644
new mode 100755
index e020c61..9a96a75
--- a/dev-tools/scripts/poll-mirrors.py
+++ b/dev-tools/scripts/poll-mirrors.py
@@ -78,16 +78,15 @@ def ftp_file_exists(url):
 
   return len(listing) > 0
 
-def check_url_list(lst):
-  ret = []
-  for url in lst:
-    if mirror_contains_file(url):
-      p('.')
-    else:
-      p('\nFAIL: ' + url + '\n' if args.details else 'X')
-      ret.append(url)
 
-  return ret
+def check_mirror(url):
+  if mirror_contains_file(url):
+    p('.')
+    return None
+  else:
+    p('\nFAIL: ' + url + '\n' if args.details else 'X')
+    return url
+
 
 desc = 'Periodically checks that all Lucene/Solr mirrors contain either a copy of a release or a specified path'
 parser = argparse.ArgumentParser(description=desc)
@@ -142,7 +141,8 @@ while True:
     maven_available = mirror_contains_file(maven_url)
 
   start = time.time()
-  pending_mirrors = check_url_list(pending_mirrors)
+  with Pool(processes=5) as pool:
+    pending_mirrors = list(filter(lambda x: x is not None, pool.map(check_mirror, pending_mirrors)))
   stop = time.time()
   remaining = args.interval - (stop - start)