You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2025/03/02 19:09:48 UTC
(lucene) branch branch_10x updated: python scripts: enable standard type checking (#14321)
This is an automated email from the ASF dual-hosted git repository.
rmuir pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new b7df84697c9 python scripts: enable standard type checking (#14321)
b7df84697c9 is described below
commit b7df84697c9404b87236c4144bb83e6616b43c2f
Author: Robert Muir <rm...@apache.org>
AuthorDate: Sun Mar 2 14:08:59 2025 -0500
python scripts: enable standard type checking (#14321)
Fix all the type issues, enable checking so that no more creep in.
A couple third-party libraries have some issues (such as totally wrong typing), I just disabled in that case.
Fixes are mostly basic stuff such as adding missing null checks, occasional cast or type-hint.
---
dev-tools/scripts/addBackcompatIndexes.py | 24 ++++----
dev-tools/scripts/addVersion.py | 12 ++--
dev-tools/scripts/buildAndPushRelease.py | 6 +-
dev-tools/scripts/create_line_file_docs.py | 20 +++----
dev-tools/scripts/diff_lucene_changes.py | 6 +-
dev-tools/scripts/githubPRs.py | 17 ++++--
dev-tools/scripts/pyproject.toml | 11 +---
dev-tools/scripts/releaseWizard.py | 86 +++++++++++++++------------
dev-tools/scripts/reproduceJenkinsFailures.py | 2 +
dev-tools/scripts/scriptutil.py | 14 +++--
dev-tools/scripts/smokeTestRelease.py | 15 +++--
11 files changed, 118 insertions(+), 95 deletions(-)
diff --git a/dev-tools/scripts/addBackcompatIndexes.py b/dev-tools/scripts/addBackcompatIndexes.py
index 3056c8268d4..e508ae0c8da 100755
--- a/dev-tools/scripts/addBackcompatIndexes.py
+++ b/dev-tools/scripts/addBackcompatIndexes.py
@@ -17,7 +17,7 @@
# For usage information, see:
-#
+#
# http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes
@@ -49,7 +49,7 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
filename = '%s.%s-%s.zip' % (prefix, index_version, indextype)
else:
filename = '%s.%s.zip' % (prefix, index_version)
-
+
print(' creating %s...' % filename, end='', flush=True)
module = 'backward-codecs'
index_dir = os.path.join('lucene', module, 'src/test/org/apache/lucene/backward_index')
@@ -76,7 +76,7 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
])
base_dir = os.getcwd()
bc_index_file = os.path.join(temp_dir, filename)
-
+
if os.path.exists(bc_index_file):
print('alreadyexists')
else:
@@ -85,7 +85,7 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
if not os.path.exists(bc_index_file):
raise Exception("Expected file can't be found: %s" %bc_index_file)
print('done')
-
+
print(' adding %s...' % filename, end='', flush=True)
scriptutil.run('cp %s %s' % (bc_index_file, os.path.join(base_dir, index_dir)))
os.chdir(base_dir)
@@ -125,7 +125,7 @@ def update_backcompat_tests(index_version, current_version):
buffer.append('\n')
buffer.append(('%s\n') % index_version)
return True
-
+
changed = scriptutil.update_file(filename, re.compile(r'.*'), edit, append)
print('done' if changed else 'uptodate')
@@ -139,7 +139,7 @@ def download_from_cdn(version, remotename, localname):
try:
urllib.request.urlretrieve(url, localname)
return True
- except urllib.error.URLError as e:
+ except urllib.error.HTTPError as e:
if e.code == 404:
return False
raise e
@@ -149,14 +149,14 @@ def download_from_archives(version, remotename, localname):
try:
urllib.request.urlretrieve(url, localname)
return True
- except urllib.error.URLError as e:
+ except urllib.error.HTTPError as e:
if e.code == 404:
return False
raise e
def download_release(version, temp_dir, force):
print(' downloading %s source release...' % version, end='', flush=True)
- source = os.path.join(temp_dir, 'lucene-%s' % version)
+ source = os.path.join(temp_dir, 'lucene-%s' % version)
if os.path.exists(source):
if force:
shutil.rmtree(source)
@@ -173,7 +173,7 @@ def download_release(version, temp_dir, force):
olddir = os.getcwd()
os.chdir(temp_dir)
scriptutil.run('tar -xvzf %s' % source_tgz)
- os.chdir(olddir)
+ os.chdir(olddir)
print('done')
return source
@@ -195,9 +195,9 @@ http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes
c = parser.parse_args()
return c
-
+
def main():
- c = read_config()
+ c = read_config()
if not os.path.exists(c.temp_dir):
os.makedirs(c.temp_dir)
@@ -216,7 +216,7 @@ def main():
create_and_add_index(source, 'dvupdates', c.version, current_version, c.temp_dir)
create_and_add_index(source, 'emptyIndex', c.version, current_version, c.temp_dir)
print ('\nMANUAL UPDATE REQUIRED: edit TestGenerateBwcIndices to enable moreterms, dvupdates, and empty index testing')
-
+
print('\nAdding backwards compatibility tests')
update_backcompat_tests(c.version, current_version)
diff --git a/dev-tools/scripts/addVersion.py b/dev-tools/scripts/addVersion.py
index 41ef19e0107..bc692825458 100755
--- a/dev-tools/scripts/addVersion.py
+++ b/dev-tools/scripts/addVersion.py
@@ -37,7 +37,7 @@ def update_changes(filename, new_version, init_changes, headers):
buffer.append('%s\n---------------------\n(No changes)\n\n' % header)
buffer.append(line)
return match is not None
-
+
changed = update_file(filename, matcher, edit)
print('done' if changed else 'uptodate')
@@ -53,7 +53,7 @@ def add_constant(new_version, deprecate):
if last.strip() != '@Deprecated':
spaces = ' ' * (len(last) - len(last.lstrip()) - 1)
del buffer[-1] # Remove comment closer line
- if (len(buffer) >= 4 and re.search('for Lucene.\s*$', buffer[-1]) is not None):
+ if (len(buffer) >= 4 and re.search(r'for Lucene.\s*$', buffer[-1]) is not None):
del buffer[-3:] # drop the trailing lines '<p> / Use this to get the latest ... / ... for Lucene.'
buffer.append(( '{0} * @deprecated ({1}) Use latest\n'
+ '{0} */\n'
@@ -73,7 +73,7 @@ def add_constant(new_version, deprecate):
buffer.append('%s@Deprecated\n' % spaces)
buffer.append('{0}public static final Version {1} = new Version({2}, {3}, {4});\n'.format
(spaces, new_version.constant, new_version.major, new_version.minor, new_version.bugfix))
-
+
class Edit(object):
found = -1
def __call__(self, buffer, match, line):
@@ -97,14 +97,14 @@ def add_constant(new_version, deprecate):
buffer.append(line)
return False
-
+
changed = update_file(filename, matcher, Edit())
print('done' if changed else 'uptodate')
def update_build_version(new_version):
print(' changing baseVersion...', end='', flush=True)
filename = 'build.gradle'
- def edit(buffer, match, line):
+ def edit(buffer, _, line):
if new_version.dot in line:
return None
buffer.append(' String baseVersion = \'' + new_version.dot + '\'\n')
@@ -118,7 +118,7 @@ def update_latest_constant(new_version):
print(' changing Version.LATEST to %s...' % new_version.constant, end='', flush=True)
filename = 'lucene/core/src/java/org/apache/lucene/util/Version.java'
matcher = re.compile('public static final Version LATEST')
- def edit(buffer, match, line):
+ def edit(buffer, _, line):
if new_version.constant in line:
return None
buffer.append(line.rpartition('=')[0] + ('= %s;\n' % new_version.constant))
diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py
index 50050800cfc..3e41c1a9bbd 100755
--- a/dev-tools/scripts/buildAndPushRelease.py
+++ b/dev-tools/scripts/buildAndPushRelease.py
@@ -51,6 +51,8 @@ def runAndSendGPGPassword(command, password):
p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
f = open(LOG, 'ab')
while True:
+ assert p.stdout
+ assert p.stdin
p.stdout.flush()
line = p.stdout.readline()
if len(line) == 0:
@@ -176,8 +178,7 @@ def checkDOAPfiles(version):
treeRoot = ET.parse(doapFile).getroot()
doapRevisions = set()
for revision in treeRoot.findall(xpathRevision):
- match = reDoapRevision.match(revision.text)
- if (match is not None):
+ if (revision.text and (match := reDoapRevision.match(revision.text))):
if (match.group(1) not in ('0', '1', '2')): # Ignore 0.X, 1.X and 2.X revisions
doapRevisions.add(normalizeVersion(match.groups()))
else:
@@ -412,6 +413,7 @@ def main():
print('Next run the smoker tester:')
p = re.compile(".*/")
m = p.match(sys.argv[0])
+ assert m
if not c.sign:
signed = "--not-signed"
else:
diff --git a/dev-tools/scripts/create_line_file_docs.py b/dev-tools/scripts/create_line_file_docs.py
index 55417d4634e..9fe53cdda4b 100644
--- a/dev-tools/scripts/create_line_file_docs.py
+++ b/dev-tools/scripts/create_line_file_docs.py
@@ -61,7 +61,7 @@ def compress_with_seek_points(file_name_in, file_name_out, num_seek_points):
break
bytes_in_chunk += len(line)
- f_out.write(line)
+ f_out.write(line) # false positive in python's crazy typing # pyright: ignore[reportArgumentType]
if bytes_in_chunk > bytes_per_chunk and chunk_count < num_seek_points:
f_out.close()
@@ -72,12 +72,12 @@ def compress_with_seek_points(file_name_in, file_name_out, num_seek_points):
for seek_point in seek_points:
f_out.write('%d\n' % seek_point)
-re_tag = re.compile('<[^>]+?>')
-re_newlines = re.compile('\n+')
-re_space = re.compile('\s')
+re_tag = re.compile(r'<[^>]+?>')
+re_newlines = re.compile(r'\n+')
+re_space = re.compile(r'\s')
# used to find word break, for splitting docs into ~1 KB sized smaller docs:
-re_next_non_word_character = re.compile('\W', re.U)
+re_next_non_word_character = re.compile(r'\W', re.U)
EUROPARL_V7_URL = 'https://www.statmt.org/europarl/v7/europarl.tgz'
@@ -101,7 +101,7 @@ def split_docs(all_out, title_string, date_string, body_string):
char_count = len(body_string)
body_string_fragment = body_string[:char_count].strip()
-
+
#print('write title %d, body %d' % (len(title_string), len(body_string_fragment)))
all_out.write('%s\t%s\t%s\n' % (title_string, date_string, body_string_fragment))
body_string = body_string[char_count:]
@@ -143,7 +143,7 @@ def sample_europarl():
next_print_time = start_time + 3
# normalize text a bit and concatenate all lines into single file, counting total lines/bytes
with open(all_txt_file_name, 'w', encoding='utf-8') as all_out:
- for dir_path, dir_names, file_names in os.walk('%s/txt' % tmp_dir_path):
+ for dir_path, _, file_names in os.walk('%s/txt' % tmp_dir_path):
for file_name in file_names:
if file_name.endswith('.txt'):
file_count += 1
@@ -155,7 +155,7 @@ def sample_europarl():
year = 2000 + year
date_string = '%04d-%02d-%02d' % (year, month, day)
-
+
# unfortunately we need errors='ignore' since in Europarl v7, one file (pl/ep-09-10-22-009.txt) has invalid utf-8:
chapter_count = 0
with open('%s/%s' % (dir_path, file_name), 'r', encoding='utf-8', errors='ignore') as f_in:
@@ -176,7 +176,7 @@ def sample_europarl():
doc_count += split_docs(all_out, last_title, date_string, s)
else:
skip_count += 1
-
+
last_text = []
chapter_count += 1
while True:
@@ -248,7 +248,7 @@ def sample_europarl():
compress_with_seek_points(file_name_out,
file_name_out + '.gz',
mb)
-
+
finally:
print('Removing tmp dir "%s"...' % tmp_dir_path)
if not DEBUG:
diff --git a/dev-tools/scripts/diff_lucene_changes.py b/dev-tools/scripts/diff_lucene_changes.py
index 91b7c84994d..b6a3f6870b7 100644
--- a/dev-tools/scripts/diff_lucene_changes.py
+++ b/dev-tools/scripts/diff_lucene_changes.py
@@ -42,8 +42,10 @@ def get_changes_url(branch_name):
return url
def extract_release_section(changes_txt, release_name):
- return re.search(f'=======+ Lucene {re.escape(release_name)} =======+(.*?)=======+ Lucene .*? =======+$',
- changes_txt.decode('utf-8'), re.MULTILINE | re.DOTALL).group(1).encode('utf-8')
+ match = re.search(f'=======+ Lucene {re.escape(release_name)} =======+(.*?)=======+ Lucene .*? =======+$',
+ changes_txt.decode('utf-8'), re.MULTILINE | re.DOTALL)
+ assert match
+ return match.group(1).encode('utf-8')
def main():
if len(sys.argv) < 3 or len(sys.argv) > 5:
diff --git a/dev-tools/scripts/githubPRs.py b/dev-tools/scripts/githubPRs.py
index d55f916d00a..9390e947048 100755
--- a/dev-tools/scripts/githubPRs.py
+++ b/dev-tools/scripts/githubPRs.py
@@ -26,8 +26,10 @@ sys.path.append(os.path.dirname(__file__))
import argparse
import json
import re
+from typing import cast
from github import Github
-from jira import JIRA
+from jira import JIRA, Issue
+from jira.client import ResultList
from jinja2 import Environment, BaseLoader
def read_config():
@@ -46,7 +48,7 @@ def out(text):
def make_html(dict):
global conf
- template = Environment(loader=BaseLoader).from_string("""
+ template = Environment(loader=BaseLoader()).from_string("""
<h1>Lucene Github PR report</h1>
<p>Number of open Pull Requests: {{ open_count }}</p>
@@ -75,7 +77,7 @@ def main():
gh = Github(token)
else:
gh = Github()
- jira = JIRA('https://issues.apache.org/jira')
+ jira = JIRA('https://issues.apache.org/jira') # this ctor has broken types in jira library. # pyright: ignore[reportArgumentType]
result = {}
repo = gh.get_repo('apache/lucene')
open_prs = repo.get_pulls(state='open')
@@ -100,19 +102,22 @@ def main():
issue_ids = []
issue_to_pr = {}
for pr in has_jira:
- jira_issue_str = re.match(r'.*\b((LUCENE)-\d{3,6})\b', pr.title).group(1)
+ match = re.match(r'.*\b((LUCENE)-\d{3,6})\b', pr.title)
+ assert match
+ jira_issue_str = match.group(1)
issue_ids.append(jira_issue_str)
issue_to_pr[jira_issue_str] = pr
- resolved_jiras = jira.search_issues(jql_str="key in (%s) AND status in ('Closed', 'Resolved')" % ", ".join(issue_ids))
+ resolved_jiras = cast(ResultList[Issue], jira.search_issues(jql_str="key in (%s) AND status in ('Closed', 'Resolved')" % ", ".join(issue_ids)))
closed_jiras = []
for issue in resolved_jiras:
pr_title = issue_to_pr[issue.key].title
pr_number = issue_to_pr[issue.key].number
assignee = issue.fields.assignee.name if issue.fields.assignee else None
+ resolution = issue.fields.resolution.name if issue.fields.resolution else None
closed_jiras.append({ 'issue_key': issue.key,
'status': issue.fields.status.name,
- 'resolution': issue.fields.resolution.name,
+ 'resolution': resolution,
'resolution_date': issue.fields.resolutiondate[:10],
'pr_number': pr_number,
'pr_title': pr_title,
diff --git a/dev-tools/scripts/pyproject.toml b/dev-tools/scripts/pyproject.toml
index 2349f77d09b..bc9a970c12f 100644
--- a/dev-tools/scripts/pyproject.toml
+++ b/dev-tools/scripts/pyproject.toml
@@ -4,16 +4,7 @@ venv = ".venv"
# TODO: improve!
# typeCheckingMode = "strict"
reportUnnecessaryTypeIgnoreComment = "error"
-typeCheckingMode = "basic"
-# TODO: we should fix these
-reportArgumentType = "none"
-reportAttributeAccessIssue = "none"
-reportCallIssue = "none"
-reportInvalidStringEscapeSequence = "none"
-reportOperatorIssue = "none"
-reportOptionalIterable = "none"
-reportOptionalMemberAccess = "none"
-reportOptionalSubscript = "none"
+typeCheckingMode = "standard"
[tool.ruff]
line-length = 200
diff --git a/dev-tools/scripts/releaseWizard.py b/dev-tools/scripts/releaseWizard.py
index d40e8d9ccc3..3c35b6dd66a 100755
--- a/dev-tools/scripts/releaseWizard.py
+++ b/dev-tools/scripts/releaseWizard.py
@@ -45,13 +45,14 @@ import subprocess
import sys
import textwrap
import time
-import urllib
+import urllib.request
from collections import OrderedDict
from datetime import datetime
from datetime import timedelta
from datetime import timezone
+from typing import cast
-import holidays
+from holidays import country_holidays
import yaml
from ics import Calendar, Event
from jinja2 import Environment
@@ -135,12 +136,11 @@ def expand_jinja(text, vars=None):
def replace_templates(text):
tpl_lines = []
for line in text.splitlines():
- if line.startswith("(( template="):
- match = re.search(r"^\(\( template=(.+?) \)\)", line)
- name = match.group(1)
- tpl_lines.append(replace_templates(templates[name].strip()))
- else:
- tpl_lines.append(line)
+ if match := re.search(r"^\(\( template=(.+?) \)\)", line):
+ name = match.group(1)
+ tpl_lines.append(replace_templates(templates[name].strip()))
+ else:
+ tpl_lines.append(line)
return "\n".join(tpl_lines)
def getScriptVersion():
@@ -270,8 +270,8 @@ class ReleaseState:
def __init__(self, config_path, release_version, script_version):
self.script_version = script_version
self.config_path = config_path
- self.todo_groups = None
- self.todos = None
+ self.todo_groups: list[TodoGroup] = []
+ self.todos: dict[str,Todo] = {}
self.latest_version = None
self.previous_rcs = {}
self.rc_number = 1
@@ -301,10 +301,13 @@ class ReleaseState:
self.release_type = 'bugfix'
def is_released(self):
- return self.get_todo_by_id('announce_lucene').is_done()
+ todo = self.get_todo_by_id('announce_lucene')
+ assert todo
+ return todo.is_done()
def get_gpg_key(self):
gpg_task = self.get_todo_by_id('gpg')
+ assert gpg_task
if gpg_task.is_done():
return gpg_task.get_state()['gpg_key']
else:
@@ -312,6 +315,7 @@ class ReleaseState:
def get_release_date(self):
publish_task = self.get_todo_by_id('publish_maven')
+ assert publish_task
if publish_task.is_done():
return unix_to_datetime(publish_task.get_state()['done_date'])
else:
@@ -587,7 +591,7 @@ class ReleaseState:
java_ver = java_versions[v.major]
java_home_var = "JAVA%s_HOME" % java_ver
if java_home_var in os.environ:
- return os.environ.get(java_home_var)
+ return os.environ[java_home_var]
else:
raise Exception("Script needs environment variable %s" % java_home_var )
@@ -674,7 +678,7 @@ class TodoGroup(SecretYamlObject):
else:
return None
- def get_subtitle(self):
+ def get_subtitle(self) -> str:
if self.depends:
ret_str = ""
for dep in ensure_list(self.depends):
@@ -684,7 +688,7 @@ class TodoGroup(SecretYamlObject):
if g and not g.is_done():
ret_str += "NOTE: Please first complete '%s'\n" % g.title
return ret_str.strip()
- return None
+ return ""
class Todo(SecretYamlObject):
@@ -714,9 +718,9 @@ class Todo(SecretYamlObject):
for t in self.types:
if t not in ['minor', 'major', 'bugfix']:
sys.exit("Wrong Todo config for '%s'. Type needs to be either 'minor', 'major' or 'bugfix'" % self.id)
- if commands:
+ if self.commands:
self.commands.todo_id = self.id
- for c in commands.commands:
+ for c in self.commands.commands:
c.todo_id = self.id
@classmethod
@@ -766,6 +770,7 @@ class Todo(SecretYamlObject):
g = state.get_group_by_id(dep)
if not g:
g = state.get_todo_by_id(dep)
+ assert g
if not g.is_done():
print("This step depends on '%s'. Please complete that first\n" % g.title)
return
@@ -815,7 +820,7 @@ class Todo(SecretYamlObject):
print("ERROR while executing todo %s (%s)" % (self.get_title(), e))
def get_menu_item(self):
- return FunctionItem(self.get_title, self.display_and_confirm)
+ return FunctionItem(self.get_title(), self.display_and_confirm)
def clone(self):
clone = Todo(self.id, self.title, description=self.description)
@@ -937,7 +942,7 @@ def unix_to_datetime(unix_stamp):
def generate_asciidoc():
base_filename = os.path.join(state.get_release_folder(),
"lucene_release_%s"
- % (state.release_version.replace("\.", "_")))
+ % (state.release_version.replace(r"\.", "_")))
filename_adoc = "%s.adoc" % base_filename
filename_html = "%s.html" % base_filename
@@ -1027,7 +1032,7 @@ def load_rc():
with open(lucenerc, 'r') as fp:
return json.load(fp)
except Exception:
- return None
+ return {}
def store_rc(release_root, release_version=None):
@@ -1061,7 +1066,7 @@ def keys_downloaded():
def dump_yaml():
file = open(os.path.join(script_path, "releaseWizard.yaml"), "w")
yaml.add_representer(str, str_presenter)
- yaml.Dumper.ignore_aliases = lambda *args : True
+ yaml.Dumper.ignore_aliases = lambda self, data : True
dump_obj = {'templates': templates,
'groups': state.todo_groups}
yaml.dump(dump_obj, width=180, stream=file, sort_keys=False, default_flow_style=False)
@@ -1230,7 +1235,7 @@ def pause(fun=None):
class CustomExitItem(ExitItem):
- def show(self, index):
+ def show(self, index, available_width=None):
return super(CustomExitItem, self).show(index)
def get_return(self):
@@ -1277,7 +1282,7 @@ def main():
else:
conf = load_rc()
release_root = conf['root']
- if 'release_version' in conf:
+ if conf and 'release_version' in conf:
release_version = conf['release_version']
else:
release_version = get_release_version()
@@ -1326,7 +1331,7 @@ def main():
todo_menu.append_item(menu_item)
main_menu.append_item(SubmenuItem(get_todo_menuitem_title, todo_menu, menu=main_menu))
- main_menu.append_item(FunctionItem(get_start_new_rc_menu_title, start_new_rc))
+ main_menu.append_item(FunctionItem(get_start_new_rc_menu_title(), start_new_rc))
main_menu.append_item(FunctionItem('Clear and restart current RC', state.clear_rc))
main_menu.append_item(FunctionItem("Clear all state, restart the %s release" % state.release_version, reset_state))
main_menu.append_item(FunctionItem('Start release for a different version', release_other_version))
@@ -1425,9 +1430,11 @@ def run_follow(command, cwd=None, fh=sys.stdout, tee=False, live=False, shell=No
universal_newlines=True, bufsize=0, close_fds=True, shell=doShell)
lines_written = 0
+ assert process.stdout
fl = fcntl.fcntl(process.stdout, fcntl.F_GETFL)
fcntl.fcntl(process.stdout, fcntl.F_SETFL, fl | os.O_NONBLOCK)
+ assert process.stderr
flerr = fcntl.fcntl(process.stderr, fcntl.F_GETFL)
fcntl.fcntl(process.stderr, fcntl.F_SETFL, flerr | os.O_NONBLOCK)
@@ -1506,7 +1513,7 @@ class Commands(SecretYamlObject):
yaml_tag = u'!Commands'
hidden_fields = ['todo_id']
cmd_continuation_char = "^" if is_windows() else "\\"
- def __init__(self, root_folder, commands_text=None, commands=None, logs_prefix=None, run_text=None, enable_execute=None,
+ def __init__(self, root_folder, commands_text=None, commands=[], logs_prefix=None, run_text=None, enable_execute=None,
confirm_each_command=None, env=None, vars=None, todo_id=None, remove_files=None):
self.root_folder = root_folder
self.commands_text = commands_text
@@ -1535,7 +1542,7 @@ class Commands(SecretYamlObject):
if self.env:
for key in self.env:
val = self.jinjaify(self.env[key])
- os.environ[key] = val
+ os.environ[key] = str(val)
if is_windows():
print("\n SET %s=%s" % (key, val))
else:
@@ -1646,7 +1653,7 @@ class Commands(SecretYamlObject):
return success
def get_root_folder(self):
- return self.jinjaify(self.root_folder)
+ return cast(str, self.jinjaify(self.root_folder))
def get_commands_text(self):
return self.jinjaify(self.commands_text)
@@ -1673,7 +1680,9 @@ class Commands(SecretYamlObject):
return None
v = self.get_vars()
if self.todo_id:
- v.update(state.get_todo_by_id(self.todo_id).get_vars())
+ todo = state.get_todo_by_id(self.todo_id)
+ assert todo
+ v.update(todo.get_vars())
if isinstance(data, list):
if join:
return expand_jinja(" ".join(data), v)
@@ -1693,7 +1702,7 @@ def abbreviate_homedir(line):
elif 'USERPROFILE' in os.environ:
return re.sub(r'([^/]|\b)%s' % os.path.expanduser('~'), "\\1%USERPROFILE%", line)
else:
- return None
+ return line
else:
return re.sub(r'([^/]|\b)%s' % os.path.expanduser('~'), "\\1~", line)
@@ -1741,7 +1750,7 @@ class Command(SecretYamlObject):
return self.jinjaify(self.redirect)
def get_cmd(self):
- return self.jinjaify(self.cmd, join=True)
+ return cast(str, self.jinjaify(self.cmd, join=True))
def get_vars(self):
myvars = {}
@@ -1760,7 +1769,9 @@ class Command(SecretYamlObject):
def jinjaify(self, data, join=False):
v = self.get_vars()
if self.todo_id:
- v.update(state.get_todo_by_id(self.todo_id).get_vars())
+ todo = state.get_todo_by_id(self.todo_id)
+ assert todo
+ v.update(todo.get_vars())
if isinstance(data, list):
if join:
return expand_jinja(" ".join(data), v)
@@ -1821,15 +1832,16 @@ class UserInput(SecretYamlObject):
def create_ical(_todo): # pylint: disable=unused-argument
if ask_yes_no("Do you want to add a Calendar reminder for the close vote time?"):
- c = Calendar()
- e = Event()
+ # TODO: this library has broken typing and seems unmaintained, replace?
+ c = Calendar() # pyright: ignore[reportArgumentType]
+ e = Event() # pyright: ignore[reportArgumentType]
e.name = "Lucene %s vote ends" % state.release_version
e.begin = vote_close_72h_date()
e.description = "Remember to sum up votes and continue release :)"
c.events.add(e)
ics_file = os.path.join(state.get_rc_folder(), 'vote_end.ics')
with open(ics_file, 'w') as my_file:
- my_file.writelines(c)
+ my_file.writelines(c) # pyright: ignore[reportArgumentType]
open_file(ics_file)
return True
@@ -1838,8 +1850,8 @@ today = datetime.now(tz=timezone.utc).date()
sundays = {(today + timedelta(days=x)): 'Sunday' for x in range(10) if (today + timedelta(days=x)).weekday() == 6}
y = datetime.now(tz=timezone.utc).year
years = [y, y+1]
-non_working = holidays.CA(years=years) + holidays.US(years=years) + holidays.UK(years=years) \
- + holidays.DE(years=years) + holidays.NO(years=years) + holidays.IN(years=years) + holidays.RU(years=years)
+non_working = country_holidays('CA', years=years) + country_holidays('US', years=years) + country_holidays('UK', years=years) \
+ + country_holidays('DE', years=years) + country_holidays('NO', years=years) + country_holidays('IN', years=years) + country_holidays('RU', years=years)
def vote_close_72h_date():
@@ -1878,16 +1890,16 @@ def prepare_announce_lucene(_todo): # pylint: disable=unused-argument
def check_artifacts_available(_todo): # pylint: disable=unused-argument
+ cdnUrl = expand_jinja("https://dlcdn.apache.org/lucene/java/{{ release_version }}/lucene-{{ release_version }}-src.tgz.asc")
try:
- cdnUrl = expand_jinja("https://dlcdn.apache.org/lucene/java/{{ release_version }}/lucene-{{ release_version }}-src.tgz.asc")
load(cdnUrl)
print("Found %s" % cdnUrl)
except Exception as e:
print("Could not fetch %s (%s)" % (cdnUrl, e))
return False
+ mavenUrl = expand_jinja("https://repo1.maven.org/maven2/org/apache/lucene/lucene-core/{{ release_version }}/lucene-core-{{ release_version }}.jar.asc")
try:
- mavenUrl = expand_jinja("https://repo1.maven.org/maven2/org/apache/lucene/lucene-core/{{ release_version }}/lucene-core-{{ release_version }}.jar.asc")
load(mavenUrl)
print("Found %s" % mavenUrl)
except Exception as e:
diff --git a/dev-tools/scripts/reproduceJenkinsFailures.py b/dev-tools/scripts/reproduceJenkinsFailures.py
index f6994c0813f..95fb35d33f3 100644
--- a/dev-tools/scripts/reproduceJenkinsFailures.py
+++ b/dev-tools/scripts/reproduceJenkinsFailures.py
@@ -187,6 +187,7 @@ def groupTestsByModule(tests):
test = match.group(1)
if test in tests:
match = reModule.match(dir)
+ assert match
module = match.group(1)
if module not in modules:
modules[module] = set()
@@ -251,6 +252,7 @@ def getLocalGitBranch():
def main():
config = readConfig()
tests = fetchAndParseJenkinsLog(config.url, numRetries = 2)
+ localGitBranch = None
if config.useGit:
localGitBranch = getLocalGitBranch()
diff --git a/dev-tools/scripts/scriptutil.py b/dev-tools/scripts/scriptutil.py
index 3995050ca55..5bc5c56de29 100644
--- a/dev-tools/scripts/scriptutil.py
+++ b/dev-tools/scripts/scriptutil.py
@@ -31,12 +31,12 @@ class Version(object):
self.bugfix = bugfix
self.prerelease = prerelease
self.previous_dot_matcher = self.make_previous_matcher()
- self.dot = '%d.%d.%d' % (self.major, self.minor, self.bugfix)
+ self.dot = '%d.%d.%d' % (self.major, self.minor, self.bugfix)
self.constant = 'LUCENE_%d_%d_%d' % (self.major, self.minor, self.bugfix)
@classmethod
def parse(cls, value):
- match = re.search(r'(\d+)\.(\d+).(\d+)(.1|.2)?', value)
+ match = re.search(r'(\d+)\.(\d+).(\d+)(.1|.2)?', value)
if match is None:
raise argparse.ArgumentTypeError('Version argument must be of format x.y.z(.1|.2)?')
parts = [int(v) for v in match.groups()[:-1]]
@@ -87,12 +87,12 @@ def run(cmd, cwd=None):
except subprocess.CalledProcessError as e:
print(e.output.decode('utf-8'))
raise e
- return output.decode('utf-8')
+ return output.decode('utf-8')
def update_file(filename, line_re, edit, append=None):
infile = open(filename, 'r')
- buffer = []
-
+ buffer = []
+
changed = False
for line in infile:
if not changed:
@@ -182,7 +182,9 @@ version_prop_re = re.compile(r'baseVersion\s*=\s*([\'"])(.*)\1')
def find_current_version():
script_path = os.path.dirname(os.path.realpath(__file__))
top_level_dir = os.path.join(os.path.abspath("%s/" % script_path), os.path.pardir, os.path.pardir)
- return version_prop_re.search(open('%s/build.gradle' % top_level_dir).read()).group(2).strip()
+ match = version_prop_re.search(open('%s/build.gradle' % top_level_dir).read())
+ assert match
+ return match.group(2).strip()
if __name__ == '__main__':
print('This is only a support module, it cannot be run')
diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py
index 57b9ddfc1ff..12e61633cf3 100755
--- a/dev-tools/scripts/smokeTestRelease.py
+++ b/dev-tools/scripts/smokeTestRelease.py
@@ -506,6 +506,7 @@ def getDirEntries(urlString):
for i, (text, _) in enumerate(links):
if text == 'Parent Directory' or text == '..':
return links[(i+1):]
+ raise RuntimeError('could not enumerate %s' % (urlString))
def unpackAndVerify(java, tmpDir, artifact, gitRevision, version, testArgs):
@@ -675,7 +676,7 @@ def testDemo(run_java, isSrc, version, jdk):
run_java(indexFilesCmd, 'index.log')
run_java(searchFilesCmd, 'search.log')
- reMatchingDocs = re.compile('(\d+) total matching documents')
+ reMatchingDocs = re.compile(r'(\d+) total matching documents')
m = reMatchingDocs.search(open('search.log', encoding='UTF-8').read())
if m is None:
raise RuntimeError('lucene demo\'s SearchFiles found no results')
@@ -928,7 +929,9 @@ def make_java_config(parser, alt_java_homes):
s = subprocess.check_output('%s; java -version' % cmd_prefix,
shell=True, stderr=subprocess.STDOUT).decode('utf-8')
- actual_version = re.search(r'version "([1-9][0-9]*)', s).group(1)
+ match = re.search(r'version "([1-9][0-9]*)', s)
+ assert match
+ actual_version = match.group(1)
print('Java %s JAVA_HOME=%s' % (actual_version, java_home))
# validate Java version
@@ -1059,7 +1062,9 @@ def confirmAllReleasesAreTestedForBackCompat(smokeVersion, unpackPath):
reIndexName = re.compile(r'^[^.]*.(.*?)-cfs.zip')
for name in testedIndicesPaths:
basename = os.path.basename(name)
- version = reIndexName.fullmatch(basename).group(1)
+ match = reIndexName.fullmatch(basename)
+ assert match
+ version = match.group(1)
tup = tuple(version.split('.'))
if len(tup) == 3:
# ok
@@ -1123,7 +1128,9 @@ def main():
c = parse_config()
# Pick <major>.<minor> part of version and require script to be from same branch
- scriptVersion = re.search(r'((\d+).(\d+)).(\d+)', scriptutil.find_current_version()).group(1).strip()
+ match = re.search(r'((\d+).(\d+)).(\d+)', scriptutil.find_current_version())
+ assert match
+ scriptVersion = match.group(1).strip()
if not c.version.startswith(scriptVersion + '.'):
raise RuntimeError('smokeTestRelease.py for %s.X is incompatible with a %s release.' % (scriptVersion, c.version))