You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2025/03/12 00:05:56 UTC
(lucene) branch main updated: python: enable all linting checks and type-hint the code (#14326)
This is an automated email from the ASF dual-hosted git repository.
rmuir pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new be742dd7166 python: enable all linting checks and type-hint the code (#14326)
be742dd7166 is described below
commit be742dd71664e86425194b51cb2c842f6d7482e5
Author: Robert Muir <rm...@apache.org>
AuthorDate: Tue Mar 11 20:05:51 2025 -0400
python: enable all linting checks and type-hint the code (#14326)
* enable all type checking
* type-hint the code to get checker flowing
* disable a few remaining checks with a TODO list
* enable all linter checks
* autofix all safe fixes (e.g. api migrations)
* disable just the remaining checks with a massive TODO list
---
dev-tools/scripts/Makefile | 10 +-
dev-tools/scripts/addBackcompatIndexes.py | 17 +-
dev-tools/scripts/addVersion.py | 35 +-
dev-tools/scripts/buildAndPushRelease.py | 41 ++-
dev-tools/scripts/create_line_file_docs.py | 30 +-
dev-tools/scripts/diff_lucene_changes.py | 7 +-
dev-tools/scripts/githubPRs.py | 23 +-
dev-tools/scripts/pyproject.toml | 152 ++++++++-
dev-tools/scripts/releaseWizard.py | 472 +++++++++++++-------------
dev-tools/scripts/releasedJirasRegex.py | 35 +-
dev-tools/scripts/reproduceJenkinsFailures.py | 46 ++-
dev-tools/scripts/scriptutil.py | 37 +-
dev-tools/scripts/smokeTestRelease.py | 201 +++++------
13 files changed, 632 insertions(+), 474 deletions(-)
diff --git a/dev-tools/scripts/Makefile b/dev-tools/scripts/Makefile
index 97ed7aaed24..f2786ba142c 100644
--- a/dev-tools/scripts/Makefile
+++ b/dev-tools/scripts/Makefile
@@ -17,7 +17,7 @@ PIP_INSTALL_ARGS=--disable-pip-version-check --no-input --upgrade
VENV=${PWD}/.venv
# don't behave strangely if these files exist
-.PHONY: lint format reformat ruff pyright env clean
+.PHONY: lint format reformat autofix ruff ruff-fix pyright env clean
# list of directories we check
SOURCES=$(wildcard *.py)
@@ -39,11 +39,19 @@ reformat: env
# reformat sources
$(VENV)/bin/ruff format $(SOURCES)
+# fixes all code problems that are safe to automatically fix (including formatting)
+autofix: ruff-fix reformat
+
# lints sources
ruff: env
# validate sources with ruff linter
$(VENV)/bin/ruff check $(SOURCES)
+# fixes (safe) issues that are autofixable such as deprecated/renamed APIs
+ruff-fix: env
+ # (safe) fixes sources with ruff linter
+ $(VENV)/bin/ruff check --fix $(SOURCES)
+
# checks types
pyright: env
# type-check sources with basedpyright
diff --git a/dev-tools/scripts/addBackcompatIndexes.py b/dev-tools/scripts/addBackcompatIndexes.py
index 3ae0ea62d4c..417ea785b12 100755
--- a/dev-tools/scripts/addBackcompatIndexes.py
+++ b/dev-tools/scripts/addBackcompatIndexes.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -34,7 +33,7 @@ import urllib.request
import scriptutil
-def create_and_add_index(source, indextype, index_version, current_version, temp_dir):
+def create_and_add_index(source: str, indextype: str, index_version: scriptutil.Version, current_version: scriptutil.Version, temp_dir: str):
if not current_version.is_back_compat_with(index_version):
prefix = "unsupported"
else:
@@ -79,7 +78,7 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
print("done")
-def update_backcompat_tests(index_version, current_version):
+def update_backcompat_tests(index_version: scriptutil.Version, current_version: scriptutil.Version):
print(" adding new indexes to backcompat tests...", end="", flush=True)
module = "lucene/backward-codecs"
@@ -91,12 +90,12 @@ def update_backcompat_tests(index_version, current_version):
strip_dash_suffix_re = re.compile(r"-.*")
- def find_version(x):
+ def find_version(x: str):
x = x.strip()
x = re.sub(strip_dash_suffix_re, "", x) # remove the -suffix if any
return scriptutil.Version.parse(x)
- def edit(buffer, match, line):
+ def edit(buffer: list[str], _match: re.Match[str], line: str):
v = find_version(line)
changed = False
if v.on_or_after(index_version):
@@ -106,7 +105,7 @@ def update_backcompat_tests(index_version, current_version):
buffer.append(line)
return changed
- def append(buffer, changed):
+ def append(buffer: list[str], changed: bool):
if changed:
return changed
if not buffer[len(buffer) - 1].endswith("\n"):
@@ -124,7 +123,7 @@ def check_backcompat_tests():
print("ok")
-def download_from_cdn(version, remotename, localname):
+def download_from_cdn(version: scriptutil.Version, remotename: str, localname: str):
url = "http://dlcdn.apache.org/lucene/java/%s/%s" % (version, remotename)
try:
urllib.request.urlretrieve(url, localname)
@@ -135,7 +134,7 @@ def download_from_cdn(version, remotename, localname):
raise e
-def download_from_archives(version, remotename, localname):
+def download_from_archives(version: scriptutil.Version, remotename: str, localname: str):
url = "http://archive.apache.org/dist/lucene/java/%s/%s" % (version, remotename)
try:
urllib.request.urlretrieve(url, localname)
@@ -146,7 +145,7 @@ def download_from_archives(version, remotename, localname):
raise e
-def download_release(version, temp_dir, force):
+def download_release(version: scriptutil.Version, temp_dir: str, force: bool):
print(" downloading %s source release..." % version, end="", flush=True)
source = os.path.join(temp_dir, "lucene-%s" % version)
if os.path.exists(source):
diff --git a/dev-tools/scripts/addVersion.py b/dev-tools/scripts/addVersion.py
index 75632d6a6dd..952177ca0c1 100755
--- a/dev-tools/scripts/addVersion.py
+++ b/dev-tools/scripts/addVersion.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -26,11 +25,11 @@ from configparser import ConfigParser, ExtendedInterpolation
from scriptutil import Version, find_branch_type, find_current_version, run, update_file
-def update_changes(filename, new_version, init_changes, headers):
+def update_changes(filename: str, new_version: Version, init_changes: str, headers: list[str]):
print(" adding new section to %s..." % filename, end="", flush=True)
matcher = re.compile(r"\d+\.\d+\.\d+\s+===")
- def edit(buffer, match, line):
+ def edit(buffer: list[str], _match: re.Match[str], line: str):
if new_version.dot in line:
return None
match = new_version.previous_dot_matcher.search(line)
@@ -46,14 +45,14 @@ def update_changes(filename, new_version, init_changes, headers):
print("done" if changed else "uptodate")
-def add_constant(new_version, deprecate):
+def add_constant(new_version: Version, deprecate: bool):
filename = "lucene/core/src/java/org/apache/lucene/util/Version.java"
print(" adding constant %s..." % new_version.constant, end="", flush=True)
constant_prefix = "public static final Version LUCENE_"
matcher = re.compile(constant_prefix)
prev_matcher = new_version.make_previous_matcher(prefix=constant_prefix, sep="_")
- def ensure_deprecated(buffer):
+ def ensure_deprecated(buffer: list[str]):
last = buffer[-1]
if last.strip() != "@Deprecated":
spaces = " " * (len(last) - len(last.lstrip()) - 1)
@@ -62,22 +61,22 @@ def add_constant(new_version, deprecate):
del buffer[-3:] # drop the trailing lines '<p> / Use this to get the latest ... / ... for Lucene.'
buffer.append(("{0} * @deprecated ({1}) Use latest\n" + "{0} */\n" + "{0}@Deprecated\n").format(spaces, new_version))
- def buffer_constant(buffer, line):
+ def buffer_constant(buffer: list[str], line: str):
spaces = " " * (len(line) - len(line.lstrip()))
buffer.append(("\n{0}/**\n" + "{0} * Match settings and bugs in Lucene's {1} release.\n").format(spaces, new_version))
if deprecate:
buffer.append("%s * @deprecated Use latest\n" % spaces)
else:
- buffer.append(("{0} * <p>Use this to get the latest & greatest settings, bug fixes, etc, for Lucene.\n").format(spaces))
+ buffer.append(f"{spaces} * <p>Use this to get the latest & greatest settings, bug fixes, etc, for Lucene.\n")
buffer.append("%s */\n" % spaces)
if deprecate:
buffer.append("%s@Deprecated\n" % spaces)
- buffer.append("{0}public static final Version {1} = new Version({2}, {3}, {4});\n".format(spaces, new_version.constant, new_version.major, new_version.minor, new_version.bugfix))
+ buffer.append(f"{spaces}public static final Version {new_version.constant} = new Version({new_version.major}, {new_version.minor}, {new_version.bugfix});\n")
- class Edit(object):
+ class Edit:
found = -1
- def __call__(self, buffer, match, line):
+ def __call__(self, buffer: list[str], _match: re.Match[str], line: str):
if new_version.constant in line:
return None # constant already exists
# outer match is just to find lines declaring version constants
@@ -88,7 +87,7 @@ def add_constant(new_version, deprecate):
elif self.found != -1:
# we didn't match, but we previously had a match, so insert new version here
# first find where to insert (first empty line before current constant)
- c = []
+ c: list[str] = []
buffer_constant(c, line)
tmp = buffer[self.found :]
buffer[self.found :] = c
@@ -103,11 +102,11 @@ def add_constant(new_version, deprecate):
print("done" if changed else "uptodate")
-def update_build_version(new_version):
+def update_build_version(new_version: Version):
print(" changing baseVersion...", end="", flush=True)
filename = "build.gradle"
- def edit(buffer, _, line):
+ def edit(buffer: list[str], _match: re.Match[str], line: str):
if new_version.dot in line:
return None
buffer.append(" String baseVersion = '" + new_version.dot + "'\n")
@@ -118,12 +117,12 @@ def update_build_version(new_version):
print("done" if changed else "uptodate")
-def update_latest_constant(new_version):
+def update_latest_constant(new_version: Version):
print(" changing Version.LATEST to %s..." % new_version.constant, end="", flush=True)
filename = "lucene/core/src/java/org/apache/lucene/util/Version.java"
matcher = re.compile("public static final Version LATEST")
- def edit(buffer, _, line):
+ def edit(buffer: list[str], _match: re.Match[str], line: str):
if new_version.constant in line:
return None
buffer.append(line.rpartition("=")[0] + ("= %s;\n" % new_version.constant))
@@ -133,7 +132,7 @@ def update_latest_constant(new_version):
print("done" if changed else "uptodate")
-def onerror(x):
+def onerror(x: Exception):
raise x
@@ -143,7 +142,7 @@ def check_lucene_version_tests():
print("ok")
-def read_config(current_version):
+def read_config(current_version: Version):
parser = argparse.ArgumentParser(description="Add a new version to CHANGES, to Version.java and build.gradle files")
parser.add_argument("version", type=Version.parse)
newconf = parser.parse_args()
@@ -157,7 +156,7 @@ def read_config(current_version):
# Hack ConfigParser, designed to parse INI files, to parse & interpolate Java .properties files
-def parse_properties_file(filename):
+def parse_properties_file(filename: str):
contents = open(filename, encoding="ISO-8859-1").read().replace("%", "%%") # Escape interpolation metachar
parser = ConfigParser(interpolation=ExtendedInterpolation()) # Handle ${property-name} interpolation
parser.read_string("[DUMMY_SECTION]\n" + contents) # Add required section
diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py
index 85952be2dd0..b000d180605 100755
--- a/dev-tools/scripts/buildAndPushRelease.py
+++ b/dev-tools/scripts/buildAndPushRelease.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -35,13 +34,13 @@ LOG = "/tmp/release.log"
dev_mode = False
-def log(msg):
+def log(msg: str):
f = open(LOG, mode="ab")
f.write(msg.encode("utf-8"))
f.close()
-def run(command):
+def run(command: str):
log("\n\n%s: RUN: %s\n" % (datetime.datetime.now(), command))
if os.system("%s >> %s 2>&1" % (command, LOG)):
msg = " FAILED: %s [see log %s]" % (command, LOG)
@@ -49,7 +48,7 @@ def run(command):
raise RuntimeError(msg)
-def runAndSendGPGPassword(command, password):
+def runAndSendGPGPassword(command: str, password: str):
p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
f = open(LOG, "ab")
while True:
@@ -63,7 +62,7 @@ def runAndSendGPGPassword(command, password):
if line.find(b"Enter GPG keystore password:") != -1:
time.sleep(1.0)
p.stdin.write((password + "\n").encode("UTF-8"))
- p.stdin.write("\n".encode("UTF-8"))
+ p.stdin.write(b"\n")
try:
result = p.wait(timeout=120)
@@ -77,7 +76,7 @@ def runAndSendGPGPassword(command, password):
raise RuntimeError(msg)
-def load(urlString, encoding="utf-8"):
+def load(urlString: str, encoding: str = "utf-8"):
try:
content = urllib.request.urlopen(urlString).read().decode(encoding)
except Exception as e:
@@ -99,7 +98,7 @@ def getGitRev():
return os.popen("git rev-parse HEAD").read().strip()
-def prepare(root, version, pause_before_sign, gpg_key_id, gpg_password, gpg_home=None, sign_gradle=False):
+def prepare(root: str, version: str, pause_before_sign: bool, gpg_key_id: str | None, gpg_password: str | None, gpg_home: str | None = None, sign_gradle: bool = False):
print()
print("Prepare release...")
if os.path.exists(LOG):
@@ -166,30 +165,30 @@ reVersion2 = re.compile(r"-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.zip<", re.IGNOREC
reDoapRevision = re.compile(r"(\d+)\.(\d+)(?:\.(\d+))?(-alpha|-beta)?", re.IGNORECASE)
-def checkDOAPfiles(version):
+def checkDOAPfiles(version: str):
# In Lucene DOAP file, verify presence of all releases less than the one being produced.
- errorMessages = []
+ errorMessages: list[str] = []
for product in ["lucene"]:
url = "https://archive.apache.org/dist/lucene/%s" % ("java" if product == "lucene" else product)
distpage = load(url)
- releases = set()
+ releases: set[str] = set()
for regex in reVersion1, reVersion2:
for tup in regex.findall(distpage):
if tup[0] in ("1", "2"): # Ignore 1.X and 2.X releases
continue
releases.add(normalizeVersion(tup))
doapNS = "{http://usefulinc.com/ns/doap#}"
- xpathRevision = "{0}Project/{0}release/{0}Version/{0}revision".format(doapNS)
+ xpathRevision = f"{doapNS}Project/{doapNS}release/{doapNS}Version/{doapNS}revision"
doapFile = "dev-tools/doap/%s.rdf" % product
treeRoot = ET.parse(doapFile).getroot()
- doapRevisions = set()
+ doapRevisions: set[str] = set()
for revision in treeRoot.findall(xpathRevision):
if revision.text and (match := reDoapRevision.match(revision.text)):
if match.group(1) not in ("0", "1", "2"): # Ignore 0.X, 1.X and 2.X revisions
doapRevisions.add(normalizeVersion(match.groups()))
else:
errorMessages.append("ERROR: Failed to parse revision: %s in %s" % (revision.text, doapFile))
- missingDoapRevisions = set()
+ missingDoapRevisions: set[str] = set()
for release in releases:
if release not in doapRevisions and release < version: # Ignore releases greater than the one being produced
missingDoapRevisions.add(release)
@@ -199,7 +198,7 @@ def checkDOAPfiles(version):
raise RuntimeError("\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)" % "\n".join(errorMessages))
-def normalizeVersion(tup):
+def normalizeVersion(tup: tuple[str, ...]):
suffix = ""
if tup[-1] is not None and tup[-1].lower() == "-alpha":
tup = tup[: (len(tup) - 1)]
@@ -214,7 +213,7 @@ def normalizeVersion(tup):
return ".".join(tup) + suffix
-def pushLocal(version, root, rcNum, localDir):
+def pushLocal(version: str, root: str, rcNum: int, localDir: str):
print("Push local [%s]..." % localDir)
os.makedirs(localDir)
@@ -243,7 +242,7 @@ def pushLocal(version, root, rcNum, localDir):
return "file://%s/%s" % (os.path.abspath(localDir), dir)
-def read_version(path):
+def read_version(_path: str):
return scriptutil.find_current_version()
@@ -271,9 +270,9 @@ def parse_config():
dest="gpg_pass_noprompt",
default=False,
action="store_true",
- help="Do not prompt for gpg passphrase. For the default gnupg method, this means your gpg-agent"
- " needs a non-TTY pin-entry program. For gradle signing method, passphrase must be provided"
- " in gradle.properties or by env.var/sysprop. See ./gradlew helpPublishing for more info",
+ help="""Do not prompt for gpg passphrase. For the default gnupg method, this means your gpg-agent
+ needs a non-TTY pin-entry program. For gradle signing method, passphrase must be provided
+ in gradle.properties or by env.var/sysprop. See ./gradlew helpPublishing for more info""",
)
parser.add_argument("--gpg-home", metavar="PATH", help="Path to gpg home containing your secring.gpg Optional, will use $HOME/.gnupg/secring.gpg by default")
parser.add_argument("--rc-num", metavar="NUM", type=int, default=1, help="Release Candidate number. Default: 1")
@@ -301,7 +300,7 @@ def parse_config():
cwd = os.getcwd()
os.chdir(config.root)
config.root = os.getcwd() # Absolutize root dir
- if os.system("git rev-parse") or 2 != len([d for d in ("dev-tools", "lucene") if os.path.isdir(d)]):
+ if os.system("git rev-parse") or len([d for d in ("dev-tools", "lucene") if os.path.isdir(d)]) != 2:
parser.error('Root path "%s" is not a valid lucene checkout' % config.root)
os.chdir(cwd)
global LOG
@@ -320,7 +319,7 @@ def check_cmdline_tools(): # Fail fast if there are cmdline tool problems
raise RuntimeError('"git --version" returned a non-zero exit code.')
-def check_key_in_keys(gpgKeyID, local_keys):
+def check_key_in_keys(gpgKeyID: str | None, local_keys: str | None):
if gpgKeyID is not None:
print(" Verify your gpg key is in the main KEYS file")
if local_keys is not None:
diff --git a/dev-tools/scripts/create_line_file_docs.py b/dev-tools/scripts/create_line_file_docs.py
index 9c753f22f79..1dd9963ecdc 100644
--- a/dev-tools/scripts/create_line_file_docs.py
+++ b/dev-tools/scripts/create_line_file_docs.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -24,16 +23,17 @@ import subprocess
import tempfile
import time
import urllib.request
+from io import TextIOWrapper
DEBUG = False
TARGET_DOC_CHARS = 1024
-def compress_with_seek_points(file_name_in, file_name_out, num_seek_points):
+def compress_with_seek_points(file_name_in: str, file_name_out: str, num_seek_points: int):
bytes_per_chunk = os.path.getsize(file_name_in) / num_seek_points
- seek_points = []
+ seek_points: list[int] = []
if os.path.exists(file_name_out):
os.remove(file_name_out)
@@ -60,7 +60,7 @@ def compress_with_seek_points(file_name_in, file_name_out, num_seek_points):
break
bytes_in_chunk += len(line)
- f_out.write(line) # false positive in python's crazy typing # pyright: ignore[reportArgumentType]
+ f_out.write(line)
if bytes_in_chunk > bytes_per_chunk and chunk_count < num_seek_points:
f_out.close()
@@ -77,16 +77,13 @@ re_newlines = re.compile(r"\n+")
re_space = re.compile(r"\s")
# used to find word break, for splitting docs into ~1 KB sized smaller docs:
-re_next_non_word_character = re.compile(r"\W", re.U)
+re_next_non_word_character = re.compile(r"\W", re.UNICODE)
EUROPARL_V7_URL = "https://www.statmt.org/europarl/v7/europarl.tgz"
-def split_docs(all_out, title_string, date_string, body_string):
- """
- Splits docs into smallish (~1 KB) sized docs, repeating same title and date
- """
-
+def split_docs(all_out: TextIOWrapper, title_string: str, date_string: str, body_string: str):
+ """Splits docs into smallish (~1 KB) sized docs, repeating same title and date"""
doc_count = 0
while len(body_string) > 0:
char_count = int(random.gauss(TARGET_DOC_CHARS, TARGET_DOC_CHARS / 4))
@@ -129,7 +126,7 @@ def sample_europarl():
if not DEBUG:
cmd = "tar xzf %s -C %s" % (file_name, tmp_dir_path)
print("Run: %s" % cmd)
- subprocess.run(cmd, shell=True)
+ subprocess.run(cmd, shell=True, check=False)
doc_count = 0
skip_count = 0
@@ -158,8 +155,8 @@ def sample_europarl():
# unfortunately we need errors='ignore' since in Europarl v7, one file (pl/ep-09-10-22-009.txt) has invalid utf-8:
chapter_count = 0
- with open("%s/%s" % (dir_path, file_name), "r", encoding="utf-8", errors="ignore") as f_in:
- last_text = []
+ with open("%s/%s" % (dir_path, file_name), encoding="utf-8", errors="ignore") as f_in:
+ last_text: list[str] = []
last_title = None
while True:
line = f_in.readline()
@@ -188,8 +185,7 @@ def sample_europarl():
if len(last_title) > 0:
break
continue
- else:
- last_text.append(line)
+ last_text.append(line)
if last_title is not None:
s = " ".join(last_text)
@@ -232,7 +228,7 @@ def sample_europarl():
)
print("Shuffle...")
- subprocess.run("shuf %s > %s.shuffled" % (all_txt_file_name, all_txt_file_name), shell=True)
+ subprocess.run("shuf %s > %s.shuffled" % (all_txt_file_name, all_txt_file_name), shell=True, check=False)
for mb in (20, 200, 2000):
print("Sample %d MB file..." % mb)
@@ -240,7 +236,7 @@ def sample_europarl():
with open(file_name_out, "w", encoding="utf-8") as f_out:
chance = mb / total_mb
- with open(all_txt_file_name + ".shuffled", "r", encoding="utf-8") as f:
+ with open(all_txt_file_name + ".shuffled", encoding="utf-8") as f:
while True:
line = f.readline()
if len(line) == 0:
diff --git a/dev-tools/scripts/diff_lucene_changes.py b/dev-tools/scripts/diff_lucene_changes.py
index d443ff32bce..4d8e5121f39 100644
--- a/dev-tools/scripts/diff_lucene_changes.py
+++ b/dev-tools/scripts/diff_lucene_changes.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -31,10 +30,8 @@ to users and avoid future annoying conflicts on backport.
# e.g. python3 -u diff_lucene_changes.py branch_9_9 main 9.9.0
-#
-
-def get_changes_url(branch_name):
+def get_changes_url(branch_name: str):
if os.path.isdir(branch_name):
url = f"file://{branch_name}/lucene/CHANGES.txt"
else:
@@ -43,7 +40,7 @@ def get_changes_url(branch_name):
return url
-def extract_release_section(changes_txt, release_name):
+def extract_release_section(changes_txt: bytes, release_name: str):
match = re.search(f"=======+ Lucene {re.escape(release_name)} =======+(.*?)=======+ Lucene .*? =======+$", changes_txt.decode("utf-8"), re.MULTILINE | re.DOTALL)
assert match
return match.group(1).encode("utf-8")
diff --git a/dev-tools/scripts/githubPRs.py b/dev-tools/scripts/githubPRs.py
index 335c05e6aa7..66a2ae30713 100755
--- a/dev-tools/scripts/githubPRs.py
+++ b/dev-tools/scripts/githubPRs.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -15,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""
-Simple script that queries GitHub for all open PRs, then finds the ones without
+"""Simple script that queries GitHub for all open PRs, then finds the ones without
issue number in title, and the ones where the linked JIRA is already closed
"""
@@ -27,13 +25,16 @@ sys.path.append(os.path.dirname(__file__))
import argparse
import json
import re
-from typing import cast
+from typing import TYPE_CHECKING, Any, cast
from github import Github
from jinja2 import BaseLoader, Environment
from jira import JIRA, Issue
from jira.client import ResultList
+if TYPE_CHECKING:
+ from github.PullRequest import PullRequest
+
def read_config():
parser = argparse.ArgumentParser(description="Find open Pull Requests that need attention")
@@ -44,13 +45,13 @@ def read_config():
return newconf
-def out(text):
+def out(text: str):
global conf
if not (conf.json or conf.html):
print(text)
-def make_html(dict):
+def make_html(dict: dict[Any, Any]):
global conf
template = Environment(loader=BaseLoader()).from_string("""
<h1>Lucene Github PR report</h1>
@@ -83,7 +84,7 @@ def main():
else:
gh = Github()
jira = JIRA("https://issues.apache.org/jira") # this ctor has broken types in jira library. # pyright: ignore[reportArgumentType]
- result = {}
+ result: dict[str, Any] = {}
repo = gh.get_repo("apache/lucene")
open_prs = repo.get_pulls(state="open")
out("Lucene Github PR report")
@@ -93,7 +94,7 @@ def main():
lack_jira = list(filter(lambda x: not re.match(r".*\b(LUCENE)-\d{3,6}\b", x.title), open_prs))
result["no_jira_count"] = len(lack_jira)
- lack_jira_list = []
+ lack_jira_list: list[dict[str, Any]] = []
for pr in lack_jira:
lack_jira_list.append({"title": pr.title, "number": pr.number, "user": pr.user.login, "created": pr.created_at.strftime("%Y-%m-%d")})
result["no_jira"] = lack_jira_list
@@ -104,8 +105,8 @@ def main():
out("\nOpen PRs with a resolved JIRA")
has_jira = list(filter(lambda x: re.match(r".*\b(LUCENE)-\d{3,6}\b", x.title), open_prs))
- issue_ids = []
- issue_to_pr = {}
+ issue_ids: list[str] = []
+ issue_to_pr: dict[str, PullRequest] = {}
for pr in has_jira:
match = re.match(r".*\b((LUCENE)-\d{3,6})\b", pr.title)
assert match
@@ -114,7 +115,7 @@ def main():
issue_to_pr[jira_issue_str] = pr
resolved_jiras = cast(ResultList[Issue], jira.search_issues(jql_str="key in (%s) AND status in ('Closed', 'Resolved')" % ", ".join(issue_ids)))
- closed_jiras = []
+ closed_jiras: list[dict[str, Any]] = []
for issue in resolved_jiras:
pr_title = issue_to_pr[issue.key].title
pr_number = issue_to_pr[issue.key].number
diff --git a/dev-tools/scripts/pyproject.toml b/dev-tools/scripts/pyproject.toml
index bc9a970c12f..24e5c507280 100644
--- a/dev-tools/scripts/pyproject.toml
+++ b/dev-tools/scripts/pyproject.toml
@@ -1,21 +1,163 @@
[tool.pyright]
+pythonVersion = "3.12"
venvPath = "."
venv = ".venv"
-# TODO: improve!
-# typeCheckingMode = "strict"
+typeCheckingMode = "strict"
+reportCallInDefaultInitializer = "error"
+reportImplicitStringConcatenation = "error"
+reportImportCycles = "error"
+reportImplicitOverride = "error"
+reportPropertyTypeMismatch = "error"
+reportUninitializedInstanceVariable = "error"
reportUnnecessaryTypeIgnoreComment = "error"
-typeCheckingMode = "standard"
+
+# This is a big TODO list of current typing problems
+# Pick an item off the list, change to "error", and send a pull request to fix it!
+# You can find more information on a rule at https://github.com/microsoft/pyright/blob/main/docs/configuration.md
+reportConstantRedefinition = "hint" # Requires fixing nonstandard variable conventions
+reportMissingSuperCall = "hint" # Requires adding missing super().__init__() calls
+reportMissingTypeStubs = "hint" # Can be fixed by switching 'ics' to alternative such as 'icalendar'
+reportPrivateUsage = "hint" # Can be fixed by using TLS properly (Ruff S323)
+reportUnnecessaryComparison = "hint" # Requires cleaning up some dead code
+reportUntypedNamedTuple = "hint" # Requires moving to different type of named tuple
+reportUnusedCallResult = "none" # Forces you to assign unused retvals to '_', very noisy.
+
+# Intentionally disabled because it slows pyright by 2x
+reportShadowedImports = "none" # Extremely slow check
[tool.ruff]
+target-version = "py312"
line-length = 200
indent-width = 2
[tool.ruff.lint]
-# TODO: improve!
-# select = ["ALL"]
+select = ["ALL"]
# disabling/enabling of rules
ignore = [
+ # This is a big TODO list of current linter problems.
+ # Pick an item off the list, remove it, and send a pull request to fix it!
+ # You can find more information on a rule at https://docs.astral.sh/ruff/rules/
+ "A001", # variable name is shadowing a python builtin
+ "A002", # function argument name is shadowing a python builtin
+ "ANN001", # missing type annotation for function argument
+ "ANN201", # missing return annotation for public function
+ "ANN202", # missing return annotation for private function
+ "ANN204", # missing return annotation for special method
+ "ANN206", # missing return annotation for classmethod
+ "ANN401", # use of Any as function parameter
+ "ARG001", # unused function argument
+ "ARG002", # unused method argument
+ "ARG005", # unused lambda argument
+ "B006", # do not use mutable datastructures for arg defaults
+ "B904", # within except clause, raise exceptions with .. from
+ "BLE001", # do not catch blind exception
+ "C408", # unnecessary dict() call
+ "C416", # unnecessary list comprehension
+ "C417", # unnecessary map() usage
+ "C901", # function is too complex
+ "D100", # missing docstring in public module
+ "D101", # missing docstring in public class
+ "D102", # missing docstring in public method
+ "D103", # missing docstring in public function
+ "D105", # missing docstring in magic method
+ "D107", # missing docstring in ctor
+ "D205", # blank line required between summary and description
+ "D400", # first docstring line should end with a period
+ "D401", # first docstring line should be in imperative mood
+ "D415", # first docstring line should end with punctuation
+ "DTZ005", # datetime.datetime.now called without tz argument
+ "EM101", # exception must not use a string literal, assign to variable first
+ "ERA001", # found commented-out code
+ "EXE001", # shebang present but file is not executable
+ "FBT001", # boolean-typed positional argument in function definition
+ "FBT002", # boolean default positional argument in function definition
+ "FBT003", # boolean positional value in function call
+ "FIX002", # line contains TODO, consider resolving issue
+ "FIX004", # line contains HACK, consider resolving issue
+ "FURB129", # instead of calling readlines(), iterate over file object directly
+ "N802", # function name should be lowercase
+ "N803", # argument name should be lowercase
+ "N806", # variable should be lowercase
+ "N816", # variable in global scope should not be mixedCase
+ "PERF203", # try-except within a loop incurs performance overhead
+ "PERF401", # use list.extend to create a transformed list
+ "PIE810", # call endswith once with a tuple
+ "PLC0206", # extracting value from dictionary without calling .items
+ "PLR0911", # too many return statements
+ "PLR0912", # too many branches
+ "PLR0913", # too many arguments in function definition
+ "PLR0915", # too many statements
+ "PLR1704", # redefining argument with local name
+ "PLR1714", # consider merging multiple comparisons
+ "PLR1722", # use sys.exit() instead of exit
+ "PLR2004", # magic value used in comparison
+ "PLW0602", # using global for variable but no assignment is done
+ "PLW0603", # using global statement to update variable is discouraged
+ "PLW2901", # loop variable overwritten by assignment target
+ "PTH100", # os.path.abspath should be replaced by Path.resolve
+ "PTH103", # os.mkdirs should be replaced by Path.mkdir(parents=True)
+ "PTH104", # os.rename should be replaced by Path.rename
+ "PTH107", # os.remove shoudl be replaced by Path.unlink
+ "PTH109", # os.getcwd should be replaced by Path.cwd
+ "PTH110", # os.path.exists should be replaced by Path.exists
+ "PTH111", # os.path.expanduser should be replaced by Path.expanduser
+ "PTH112", # os.path.isdir should be replaced by Path.is_dir
+ "PTH113", # os.path.isfile should be replaced by Path.is_file
+ "PTH116", # os.stat should be replaced by Path.stat, Path.owner, or Path.group
+ "PTH117", # os.path.isabs should be replaced by Path.is_absolute
+ "PTH118", # os.path.join should be replaced by Path with '/' operator
+ "PTH119", # os.path.basename should be replaced by Path.name
+ "PTH120", # os.path.dirname should be replaced by Path.parent
+ "PTH123", # open should be replaced by Path.open
+ "PTH202", # os.path.getsize should be replaced by Path.stat.st_size
+ "PTH207", # glob should be replaced by Path.glob or Path.rglob
+ "PYI024", # use typing.NamedTuple instead of collections.namedtuple
+ "RET503", # missing explicit return at end of function able to return non-None
+ "RET504", # unnecessary assignment before return
+ "RUF005", # unnecessary concatenation
+ "RUF012", # mutable class attributes should be annotated with typing.ClassVar
+ "S101", # use of 'assert' detected
+ "S108", # probable insecure usage of temp file/directory
+ "S110", # try-except-pass detected, consider logging the exception
+ "S307", # use of insecure eval, consider using ast.literal_eval
+ "S310", # audit url open for permitted schemes. allowing use of file: is often unexpected
+ "S311", # standard prng not suitable for cryptographic purposes
+ "S314", # using xml to parse untrusted data, use defusedxml equivalents
+ "S323", # use of insecure TLS context
+ "S324", # use of insecure hash function: sha1
+ "S506", # probable use of unsafe yaml loader, consider yaml.safe_load
+ "S602", # subprocess call with shell=True identified, security issue
+ "S603", # subprocess call: check for execution of untrusted input
+ "S605", # starting a process wtih a shell, seems safe, but may be changed in future
+ "S607", # starting a process with a partial executable path
+ "S701", # consider using jinja autoescape=True to mitigate XSS
+ "SIM102", # use single if statement instead of nested if statement
+ "SIM108", # use ternary operator instead of if-else-block
+ "SIM112", # use capitalized environment variable
+ "SIM113", # use enumerate for index variable in loop
+ "SIM115", # use a context manager for opening files
+ "SIM118", # use 'key in dict' instead of 'key in dict.keys'
+ "SIM201", # use returncode != 0 instead of not returncode == 0
+ "SLF001", # private member accessed
+ "T201", # print function found
+ "TD002", # missing author for this TODO
+ "TD003", # missing issue link for this TODO
+ "TD004", # missing colon for this TODO
+ "TD005", # missing issue description for this TODO
+ "TRY002", # create your own exception
+ "TRY003", # avoid specifying long messages outside exception class
+ "TRY201", # use raise without specifying exception name
+ "TRY300", # consider moving this statement to else block
+ "UP008", # use super() instead of super(__class__, self)
+ "UP036", # version block is outdated for minimum python version
+ "UP031", # use format specifiers instead of percent format
+
+ # These rules are always disabled, conflict with other rules
+ # don't enable! (switch to the alternative instead)
+ "D203", # disabled in favor of D211 (no-black-line-before-class)
+ "D213", # disabled in favor of D212 (multi-line-summary-first-line)
+
# These rules are always disabled: conflict with the formatter
# don't enable! https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
"W191", "E111", "E114", "E117", "D206", "D300", "Q000", "Q001",
diff --git a/dev-tools/scripts/releaseWizard.py b/dev-tools/scripts/releaseWizard.py
index fa64491d77a..bfc83fdc000 100755
--- a/dev-tools/scripts/releaseWizard.py
+++ b/dev-tools/scripts/releaseWizard.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -47,8 +46,9 @@ import textwrap
import time
import urllib.request
from collections import OrderedDict
-from datetime import datetime, timedelta, timezone
-from typing import cast
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+from typing import Any, Self, TextIO, cast, override
import yaml
from consolemenu import ConsoleMenu
@@ -66,8 +66,8 @@ editor = None
# Edit this to add other global jinja2 variables or filters
-def expand_jinja(text, vars=None):
- global_vars = OrderedDict(
+def expand_jinja(text: str, vars: dict[str, Any] | None = None):
+ global_vars: dict[str, Any] = OrderedDict(
{
"script_version": state.script_version,
"release_version": state.release_version,
@@ -96,7 +96,7 @@ def expand_jinja(text, vars=None):
"state": state,
"gpg_key": state.get_gpg_key(),
"gradle_cmd": "gradlew.bat" if is_windows() else "./gradlew",
- "epoch": unix_time_millis(datetime.now(tz=timezone.utc)),
+ "epoch": unix_time_millis(datetime.now(tz=UTC)),
"get_next_version": state.get_next_version(),
"current_git_rev": state.get_current_git_rev(),
"keys_downloaded": keys_downloaded(),
@@ -124,9 +124,9 @@ def expand_jinja(text, vars=None):
try:
env = Environment(lstrip_blocks=True, keep_trailing_newline=False, trim_blocks=True)
- env.filters["path_join"] = lambda paths: os.path.join(*paths)
- env.filters["expanduser"] = lambda path: os.path.expanduser(path)
- env.filters["formatdate"] = lambda date: (datetime.strftime(date, "%-d %B %Y") if date else "<date>")
+ env.filters["path_join"] = lambda paths: os.path.join(*(cast(list[str], paths)))
+ env.filters["expanduser"] = lambda path: os.path.expanduser(str(path))
+ env.filters["formatdate"] = lambda date: (datetime.strftime(date, "%-d %B %Y") if date and isinstance(date, datetime) else "<date>")
template = env.from_string(str(filled), globals=global_vars)
filled = template.render()
except Exception as e:
@@ -134,8 +134,8 @@ def expand_jinja(text, vars=None):
return filled
-def replace_templates(text):
- tpl_lines = []
+def replace_templates(text: str):
+ tpl_lines: list[str] = []
for line in text.splitlines():
if match := re.search(r"^\(\( template=(.+?) \)\)", line):
name = match.group(1)
@@ -165,7 +165,7 @@ def get_editor():
return editor
-def check_prerequisites(todo=None):
+def check_prerequisites(todo: Any = None):
if sys.version_info < (3, 4):
sys.exit("Script requires Python v3.4 or later")
try:
@@ -197,14 +197,14 @@ def check_prerequisites(todo=None):
return True
-epoch = datetime.fromtimestamp(timestamp=0, tz=timezone.utc)
+epoch = datetime.fromtimestamp(timestamp=0, tz=UTC)
-def unix_time_millis(dt):
+def unix_time_millis(dt: datetime):
return int((dt - epoch).total_seconds() * 1000.0)
-def bootstrap_todos(todo_list):
+def bootstrap_todos(todo_list: list[Any]):
# Establish links from commands to to_do for finding todo vars
for tg in todo_list:
if dry_run:
@@ -248,10 +248,11 @@ def maybe_remove_rc_from_svn():
# To be able to hide fields when dumping Yaml
class SecretYamlObject(yaml.YAMLObject):
- hidden_fields = []
+ hidden_fields: list[str] = []
@classmethod
- def to_yaml(cls, dumper, data):
+ @override
+ def to_yaml(cls: type[Self], dumper: yaml.Dumper, data: Any):
print("Dumping object %s" % type(data))
new_data = copy.deepcopy(data)
@@ -261,27 +262,27 @@ class SecretYamlObject(yaml.YAMLObject):
for item in data.__dict__:
if item in new_data.__dict__ and new_data.__dict__[item] is None:
del new_data.__dict__[item]
- return dumper.represent_yaml_object(cls.yaml_tag, new_data, cls, flow_style=cls.yaml_flow_style)
+ return dumper.represent_yaml_object(cls.yaml_tag, new_data, cls, flow_style=cls.yaml_flow_style) # TODO: fix me # pyright: ignore[reportUnknownMemberType]
-def str_presenter(dumper, data):
+def str_presenter(dumper: yaml.Dumper, data: str):
if len(data.split("\n")) > 1: # check for multiline string
- return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
- return dumper.represent_scalar("tag:yaml.org,2002:str", data)
+ return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") # TODO: fix me # pyright: ignore[reportUnknownMemberType]
+ return dumper.represent_scalar("tag:yaml.org,2002:str", data) # TODO: fix me # pyright: ignore[reportUnknownMemberType]
class ReleaseState:
- def __init__(self, config_path, release_version, script_version):
+ def __init__(self, config_path: str, release_version: str, script_version: str):
self.script_version = script_version
self.config_path = config_path
self.todo_groups: list[TodoGroup] = []
self.todos: dict[str, Todo] = {}
- self.latest_version = None
- self.previous_rcs = {}
- self.rc_number = 1
- self.start_date = unix_time_millis(datetime.now(tz=timezone.utc))
- self.script_branch = run("git rev-parse --abbrev-ref HEAD").strip()
- self.mirrored_versions = None
+ self.latest_version: str | None = None
+ self.previous_rcs: dict[str, Any] = {}
+ self.rc_number: int = 1
+ self.start_date = unix_time_millis(datetime.now(tz=UTC))
+ self.script_branch: str = run("git rev-parse --abbrev-ref HEAD").strip()
+ self.mirrored_versions: list[str] | None = None
try:
self.script_branch_type = scriptutil.find_branch_type()
except Exception:
@@ -289,7 +290,7 @@ class ReleaseState:
self.script_branch_type = "feature"
self.set_release_version(release_version)
- def set_release_version(self, version):
+ def set_release_version(self, version: str):
self.validate_release_version(self.script_branch_type, self.script_branch, version)
self.release_version = version
v = Version.parse(version)
@@ -314,23 +315,20 @@ class ReleaseState:
assert gpg_task
if gpg_task.is_done():
return gpg_task.get_state()["gpg_key"]
- else:
- return None
+ return None
def get_release_date(self):
publish_task = self.get_todo_by_id("publish_maven")
assert publish_task
if publish_task.is_done():
return unix_to_datetime(publish_task.get_state()["done_date"])
- else:
- return None
+ return None
def get_release_date_iso(self):
release_date = self.get_release_date()
if release_date is None:
return "yyyy-mm-dd"
- else:
- return release_date.isoformat()[:10]
+ return release_date.isoformat()[:10]
def get_latest_version(self):
if self.latest_version is None:
@@ -341,12 +339,13 @@ class ReleaseState:
latest = ver
self.latest_version = latest
self.save()
+ assert state.latest_version
return state.latest_version
def get_mirrored_versions(self):
if state.mirrored_versions is None:
releases_str = load("https://projects.apache.org/json/foundation/releases.json", "utf-8")
- releases = json.loads(releases_str)["lucene"]
+ releases: dict[str, str] = json.loads(releases_str)["lucene"]
state.mirrored_versions = [r for r in list(map(lambda y: y[7:], filter(lambda x: x.startswith("lucene-"), list(releases.keys()))))]
return state.mirrored_versions
@@ -381,7 +380,7 @@ class ReleaseState:
latest_lts = ver
return latest_lts
- def validate_release_version(self, branch_type, branch, release_version):
+ def validate_release_version(self, branch_type: BranchType | str, branch: str, release_version: str):
ver = Version.parse(release_version)
# print("release_version=%s, ver=%s" % (release_version, ver))
if branch_type == BranchType.release:
@@ -406,12 +405,11 @@ class ReleaseState:
v = Version.parse(self.release_version)
if v.is_major_release():
return "main"
- elif v.is_minor_release():
+ if v.is_minor_release():
return self.get_stable_branch_name()
- elif v.major == Version.parse(self.get_latest_version()).major:
+ if v.major == Version.parse(self.get_latest_version()).major:
return self.get_minor_branch_name()
- else:
- return self.release_branch
+ return self.release_branch
def clear_rc(self):
if ask_yes_no("Are you sure? This will clear and restart RC%s" % self.rc_number):
@@ -445,7 +443,7 @@ class ReleaseState:
for todo_id in self.todos:
t = self.todos[todo_id]
tmp_todos[todo_id] = copy.deepcopy(t.state)
- dict = {
+ dictionary: dict[str, Any] = {
"script_version": self.script_version,
"release_version": self.release_version,
"start_date": self.start_date,
@@ -455,33 +453,30 @@ class ReleaseState:
"previous_rcs": self.previous_rcs,
}
if self.latest_version:
- dict["latest_version"] = self.latest_version
- return dict
-
- def restore_from_dict(self, dict):
- self.script_version = dict["script_version"]
- assert dict["release_version"] == self.release_version
- if "start_date" in dict:
- self.start_date = dict["start_date"]
- if "latest_version" in dict:
- self.latest_version = dict["latest_version"]
- else:
- self.latest_version = None
- self.rc_number = dict["rc_number"]
- self.script_branch = dict["script_branch"]
- self.previous_rcs = copy.deepcopy(dict["previous_rcs"])
- for todo_id in dict["todos"]:
+ dictionary["latest_version"] = self.latest_version
+ return dictionary
+
+ def restore_from_dict(self, dictionary: dict[str, Any]):
+ self.script_version = dictionary["script_version"]
+ assert dictionary["release_version"] == self.release_version
+ if "start_date" in dictionary:
+ self.start_date = dictionary["start_date"]
+ self.latest_version = dictionary.get("latest_version")
+ self.rc_number = dictionary["rc_number"]
+ self.script_branch = dictionary["script_branch"]
+ self.previous_rcs = copy.deepcopy(dictionary["previous_rcs"])
+ for todo_id in dictionary["todos"]:
if todo_id in self.todos:
t = self.todos[todo_id]
- for k in dict["todos"][todo_id]:
- t.state[k] = dict["todos"][todo_id][k]
+ for k in dictionary["todos"][todo_id]:
+ t.state[k] = dictionary["todos"][todo_id][k]
else:
print("Warning: Could not restore state for %s, Todo definition not found" % todo_id)
def load(self):
if os.path.exists(os.path.join(self.config_path, self.release_version, "state.yaml")):
state_file = os.path.join(self.config_path, self.release_version, "state.yaml")
- with open(state_file, "r") as fp:
+ with open(state_file) as fp:
try:
dict = yaml.load(fp, Loader=yaml.Loader)
self.restore_from_dict(dict)
@@ -515,26 +510,23 @@ class ReleaseState:
except Exception:
return "<git-rev>"
- def get_group_by_id(self, id):
- lst = list(filter(lambda x: x.id == id, self.todo_groups))
+ def get_group_by_id(self, id: str):
+ lst: list[TodoGroup] = list(filter(lambda x: x.id == id, self.todo_groups))
if len(lst) == 1:
return lst[0]
- else:
- return None
+ return None
- def get_todo_by_id(self, id):
- lst = list(filter(lambda x: x.id == id, self.todos.values()))
+ def get_todo_by_id(self, id: str):
+ lst: list[Todo] = list(filter(lambda x: x.id == id, self.todos.values()))
if len(lst) == 1:
return lst[0]
- else:
- return None
+ return None
- def get_todo_state_by_id(self, id):
- lst = list(filter(lambda x: x.id == id, self.todos.values()))
+ def get_todo_state_by_id(self, id: str):
+ lst: list[Todo] = list(filter(lambda x: x.id == id, self.todos.values()))
if len(lst) == 1:
return lst[0].state
- else:
- return {}
+ return cast(dict[Any, Any], {})
def get_release_folder(self):
folder = os.path.join(self.config_path, self.release_version)
@@ -564,11 +556,8 @@ class ReleaseState:
def get_minor_branch_name(self):
latest = state.get_latest_version()
- if latest is not None:
- v = Version.parse(latest)
- return "branch_%s_%s" % (v.major, v.minor)
- else:
- raise Exception("Cannot find latest version")
+ v = Version.parse(latest)
+ return "branch_%s_%s" % (v.major, v.minor)
def get_stable_branch_name(self):
if self.release_type == "major":
@@ -589,30 +578,29 @@ class ReleaseState:
def get_java_home(self):
return self.get_java_home_for_version(self.release_version)
- def get_java_home_for_version(self, version):
+ def get_java_home_for_version(self, version: str):
v = Version.parse(version)
java_ver = java_versions[v.major]
java_home_var = "JAVA%s_HOME" % java_ver
if java_home_var in os.environ:
return os.environ[java_home_var]
- else:
- raise Exception("Script needs environment variable %s" % java_home_var)
+ raise Exception("Script needs environment variable %s" % java_home_var)
- def get_java_cmd_for_version(self, version):
+ def get_java_cmd_for_version(self, version: str):
return os.path.join(self.get_java_home_for_version(version), "bin", "java")
def get_java_cmd(self):
return os.path.join(self.get_java_home(), "bin", "java")
def get_todo_states(self):
- states = {}
+ states: dict[str, dict[Any, Any]] = {}
if self.todos:
for todo_id in self.todos:
t = self.todos[todo_id]
states[todo_id] = copy.deepcopy(t.state)
return states
- def init_todos(self, groups):
+ def init_todos(self, groups: list[Any]):
self.todo_groups = groups
self.todos = {}
for g in self.todo_groups:
@@ -624,7 +612,7 @@ class TodoGroup(SecretYamlObject):
yaml_tag = "!TodoGroup"
hidden_fields = []
- def __init__(self, id, title, description, todos, is_in_rc_loop=None, depends=None):
+ def __init__(self, id: str, title: str, description: str, todos: list[Any], is_in_rc_loop: bool | None = None, depends: str | list[str] | None = None):
self.id = id
self.title = title
self.description = description
@@ -633,9 +621,10 @@ class TodoGroup(SecretYamlObject):
self.todos = todos
@classmethod
- def from_yaml(cls, loader, node):
+ @override
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
fields = loader.construct_mapping(node, deep=True)
- return TodoGroup(**fields)
+ return TodoGroup(**cast(dict[str, Any], fields))
def num_done(self):
return sum(1 for x in self.todos if x.is_done() > 0)
@@ -678,8 +667,7 @@ class TodoGroup(SecretYamlObject):
desc = self.description
if desc:
return expand_jinja(desc)
- else:
- return None
+ return None
def get_subtitle(self) -> str:
if self.depends:
@@ -700,26 +688,25 @@ class Todo(SecretYamlObject):
def __init__(
self,
- id,
- title,
- description=None,
- post_description=None,
- done=None,
- types=None,
- links=None,
- commands=None,
- user_input=None,
- depends=None,
- vars=None,
- asciidoc=None,
- persist_vars=None,
- function=None,
+ id: str,
+ title: str,
+ description: str | None = None,
+ post_description: str | None = None,
+ done: bool | None = None,
+ types: list[str] | str | None = None,
+ links: list[str] | None = None,
+ commands: Any = None,
+ user_input: Any = None,
+ depends: str | list[str] | None = None,
+ vars: dict[str, Any] | None = None,
+ asciidoc: str | None = None,
+ persist_vars: dict[str, str] | None = None,
+ function: str | None = None,
):
self.id = id
self.title = title
self.description = description
self.asciidoc = asciidoc
- self.types = types
self.depends = depends
self.vars = vars
self.persist_vars = persist_vars
@@ -728,38 +715,40 @@ class Todo(SecretYamlObject):
self.commands = commands
self.post_description = post_description
self.links = links
- self.state = {}
+ self.state: dict[Any, Any] = {}
self.set_done(done)
- if self.types:
- self.types = ensure_list(self.types)
- for t in self.types:
- if t not in ["minor", "major", "bugfix"]:
- sys.exit("Wrong Todo config for '%s'. Type needs to be either 'minor', 'major' or 'bugfix'" % self.id)
+ self.types: list[str] = ensure_list(types) if types else []
+
+ for t in self.types:
+ if t not in ["minor", "major", "bugfix"]:
+ sys.exit("Wrong Todo config for '%s'. Type needs to be either 'minor', 'major' or 'bugfix'" % self.id)
+
if self.commands:
self.commands.todo_id = self.id
for c in self.commands.commands:
c.todo_id = self.id
@classmethod
- def from_yaml(cls, loader, node):
+ @override
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
fields = loader.construct_mapping(node, deep=True)
- return Todo(**fields)
+ return Todo(**cast(dict[str, Any], fields))
def get_vars(self):
- myvars = {}
+ myvars: dict[str, str] = {}
if self.vars:
for k in self.vars:
val = self.vars[k]
if callable(val):
- myvars[k] = expand_jinja(val(), vars=myvars)
+ myvars[k] = expand_jinja(str(val()), vars=myvars)
else:
myvars[k] = expand_jinja(val, vars=myvars)
return myvars
- def set_done(self, is_done):
+ def set_done(self, is_done: bool | None):
if is_done:
- self.state["done_date"] = unix_time_millis(datetime.now(tz=timezone.utc))
+ self.state["done_date"] = unix_time_millis(datetime.now(tz=UTC))
if self.persist_vars:
for k in self.persist_vars:
self.state[k] = self.get_vars()[k]
@@ -767,7 +756,7 @@ class Todo(SecretYamlObject):
self.state.clear()
self.state["done"] = is_done
- def applies(self, type):
+ def applies(self, type: str):
if self.types:
return type in self.types
return True
@@ -856,14 +845,12 @@ class Todo(SecretYamlObject):
desc = self.description
if desc:
return expand_jinja(desc, vars=self.get_vars_and_state())
- else:
- return None
+ return None
def get_post_description(self):
if self.post_description:
return expand_jinja(self.post_description, vars=self.get_vars_and_state())
- else:
- return None
+ return None
def get_commands(self):
cmds = self.commands
@@ -872,8 +859,7 @@ class Todo(SecretYamlObject):
def get_asciidoc(self):
if self.asciidoc:
return expand_jinja(self.asciidoc, vars=self.get_vars_and_state())
- else:
- return None
+ return None
def get_vars_and_state(self):
d = self.get_vars().copy()
@@ -887,7 +873,7 @@ def get_release_version():
version = Version.parse(v)
except Exception:
print("Not a valid version %s" % v)
- return get_release_version()
+ raise
return str(version)
@@ -923,7 +909,7 @@ def reset_state():
state.clear()
-def template(name, vars=None):
+def template(name: str, vars: dict[str, Any] | None = None):
return expand_jinja(templates[name], vars=vars)
@@ -932,16 +918,15 @@ def help():
pause()
-def ensure_list(o):
+def ensure_list(o: Any):
if o is None:
- return []
+ return cast(list[Any], [])
if not isinstance(o, list):
return [o]
- else:
- return o
+ return cast(list[Any], o)
-def open_file(filename):
+def open_file(filename: str):
print("Opening file %s" % filename)
if platform.system().startswith("Win"):
run("start %s" % filename)
@@ -949,12 +934,12 @@ def open_file(filename):
run("open %s" % filename)
-def expand_multiline(cmd_txt, indent=0):
+def expand_multiline(cmd_txt: str, indent: int = 0):
return re.sub(r" +", " %s\n %s" % (Commands.cmd_continuation_char, " " * indent), cmd_txt)
-def unix_to_datetime(unix_stamp):
- return datetime.fromtimestamp(timestamp=unix_stamp / 1000, tz=timezone.utc)
+def unix_to_datetime(unix_stamp: int):
+ return datetime.fromtimestamp(timestamp=unix_stamp / 1000, tz=UTC)
def generate_asciidoc():
@@ -965,7 +950,7 @@ def generate_asciidoc():
fh = open(filename_adoc, "w")
fh.write("= Lucene Release %s\n\n" % state.release_version)
- fh.write("(_Generated by releaseWizard.py v%s at %s_)\n\n" % (getScriptVersion(), datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC")))
+ fh.write("(_Generated by releaseWizard.py v%s at %s_)\n\n" % (getScriptVersion(), datetime.now(tz=UTC).strftime("%Y-%m-%d %H:%M UTC")))
fh.write(":numbered:\n\n")
fh.write("%s\n\n" % template("help"))
for group in state.todo_groups:
@@ -992,8 +977,9 @@ def generate_asciidoc():
fh.write(".Variables collected in this step\n")
fh.write("|===\n")
fh.write("|Variable |Value\n")
- mykeys = set()
- for e in ensure_list(todo.user_input):
+ mykeys: set[str] = set()
+ myinputs: list[UserInput] = ensure_list(todo.user_input)
+ for e in myinputs:
mykeys.add(e.name)
for e in state_copy.keys():
mykeys.add(e)
@@ -1016,7 +1002,7 @@ def generate_asciidoc():
else:
fh.write("export %s=%s\n" % (key, val))
fh.write(abbreviate_homedir("cd %s\n" % cmds.get_root_folder()))
- cmds2 = ensure_list(cmds.commands)
+ cmds2: list[Command] = ensure_list(cmds.commands)
for c in cmds2:
for line in c.display_cmd():
fh.write("%s\n" % line)
@@ -1043,13 +1029,13 @@ def generate_asciidoc():
def load_rc():
lucenerc = os.path.expanduser("~/.lucenerc")
try:
- with open(lucenerc, "r") as fp:
- return json.load(fp)
+ with open(lucenerc) as fp:
+ return cast(dict[str, Any], json.load(fp))
except Exception:
- return {}
+ return cast(dict[str, Any], {})
-def store_rc(release_root, release_version=None):
+def store_rc(release_root: str, release_version: str | None = None):
lucenerc = os.path.expanduser("~/.lucenerc")
dict = {}
dict["root"] = release_root
@@ -1067,7 +1053,7 @@ def release_other_version():
sys.exit(0)
-def file_to_string(filename):
+def file_to_string(filename: str):
with open(filename, encoding="utf8") as f:
return f.read().strip()
@@ -1083,8 +1069,8 @@ def keys_downloaded():
def dump_yaml():
file = open(os.path.join(script_path, "releaseWizard.yaml"), "w")
yaml.add_representer(str, str_presenter)
- yaml.Dumper.ignore_aliases = lambda self, data: True
- dump_obj = {"templates": templates, "groups": state.todo_groups}
+ yaml.Dumper.ignore_aliases = lambda self, data: True # TODO: fix me # pyright: ignore[reportUnknownLambdaType]
+ dump_obj: dict[str, Any] = {"templates": templates, "groups": state.todo_groups}
yaml.dump(dump_obj, width=180, stream=file, sort_keys=False, default_flow_style=False)
@@ -1098,7 +1084,7 @@ def parse_config():
return config
-def load(urlString, encoding="utf-8"):
+def load(urlString: str, encoding: str = "utf-8"):
try:
content = urllib.request.urlopen(urlString).read().decode(encoding)
except Exception as e:
@@ -1107,7 +1093,7 @@ def load(urlString, encoding="utf-8"):
return content
-def configure_pgp(gpg_todo):
+def configure_pgp(gpg_todo: Todo):
print("Based on your Apache ID we'll lookup your key online\nand through this complete the 'gpg' prerequisite task.\n")
gpg_state = gpg_todo.get_state()
id = str(input("Please enter your Apache id: (ENTER=skip) "))
@@ -1137,7 +1123,7 @@ def configure_pgp(gpg_todo):
gpg_fingerprint = str(input("Enter your key fingerprint manually, all 40 characters (ENTER=skip): "))
if gpg_fingerprint.strip() == "":
return False
- elif len(gpg_fingerprint) != 40:
+ if len(gpg_fingerprint) != 40:
print("gpg fingerprint must be 40 characters long, do not just input the last 8")
gpg_fingerprint = gpg_fingerprint.upper()
gpg_id = gpg_fingerprint[-8:]
@@ -1252,16 +1238,18 @@ def configure_pgp(gpg_todo):
return True
-def pause(fun=None):
+def pause(fun: Callable[[], None] | None = None):
if fun:
fun()
input("\nPress ENTER to continue...")
class CustomExitItem(ExitItem):
- def show(self, index, available_width=None):
+ @override
+ def show(self, index: int, available_width: int | None = None):
return super(CustomExitItem, self).show(index)
+ @override
def get_return(self):
return ""
@@ -1314,7 +1302,7 @@ def main():
check_prerequisites()
try:
- y = yaml.load(open(os.path.join(script_path, "releaseWizard.yaml"), "r"), Loader=yaml.Loader)
+ y = yaml.load(open(os.path.join(script_path, "releaseWizard.yaml")), Loader=yaml.Loader)
templates = y.get("templates")
todo_list = y.get("groups")
state = ReleaseState(release_root, release_version, getScriptVersion())
@@ -1335,8 +1323,8 @@ def main():
main_menu = ConsoleMenu(
title="Lucene ReleaseWizard",
subtitle=get_releasing_text,
- prologue_text="Welcome to the release wizard. From here you can manage the process including creating new RCs. "
- "All changes are persisted, so you can exit any time and continue later. Make sure to read the Help section.",
+ prologue_text="""Welcome to the release wizard. From here you can manage the process including creating new RCs.
+ All changes are persisted, so you can exit any time and continue later. Make sure to read the Help section.""",
epilogue_text="® 2022 The Lucene project. Licensed under the Apache License 2.0\nScript version v%s)" % getScriptVersion(),
clear_screen=False,
)
@@ -1376,20 +1364,18 @@ def git_checkout_folder():
return state.get_git_checkout_folder()
-def tail_file(file, lines):
+def tail_file(file: str, lines: int):
bufsize = 8192
fsize = os.stat(file).st_size
with open(file) as f:
- if bufsize >= fsize:
- bufsize = fsize
+ bufsize = min(fsize, bufsize)
idx = 0
while True:
idx += 1
seek_pos = fsize - bufsize * idx
- if seek_pos < 0:
- seek_pos = 0
+ seek_pos = max(seek_pos, 0)
f.seek(seek_pos)
- data = []
+ data: list[str] = []
data.extend(f.readlines())
if len(data) >= lines or f.tell() == 0 or seek_pos == 0:
if not seek_pos == 0:
@@ -1398,7 +1384,7 @@ def tail_file(file, lines):
break
-def run_with_log_tail(command, cwd, logfile=None, tail_lines=10, tee=False, live=False, shell=None):
+def run_with_log_tail(command: str | list[str], cwd: str | None, logfile: str | None = None, tail_lines: int = 10, tee: bool | None = False, live: bool | None = False, shell: bool | None = None):
fh = sys.stdout
if logfile:
logdir = os.path.dirname(logfile)
@@ -1413,7 +1399,7 @@ def run_with_log_tail(command, cwd, logfile=None, tail_lines=10, tee=False, live
return rc
-def ask_yes_no(text):
+def ask_yes_no(text: str):
answer = None
while answer not in ["y", "n"]:
answer = str(input("\nQ: %s (y/n): " % text))
@@ -1421,7 +1407,7 @@ def ask_yes_no(text):
return answer == "y"
-def abbreviate_line(line, width):
+def abbreviate_line(line: str, width: int):
line = line.rstrip()
if len(line) > width:
line = "%s.....%s" % (line[: (width / 2 - 5)], line[-(width / 2) :])
@@ -1430,18 +1416,17 @@ def abbreviate_line(line, width):
return line
-def print_line_cr(line, linenum, stdout=True, tee=False):
+def print_line_cr(line: str, linenum: int, stdout: bool = True, tee: bool | None = False):
if not tee:
if not stdout:
print("[line %s] %s" % (linenum, abbreviate_line(line, 80)), end="\r")
+ elif line.endswith("\r"):
+ print(line.rstrip(), end="\r")
else:
- if line.endswith("\r"):
- print(line.rstrip(), end="\r")
- else:
- print(line.rstrip())
+ print(line.rstrip())
-def run_follow(command, cwd=None, fh=sys.stdout, tee=False, live=False, shell=None):
+def run_follow(command: str | list[str], cwd: str | None = None, fh: TextIO = sys.stdout, tee: bool | None = False, live: bool | None = False, shell: bool | None = None):
doShell = "&&" in command or "&" in command or shell is not None
if not doShell and not isinstance(command, list):
command = shlex.split(command)
@@ -1457,7 +1442,7 @@ def run_follow(command, cwd=None, fh=sys.stdout, tee=False, live=False, shell=No
fcntl.fcntl(process.stderr, fcntl.F_SETFL, flerr | os.O_NONBLOCK)
endstdout = endstderr = False
- errlines = []
+ errlines: list[str] = []
while not (endstderr and endstdout):
lines_before = lines_written
if not endstdout:
@@ -1536,7 +1521,18 @@ class Commands(SecretYamlObject):
cmd_continuation_char = "^" if is_windows() else "\\"
def __init__(
- self, root_folder, commands_text=None, commands=[], logs_prefix=None, run_text=None, enable_execute=None, confirm_each_command=None, env=None, vars=None, todo_id=None, remove_files=None
+ self,
+ root_folder: str,
+ commands_text: str | None = None,
+ commands: list[Any] | None = None,
+ logs_prefix: str | None = None,
+ run_text: str | None = None,
+ enable_execute: bool | None = None,
+ confirm_each_command: bool | None = None,
+ env: dict[str, str] | None = None,
+ vars: dict[str, Any] | None = None,
+ todo_id: str | None = None,
+ remove_files: list[str] | None = None,
):
self.root_folder = root_folder
self.commands_text = commands_text
@@ -1548,14 +1544,15 @@ class Commands(SecretYamlObject):
self.logs_prefix = logs_prefix
self.enable_execute = enable_execute
self.confirm_each_command = confirm_each_command
- self.commands = commands
+ self.commands: list[Command] = commands if commands else []
for c in self.commands:
c.todo_id = todo_id
@classmethod
- def from_yaml(cls, loader, node):
+ @override
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
fields = loader.construct_mapping(node, deep=True)
- return Commands(**fields)
+ return Commands(**cast(dict[str, Any], fields))
def run(self): # pylint: disable=inconsistent-return-statements # TODO
root = self.get_root_folder()
@@ -1571,7 +1568,7 @@ class Commands(SecretYamlObject):
else:
print("\n export %s=%s" % (key, val))
print(abbreviate_homedir("\n cd %s" % root))
- commands = ensure_list(self.commands)
+ commands: list[Command] = ensure_list(self.commands)
for cmd in commands:
for line in cmd.display_cmd():
print(" %s" % line)
@@ -1661,14 +1658,12 @@ class Commands(SecretYamlObject):
print("WARN: Command %s returned with error" % cmd.get_cmd())
success = False
break
- else:
- if cmd.should_fail and not dry_run:
- print("Expected command to fail, but it succeeded.")
- success = False
- break
- else:
- if elapsed > 30:
- print("Command completed in %s seconds" % elapsed)
+ elif cmd.should_fail and not dry_run:
+ print("Expected command to fail, but it succeeded.")
+ success = False
+ break
+ elif elapsed > 30:
+ print("Command completed in %s seconds" % elapsed)
if not success:
print("WARNING: One or more commands failed, you may want to check the logs")
return success
@@ -1686,17 +1681,17 @@ class Commands(SecretYamlObject):
return self.jinjaify(self.remove_files)
def get_vars(self):
- myvars = {}
+ myvars: dict[str, str] = {}
if self.vars:
for k in self.vars:
val = self.vars[k]
if callable(val):
- myvars[k] = expand_jinja(val(), vars=myvars)
+ myvars[k] = expand_jinja(str(val()), vars=myvars)
else:
myvars[k] = expand_jinja(val, vars=myvars)
return myvars
- def jinjaify(self, data, join=False):
+ def jinjaify(self, data: str | list[str] | None, join: bool = False):
if not data:
return None
v = self.get_vars()
@@ -1707,32 +1702,43 @@ class Commands(SecretYamlObject):
if isinstance(data, list):
if join:
return expand_jinja(" ".join(data), v)
- else:
- res = []
- for rf in data:
- res.append(expand_jinja(rf, v))
- return res
- else:
- return expand_jinja(data, v)
+ res: list[str] = []
+ for rf in data:
+ res.append(expand_jinja(rf, v))
+ return res
+ return expand_jinja(data, v)
-def abbreviate_homedir(line):
+def abbreviate_homedir(line: str):
if is_windows():
if "HOME" in os.environ:
return re.sub(r"([^/]|\b)%s" % os.path.expanduser("~"), "\\1%HOME%", line)
- elif "USERPROFILE" in os.environ:
+ if "USERPROFILE" in os.environ:
return re.sub(r"([^/]|\b)%s" % os.path.expanduser("~"), "\\1%USERPROFILE%", line)
- else:
- return line
- else:
- return re.sub(r"([^/]|\b)%s" % os.path.expanduser("~"), "\\1~", line)
+ return line
+ return re.sub(r"([^/]|\b)%s" % os.path.expanduser("~"), "\\1~", line)
class Command(SecretYamlObject):
yaml_tag = "!Command"
hidden_fields = ["todo_id"]
- def __init__(self, cmd, cwd=None, stdout=None, logfile=None, tee=None, live=None, comment=None, vars=None, todo_id=None, should_fail=None, redirect=None, redirect_append=None, shell=None):
+ def __init__(
+ self,
+ cmd: str,
+ cwd: str | None = None,
+ stdout: bool | None = None,
+ logfile: str | None = None,
+ tee: bool | None = None,
+ live: bool | None = None,
+ comment: str | None = None,
+ vars: dict[str, Any] | None = None,
+ todo_id: str | None = None,
+ should_fail: bool | None = None,
+ redirect: str | None = None,
+ redirect_append: bool | None = None,
+ shell: bool | None = None,
+ ):
self.cmd = cmd
self.cwd = cwd
self.comment = comment
@@ -1760,34 +1766,38 @@ class Command(SecretYamlObject):
print("Command %s specifies 'redirect' and other out options at the same time. Using redirect only" % self.cmd)
@classmethod
- def from_yaml(cls, loader, node):
+ @override
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
fields = loader.construct_mapping(node, deep=True)
- return Command(**fields)
+ return Command(**cast(dict[str, Any], fields))
def get_comment(self):
- return self.jinjaify(self.comment)
+ return str(self.jinjaify(self.comment))
def get_redirect(self):
- return self.jinjaify(self.redirect)
+ return str(self.jinjaify(self.redirect))
def get_cmd(self):
- return cast(str, self.jinjaify(self.cmd, join=True))
+ return str(self.jinjaify(self.cmd, join=True))
def get_vars(self):
- myvars = {}
+ myvars: dict[str, str] = {}
if self.vars:
for k in self.vars:
val = self.vars[k]
if callable(val):
- myvars[k] = expand_jinja(val(), vars=myvars)
+ myvars[k] = expand_jinja(str(val()), vars=myvars)
else:
myvars[k] = expand_jinja(val, vars=myvars)
return myvars
+ @override
def __str__(self):
return self.get_cmd()
- def jinjaify(self, data, join=False):
+ def jinjaify(self, data: str | list[str] | None, join: bool = False):
+ if data is None:
+ return None
v = self.get_vars()
if self.todo_id:
todo = state.get_todo_by_id(self.todo_id)
@@ -1796,16 +1806,14 @@ class Command(SecretYamlObject):
if isinstance(data, list):
if join:
return expand_jinja(" ".join(data), v)
- else:
- res = []
- for rf in data:
- res.append(expand_jinja(rf, v))
- return res
- else:
- return expand_jinja(data, v)
+ res: list[str] = []
+ for rf in data:
+ res.append(expand_jinja(rf, v))
+ return res
+ return expand_jinja(data, v)
def display_cmd(self):
- lines = []
+ lines: list[str] = []
if self.comment:
if is_windows():
lines.append("REM %s" % self.get_comment())
@@ -1826,17 +1834,18 @@ class Command(SecretYamlObject):
class UserInput(SecretYamlObject):
yaml_tag = "!UserInput"
- def __init__(self, name, prompt, type=None):
+ def __init__(self, name: str, prompt: str, type: str | None = None):
self.type = type
self.prompt = prompt
self.name = name
@classmethod
- def from_yaml(cls, loader, node):
+ @override
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
fields = loader.construct_mapping(node, deep=True)
- return UserInput(**fields)
+ return UserInput(**cast(dict[str, Any], fields))
- def run(self, dict=None):
+ def run(self, dict: dict[str, Any] | None = None):
correct = False
while not correct:
try:
@@ -1852,7 +1861,7 @@ class UserInput(SecretYamlObject):
return result
-def create_ical(_todo): # pylint: disable=unused-argument
+def create_ical(_todo: Todo): # pylint: disable=unused-argument
if ask_yes_no("Do you want to add a Calendar reminder for the close vote time?"):
# TODO: this library has broken typing and seems unmaintained, replace?
c = Calendar() # pyright: ignore[reportArgumentType]
@@ -1868,9 +1877,9 @@ def create_ical(_todo): # pylint: disable=unused-argument
return True
-today = datetime.now(tz=timezone.utc).date()
+today = datetime.now(tz=UTC).date()
sundays = {(today + timedelta(days=x)): "Sunday" for x in range(10) if (today + timedelta(days=x)).weekday() == 6}
-y = datetime.now(tz=timezone.utc).year
+y = datetime.now(tz=UTC).year
years = [y, y + 1]
non_working = (
country_holidays("CA", years=years)
@@ -1885,13 +1894,13 @@ non_working = (
def vote_close_72h_date():
# Voting open at least 72 hours according to ASF policy
- return datetime.now(tz=timezone.utc) + timedelta(hours=73)
+ return datetime.now(tz=UTC) + timedelta(hours=73)
def vote_close_72h_holidays():
days = 0
day_offset = -1
- holidays = []
+ holidays: list[str] = []
# Warn RM about major holidays coming up that should perhaps extend the voting deadline
# Warning will be given for Sunday or a public holiday observed by 3 or more [CA, US, EN, DE, NO, IND, RU]
while days < 3:
@@ -1899,15 +1908,14 @@ def vote_close_72h_holidays():
d = today + timedelta(days=day_offset)
if not (d in sundays or (d in non_working and len(non_working[d]) >= 2)):
days += 1
+ elif d in sundays:
+ holidays.append("%s (Sunday)" % d)
else:
- if d in sundays:
- holidays.append("%s (Sunday)" % d)
- else:
- holidays.append("%s (%s)" % (d, non_working[d]))
+ holidays.append("%s (%s)" % (d, non_working[d]))
return holidays if len(holidays) > 0 else None
-def prepare_announce_lucene(_todo): # pylint: disable=unused-argument
+def prepare_announce_lucene(_todo: Todo): # pylint: disable=unused-argument
if not os.path.exists(lucene_news_file):
lucene_text = expand_jinja("(( template=announce_lucene ))")
with open(lucene_news_file, "w") as fp:
@@ -1918,7 +1926,7 @@ def prepare_announce_lucene(_todo): # pylint: disable=unused-argument
return True
-def check_artifacts_available(_todo): # pylint: disable=unused-argument
+def check_artifacts_available(_todo: Todo): # pylint: disable=unused-argument
cdnUrl = expand_jinja("https://dlcdn.apache.org/lucene/java/{{ release_version }}/lucene-{{ release_version }}-src.tgz.asc")
try:
load(cdnUrl)
@@ -1938,14 +1946,14 @@ def check_artifacts_available(_todo): # pylint: disable=unused-argument
return True
-def set_java_home(version):
+def set_java_home(version: str):
os.environ["JAVA_HOME"] = state.get_java_home_for_version(version)
os.environ["JAVACMD"] = state.get_java_cmd_for_version(version)
-def load_lines(file, from_line=0):
+def load_lines(file: str, from_line: int = 0):
if os.path.exists(file):
- with open(file, "r") as fp:
+ with open(file) as fp:
return fp.readlines()[from_line:]
else:
return ["<Please paste the announcement text here>\n"]
diff --git a/dev-tools/scripts/releasedJirasRegex.py b/dev-tools/scripts/releasedJirasRegex.py
index f6887bc9923..0dd5ccd6360 100755
--- a/dev-tools/scripts/releasedJirasRegex.py
+++ b/dev-tools/scripts/releasedJirasRegex.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -33,7 +32,7 @@ from scriptutil import Version
# does not find Bugzilla bugs or JIRAs not mentioned at the beginning of
# bullets or numbered entries.
#
-def print_released_jiras_regex(version, filename):
+def print_released_jiras_regex(version: str, filename: str):
release_boundary_re = re.compile(r"\s*====*\s+(.*)\s+===")
version_re = re.compile(r"%s(?:$|[^-])" % version)
bullet_re = re.compile(r"\s*(?:[-*]|\d+\.(?=(?:\s|(?:LUCENE)-)))(.*)")
@@ -45,28 +44,26 @@ def print_released_jiras_regex(version, filename):
under_requested_version = False
requested_version_found = False
more_jiras_on_next_line = False
- lucene_jiras = []
- with open(filename, "r") as changes:
+ lucene_jiras: list[str] = []
+ with open(filename) as changes:
for line in changes:
version_boundary = release_boundary_re.match(line)
if version_boundary is not None:
if under_requested_version:
break # No longer under the requested version - stop looking for JIRAs
- else:
- if version_re.search(version_boundary.group(1)):
- under_requested_version = True # Start looking for JIRAs
- requested_version_found = True
- else:
- if under_requested_version:
- bullet_match = bullet_re.match(line)
- if more_jiras_on_next_line or bullet_match is not None:
- content = line if bullet_match is None else bullet_match.group(1)
- jira_list_match = jira_list_re.match(content)
- if jira_list_match is not None:
- jira_match = jira_re.findall(jira_list_match.group(0))
- for jira in jira_match:
- lucene_jiras.append(jira.rsplit("-", 1)[-1])
- more_jiras_on_next_line = more_jiras_on_next_line_re.match(content)
+ if version_re.search(version_boundary.group(1)):
+ under_requested_version = True # Start looking for JIRAs
+ requested_version_found = True
+ elif under_requested_version:
+ bullet_match = bullet_re.match(line)
+ if more_jiras_on_next_line or bullet_match is not None:
+ content = line if bullet_match is None else bullet_match.group(1)
+ jira_list_match = jira_list_re.match(content)
+ if jira_list_match is not None:
+ jira_match = jira_re.findall(jira_list_match.group(0))
+ for jira in jira_match:
+ lucene_jiras.append(jira.rsplit("-", 1)[-1])
+ more_jiras_on_next_line = more_jiras_on_next_line_re.match(content)
if not requested_version_found:
raise Exception("Could not find %s in %s" % (version, filename))
print()
diff --git a/dev-tools/scripts/reproduceJenkinsFailures.py b/dev-tools/scripts/reproduceJenkinsFailures.py
index 372091869ef..37b85c971a0 100644
--- a/dev-tools/scripts/reproduceJenkinsFailures.py
+++ b/dev-tools/scripts/reproduceJenkinsFailures.py
@@ -79,7 +79,7 @@ def readConfig():
return parser.parse_args()
-def runOutput(cmd):
+def runOutput(cmd: str):
print("[repro] %s" % cmd)
try:
return subprocess.check_output(cmd.split(" "), universal_newlines=True).strip()
@@ -88,23 +88,23 @@ def runOutput(cmd):
# Remembers non-zero exit code in lastFailureCode unless rememberFailure==False
-def run(cmd, rememberFailure=True):
+def run(cmd: str, rememberFailure: bool = True):
global lastFailureCode
print("[repro] %s" % cmd)
code = os.system(cmd)
- if 0 != code and rememberFailure:
+ if code != 0 and rememberFailure:
print("\n[repro] Setting last failure code to %d\n" % code)
lastFailureCode = code
return code
-def fetchAndParseJenkinsLog(url, numRetries):
+def fetchAndParseJenkinsLog(url: str, numRetries: int) -> dict[str, str]:
global revisionFromLog
global branchFromLog
global antOptions
revisionFromLog = None
antOptions = ""
- tests = {}
+ tests: dict[str, str] = {}
print("[repro] Jenkins log URL: %s\n" % url)
try:
# HTTPS fails at certificate validation, see LUCENE-9412, PEP-476
@@ -137,50 +137,48 @@ def fetchAndParseJenkinsLog(url, numRetries):
print("[repro] Encountered IncompleteRead exception, pausing and then retrying...")
time.sleep(2) # pause for 2 seconds
return fetchAndParseJenkinsLog(url, numRetries - 1)
- else:
- print("[repro] Encountered IncompleteRead exception, aborting after too many retries.")
- raise RuntimeError("ERROR: fetching %s : %s" % (url, e))
+ print("[repro] Encountered IncompleteRead exception, aborting after too many retries.")
+ raise RuntimeError("ERROR: fetching %s : %s" % (url, e))
if revisionFromLog is None:
if reJenkinsURLWithoutConsoleText.match(url):
print('[repro] Not a Jenkins log. Appending "/consoleText" and retrying ...\n')
return fetchAndParseJenkinsLog(url + "/consoleText", numRetries)
- else:
- raise RuntimeError("ERROR: %s does not appear to be a Jenkins log." % url)
- if 0 == len(tests):
+ raise RuntimeError("ERROR: %s does not appear to be a Jenkins log." % url)
+ if len(tests) == 0:
print('[repro] No "reproduce with" lines found; exiting.')
sys.exit(0)
return tests
-def prepareWorkspace(useGit, gitRef):
+def prepareWorkspace(useGit: bool, gitRef: str | None):
global gitCheckoutSucceeded
if useGit:
code = run("git fetch")
- if 0 != code:
+ if code != 0:
raise RuntimeError('ERROR: "git fetch" failed. See above.')
checkoutCmd = "git checkout %s" % gitRef
code = run(checkoutCmd)
- if 0 != code:
+ if code != 0:
addWantedBranchCmd = "git remote set-branches --add origin %s" % gitRef
checkoutBranchCmd = "git checkout -t -b %s origin/%s" % (gitRef, gitRef) # Checkout remote branch as new local branch
print('"%s" failed. Trying "%s" and "%s".' % (checkoutCmd, addWantedBranchCmd, checkoutBranchCmd))
code = run(addWantedBranchCmd)
- if 0 != code:
+ if code != 0:
raise RuntimeError('ERROR: "%s" failed. See above.' % addWantedBranchCmd)
code = run(checkoutBranchCmd)
- if 0 != code:
+ if code != 0:
raise RuntimeError('ERROR: "%s" failed. See above.' % checkoutBranchCmd)
gitCheckoutSucceeded = True
run("git merge --ff-only", rememberFailure=False) # Ignore failure on non-branch ref
code = run("ant clean")
- if 0 != code:
+ if code != 0:
raise RuntimeError('ERROR: "ant clean" failed. See above.')
-def groupTestsByModule(tests):
- modules = {}
+def groupTestsByModule(tests: dict[str, str]):
+ modules: dict[str, set[str]] = {}
for dir, _, files in os.walk("."):
for file in files:
match = reJavaFile.search(file)
@@ -201,7 +199,7 @@ def groupTestsByModule(tests):
return modules
-def runTests(testIters, modules, tests):
+def runTests(testIters: int, modules: dict[str, set[str]], tests: dict[str, str]):
cwd = os.getcwd()
testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s %s'
for module in modules:
@@ -212,15 +210,15 @@ def runTests(testIters, modules, tests):
os.chdir(module)
code = run("ant compile-test")
try:
- if 0 != code:
+ if code != 0:
raise RuntimeError("ERROR: Compile failed in %s/ with code %d. See above." % (module, code))
run(testCmdline % (testIters, testIters * numTests, testList, antOptions, params))
finally:
os.chdir(cwd)
-def printAndMoveReports(testIters, newSubDir, location):
- failures = {}
+def printAndMoveReports(testIters: int, newSubDir: str, location: str):
+ failures: dict[str, int] = {}
for start in ("lucene/build", "solr/build"):
for dir, _, files in os.walk(start):
for file in files:
@@ -275,7 +273,7 @@ def main():
if config.useGit:
# Retest 100% failures at the tip of the branch
oldTests = tests
- tests = {}
+ tests: dict[str, str] = {}
for fullClass in failures:
testcase = fullClass[(fullClass.rindex(".") + 1) :]
if failures[fullClass] == config.testIters:
diff --git a/dev-tools/scripts/scriptutil.py b/dev-tools/scripts/scriptutil.py
index 4b054dfaa0c..01c7e9d81ba 100644
--- a/dev-tools/scripts/scriptutil.py
+++ b/dev-tools/scripts/scriptutil.py
@@ -22,11 +22,15 @@ import time
import urllib.error
import urllib.parse
import urllib.request
+from collections.abc import Callable
from enum import Enum
+from re import Match, Pattern
+from typing import Self, override
-class Version(object):
- def __init__(self, major, minor, bugfix, prerelease):
+class Version:
+ def __init__(self, major: int, minor: int, bugfix: int, prerelease: int):
+ super().__init__()
self.major = major
self.minor = minor
self.bugfix = bugfix
@@ -36,7 +40,7 @@ class Version(object):
self.constant = "LUCENE_%d_%d_%d" % (self.major, self.minor, self.bugfix)
@classmethod
- def parse(cls, value):
+ def parse(cls, value: str):
match = re.search(r"(\d+)\.(\d+).(\d+)(.1|.2)?", value)
if match is None:
raise argparse.ArgumentTypeError("Version argument must be of format x.y.z(.1|.2)?")
@@ -44,10 +48,11 @@ class Version(object):
parts.append({None: 0, ".1": 1, ".2": 2}[match.groups()[-1]])
return Version(*parts)
+ @override
def __str__(self):
return self.dot
- def make_previous_matcher(self, prefix="", suffix="", sep="\\."):
+ def make_previous_matcher(self, prefix: str = "", suffix: str = "", sep: str = "\\."):
if self.is_bugfix_release():
pattern = "%s%s%s%s%d" % (self.major, sep, self.minor, sep, self.bugfix - 1)
elif self.is_minor_release():
@@ -66,23 +71,21 @@ class Version(object):
def is_major_release(self):
return self.bugfix == 0 and self.minor == 0
- def on_or_after(self, other):
- return (
- self.major > other.major
- or self.major == other.major
- and (self.minor > other.minor or self.minor == other.minor and (self.bugfix > other.bugfix or self.bugfix == other.bugfix and self.prerelease >= other.prerelease))
+ def on_or_after(self, other: Self):
+ return self.major > other.major or (
+ self.major == other.major and (self.minor > other.minor or (self.minor == other.minor and (self.bugfix > other.bugfix or (self.bugfix == other.bugfix and self.prerelease >= other.prerelease))))
)
- def gt(self, other):
+ def gt(self, other: Self):
return self.major > other.major or (self.major == other.major and self.minor > other.minor) or (self.major == other.major and self.minor == other.minor and self.bugfix > other.bugfix)
- def is_back_compat_with(self, other):
+ def is_back_compat_with(self, other: Self):
if not self.on_or_after(other):
raise Exception("Back compat check disallowed for newer version: %s < %s" % (self, other))
return other.major + 1 >= self.major
-def run(cmd, cwd=None):
+def run(cmd: str, cwd: str | None = None):
try:
output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, cwd=cwd)
except subprocess.CalledProcessError as e:
@@ -91,9 +94,9 @@ def run(cmd, cwd=None):
return output.decode("utf-8")
-def update_file(filename, line_re, edit, append=None):
- infile = open(filename, "r")
- buffer = []
+def update_file(filename: str, line_re: Pattern[str], edit: Callable[[list[str], Match[str], str], bool | None], append: Callable[[list[str], bool], bool] | None = None):
+ infile = open(filename)
+ buffer: list[str] = []
changed = False
for line in infile:
@@ -139,7 +142,7 @@ def find_branch_type():
raise Exception("Cannot run %s on feature branch" % sys.argv[0].rsplit("/", 1)[-1])
-def download(name, urlString, tmpDir, quiet=False, force_clean=True):
+def download(name: str, urlString: str, tmpDir: str, quiet: bool = False, force_clean: bool = True):
if not quiet:
print("Downloading %s" % urlString)
startTime = time.time()
@@ -162,7 +165,7 @@ def download(name, urlString, tmpDir, quiet=False, force_clean=True):
print(" %.1f MB in %.2f sec (%.1f MB/sec)" % (sizeMB, t, sizeMB / t))
-def attemptDownload(urlString, fileName):
+def attemptDownload(urlString: str, fileName: str):
fIn = urllib.request.urlopen(urlString)
fOut = open(fileName, "wb")
success = False
diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py
index e29d95ca4ab..05f479c97a3 100755
--- a/dev-tools/scripts/smokeTestRelease.py
+++ b/dev-tools/scripts/smokeTestRelease.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -36,6 +35,8 @@ import urllib.request
import xml.etree.ElementTree as ET
import zipfile
from collections import namedtuple
+from collections.abc import Callable
+from typing import Any
import scriptutil
@@ -49,14 +50,14 @@ cygwin = platform.system().lower().startswith("cygwin")
cygwinWindowsRoot = os.popen("cygpath -w /").read().strip().replace("\\", "/") if cygwin else ""
-def unshortenURL(url):
+def unshortenURL(url: str):
parsed = urllib.parse.urlparse(url)
if parsed[0] in ("http", "https"):
h = http.client.HTTPConnection(parsed.netloc)
h.request("HEAD", parsed.path)
response = h.getresponse()
if int(response.status / 100) == 3 and response.getheader("Location"):
- return response.getheader("Location")
+ return str(response.getheader("Location"))
return url
@@ -70,7 +71,7 @@ reHREF = re.compile('<a href="(.*?)">(.*?)</a>')
FORCE_CLEAN = True
-def getHREFs(urlString):
+def getHREFs(urlString: str):
# Deref any redirects
while True:
url = urllib.parse.urlparse(urlString)
@@ -88,7 +89,7 @@ def getHREFs(urlString):
else:
break
- links = []
+ links: list[tuple[str, str]] = []
try:
html = load(urlString)
except:
@@ -102,7 +103,7 @@ def getHREFs(urlString):
return links
-def load(urlString):
+def load(urlString: str):
try:
content = urllib.request.urlopen(urlString).read().decode("utf-8")
except Exception as e:
@@ -111,14 +112,14 @@ def load(urlString):
return content
-def noJavaPackageClasses(desc, file):
+def noJavaPackageClasses(desc: str, file: str):
with zipfile.ZipFile(file) as z2:
for name2 in z2.namelist():
if name2.endswith(".class") and (name2.startswith("java/") or name2.startswith("javax/")):
raise RuntimeError('%s contains sheisty class "%s"' % (desc, name2))
-def decodeUTF8(bytes):
+def decodeUTF8(bytes: bytes):
return codecs.getdecoder("UTF-8")(bytes)[0]
@@ -127,7 +128,7 @@ NOTICE_FILE_NAME = "META-INF/NOTICE.txt"
LICENSE_FILE_NAME = "META-INF/LICENSE.txt"
-def checkJARMetaData(desc, jarFile, gitRevision, version):
+def checkJARMetaData(desc: str, jarFile: str, gitRevision: str, version: str):
with zipfile.ZipFile(jarFile, "r") as z:
for name in (MANIFEST_FILE_NAME, NOTICE_FILE_NAME, LICENSE_FILE_NAME):
try:
@@ -159,8 +160,7 @@ def checkJARMetaData(desc, jarFile, gitRevision, version):
else:
if len(verify) == 1:
raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFEST.MF: %s' % (desc, verify[0], s))
- else:
- raise RuntimeError('%s is missing one of "%s" inside its META-INF/MANIFEST.MF: %s' % (desc, verify, s))
+ raise RuntimeError('%s is missing one of "%s" inside its META-INF/MANIFEST.MF: %s' % (desc, verify, s))
if gitRevision != "skip":
# Make sure this matches the version and git revision we think we are releasing:
@@ -186,11 +186,11 @@ def checkJARMetaData(desc, jarFile, gitRevision, version):
raise RuntimeError("%s: %s contents doesn't match main LICENSE.txt" % (desc, LICENSE_FILE_NAME))
-def normSlashes(path):
+def normSlashes(path: str):
return path.replace(os.sep, "/")
-def checkAllJARs(topDir, gitRevision, version):
+def checkAllJARs(topDir: str, gitRevision: str, version: str):
print(" verify JAR metadata/identity/no javax.* or java.* classes...")
for root, _, files in os.walk(topDir):
normRoot = normSlashes(root)
@@ -205,24 +205,24 @@ def checkAllJARs(topDir, gitRevision, version):
checkJARMetaData('JAR file "%s"' % fullPath, fullPath, gitRevision, version)
-def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
+def checkSigs(urlString: str, version: str, tmpDir: str, isSigned: bool, keysFile: str):
print(" test basics...")
ents = getDirEntries(urlString)
artifact = None
changesURL = None
mavenURL = None
artifactURL = None
- expectedSigs = []
+ expectedSigs: list[str] = []
if isSigned:
expectedSigs.append("asc")
expectedSigs.extend(["sha512"])
- sigs = []
- artifacts = []
+ sigs: list[str] = []
+ artifacts: list[tuple[str, str]] = []
for text, subURL in ents:
if text == "KEYS":
raise RuntimeError("lucene: release dir should not contain a KEYS file - only toplevel /dist/lucene/KEYS is used")
- elif text == "maven/":
+ if text == "maven/":
mavenURL = subURL
elif text.startswith("changes"):
if text not in ("changes/", "changes-%s/" % version):
@@ -240,12 +240,15 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
else:
if sigs != expectedSigs:
raise RuntimeError("lucene: artifact %s has wrong sigs: expected %s but got %s" % (artifact, expectedSigs, sigs))
+ assert artifactURL is not None
artifacts.append((artifact, artifactURL))
artifact = text
artifactURL = subURL
sigs = []
if sigs != []:
+ assert artifact is not None
+ assert artifactURL is not None
artifacts.append((artifact, artifactURL))
if sigs != expectedSigs:
raise RuntimeError("lucene: artifact %s has wrong sigs: expected %s but got %s" % (artifact, expectedSigs, sigs))
@@ -302,7 +305,7 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
print(" GPG: %s" % line.strip())
-def testChanges(version, changesURLString):
+def testChanges(version: str, changesURLString: str):
print(" check changes HTML...")
changesURL = None
for text, subURL in getDirEntries(changesURLString):
@@ -316,7 +319,7 @@ def testChanges(version, changesURLString):
checkChangesContent(s, version, changesURL, True)
-def testChangesText(dir, version):
+def testChangesText(dir: str, version: str):
"Checks all CHANGES.txt under this dir."
for root, _, files in os.walk(dir):
# NOTE: O(N) but N should be smallish:
@@ -331,7 +334,7 @@ reUnderbarNotDashHTML = re.compile(r"<li>(\s*(LUCENE)_\d\d\d\d+)")
reUnderbarNotDashTXT = re.compile(r"\s+((LUCENE)_\d\d\d\d+)", re.MULTILINE)
-def checkChangesContent(s, version, name, isHTML):
+def checkChangesContent(s: str, version: str, name: str, isHTML: bool):
currentVersionTuple = versionToTuple(version, name)
if isHTML and s.find("Release %s" % version) == -1:
@@ -359,8 +362,8 @@ def checkChangesContent(s, version, name, isHTML):
if isHTML:
# Make sure that a section only appears once under each release,
# and that each release is not greater than the current version
- seenIDs = set()
- seenText = set()
+ seenIDs: set[str] = set()
+ seenText: set[str] = set()
release = None
for id, text in reChangesSectionHREF.findall(s):
@@ -381,7 +384,7 @@ def checkChangesContent(s, version, name, isHTML):
reVersion = re.compile(r"(\d+)\.(\d+)(?:\.(\d+))?\s*(-alpha|-beta|final|RC\d+)?\s*(?:\[.*\])?", re.IGNORECASE)
-def versionToTuple(version, name):
+def versionToTuple(version: str, name: str):
versionMatch = reVersion.match(version)
if versionMatch is None:
raise RuntimeError("Version %s in %s cannot be parsed" % (version, name))
@@ -402,7 +405,7 @@ def versionToTuple(version, name):
reUnixPath = re.compile(r'\b[a-zA-Z_]+=(?:"(?:\\"|[^"])*"' + "|(?:\\\\.|[^\"'\\s])*" + r"|'(?:\\'|[^'])*')" + r'|(/(?:\\.|[^"\'\s])*)' + r'|("/(?:\\.|[^"])*")' + r"|('/(?:\\.|[^'])*')")
-def unix2win(matchobj):
+def unix2win(matchobj: re.Match[str]):
if matchobj.group(1) is not None:
return cygwinWindowsRoot + matchobj.group()
if matchobj.group(2) is not None:
@@ -412,7 +415,7 @@ def unix2win(matchobj):
return matchobj.group()
-def cygwinifyPaths(command):
+def cygwinifyPaths(command: str):
# The problem: Native Windows applications running under Cygwin can't
# handle Cygwin's Unix-style paths. However, environment variable
# values are automatically converted, so only paths outside of
@@ -423,7 +426,7 @@ def cygwinifyPaths(command):
return command
-def printFileContents(fileName):
+def printFileContents(fileName: str):
# Assume log file was written in system's default encoding, but
# even if we are wrong, we replace errors ... the ASCII chars
# (which is what we mostly care about eg for the test seed) should
@@ -440,7 +443,7 @@ def printFileContents(fileName):
print()
-def run(command, logFile):
+def run(command: str, logFile: str):
if cygwin:
command = cygwinifyPaths(command)
if os.system("%s > %s 2>&1" % (command, logFile)):
@@ -450,7 +453,7 @@ def run(command, logFile):
raise RuntimeError('command "%s" failed; see log file %s' % (command, logPath))
-def verifyDigests(artifact, urlString, tmpDir):
+def verifyDigests(artifact: str, urlString: str, tmpDir: str):
print(" verify sha512 digest")
sha512Expected, t = load(urlString + ".sha512").strip().split()
if t != "*" + artifact:
@@ -469,18 +472,17 @@ def verifyDigests(artifact, urlString, tmpDir):
raise RuntimeError("SHA512 digest mismatch for %s: expected %s but got %s" % (artifact, sha512Expected, sha512Actual))
-def getDirEntries(urlString):
+def getDirEntries(urlString: str):
if urlString.startswith("file:/") and not urlString.startswith("file://"):
# stupid bogus ant URI
urlString = "file:///" + urlString[6:]
if urlString.startswith("file://"):
path = urlString[7:]
- if path.endswith("/"):
- path = path[:-1]
+ path = path.removesuffix("/")
if cygwin: # Convert Windows path to Cygwin path
path = re.sub(r"^/([A-Za-z]):/", r"/cygdrive/\1/", path)
- files = []
+ files: list[tuple[str, str]] = []
for ent in os.listdir(path):
entPath = "%s/%s" % (path, ent)
if os.path.isdir(entPath):
@@ -489,15 +491,14 @@ def getDirEntries(urlString):
files.append((ent, "file://%s" % entPath))
files.sort()
return files
- else:
- links = getHREFs(urlString)
- for i, (text, _) in enumerate(links):
- if text == "Parent Directory" or text == "..":
- return links[(i + 1) :]
- raise RuntimeError("could not enumerate %s" % (urlString))
+ links = getHREFs(urlString)
+ for i, (text, _) in enumerate(links):
+ if text == "Parent Directory" or text == "..":
+ return links[(i + 1) :]
+ raise RuntimeError("could not enumerate %s" % (urlString))
-def unpackAndVerify(java, tmpDir, artifact, gitRevision, version, testArgs):
+def unpackAndVerify(java: Any, tmpDir: str, artifact: str, gitRevision: str, version: str, testArgs: str):
destDir = "%s/unpack" % tmpDir
if os.path.exists(destDir):
shutil.rmtree(destDir)
@@ -525,7 +526,7 @@ LUCENE_NOTICE = None
LUCENE_LICENSE = None
-def is_in_list(in_folder, files, indent=4):
+def is_in_list(in_folder: list[str], files: list[str], indent: int = 4):
for fileName in files:
print("%sChecking %s" % (" " * indent, fileName))
found = False
@@ -537,7 +538,7 @@ def is_in_list(in_folder, files, indent=4):
raise RuntimeError('file "%s" is missing' % fileName)
-def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
+def verifyUnpacked(java: Any, artifact: str, unpackPath: str, gitRevision: str, version: str, testArgs: str):
global LUCENE_NOTICE
global LUCENE_LICENSE
@@ -547,7 +548,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
# Check text files in release
print(" %s" % artifact)
in_root_folder = list(filter(lambda x: x[0] != ".", os.listdir(unpackPath)))
- in_lucene_folder = []
+ in_lucene_folder: list[str] = []
if isSrc:
in_lucene_folder.extend(os.listdir(os.path.join(unpackPath, "lucene")))
is_in_list(in_root_folder, ["LICENSE", "NOTICE", "README"])
@@ -656,7 +657,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
testDemo(java.run_java, isSrc, version, BASE_JAVA_VERSION)
if java.run_alt_javas:
- for run_alt_java, alt_java_version in zip(java.run_alt_javas, java.alt_java_versions):
+ for run_alt_java, alt_java_version in zip(java.run_alt_javas, java.alt_java_versions, strict=False):
print(" run tests w/ Java %s and testArgs='%s'..." % (alt_java_version, testArgs))
run_alt_java("./gradlew --no-daemon test %s" % testArgs, "%s/test.log" % unpackPath)
print(" compile jars w/ Java %s" % alt_java_version)
@@ -671,13 +672,13 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
testDemo(java.run_java, isSrc, version, BASE_JAVA_VERSION)
if java.run_alt_javas:
- for run_alt_java, alt_java_version in zip(java.run_alt_javas, java.alt_java_versions):
+ for run_alt_java, alt_java_version in zip(java.run_alt_javas, java.alt_java_versions, strict=False):
testDemo(run_alt_java, isSrc, version, alt_java_version)
testChangesText(".", version)
-def testDemo(run_java, isSrc, version, jdk):
+def testDemo(run_java: Callable[[str, str], None], isSrc: bool, version: str, jdk: str):
if os.path.exists("index"):
shutil.rmtree("index") # nuke any index from any previous iteration
@@ -710,11 +711,10 @@ def testDemo(run_java, isSrc, version, jdk):
m = reMatchingDocs.search(open("search.log", encoding="UTF-8").read())
if m is None:
raise RuntimeError("lucene demo's SearchFiles found no results")
- else:
- numHits = int(m.group(1))
- if numHits < 100:
- raise RuntimeError("lucene demo's SearchFiles found too few results: %s" % numHits)
- print(' got %d hits for query "lucene"' % numHits)
+ numHits = int(m.group(1))
+ if numHits < 100:
+ raise RuntimeError("lucene demo's SearchFiles found too few results: %s" % numHits)
+ print(' got %d hits for query "lucene"' % numHits)
print(" checkindex with %s..." % jdk)
run_java(checkIndexCmd, "checkindex.log")
@@ -727,13 +727,13 @@ def testDemo(run_java, isSrc, version, jdk):
raise RuntimeError('wrong version from CheckIndex: got "%s" but expected "%s"' % (actualVersion, version))
-def removeTrailingZeros(version):
+def removeTrailingZeros(version: str):
return re.sub(r"(\.0)*$", "", version)
-def checkMaven(baseURL, tmpDir, gitRevision, version, isSigned, keysFile):
+def checkMaven(baseURL: str, tmpDir: str, gitRevision: str, version: str, isSigned: bool, keysFile: str):
print(" download artifacts")
- artifacts = []
+ artifacts: list[str] = []
artifactsURL = "%s/lucene/maven/org/apache/lucene/" % baseURL
targetDir = "%s/maven/org/apache/lucene" % tmpDir
if not os.path.exists(targetDir):
@@ -753,7 +753,7 @@ def checkMaven(baseURL, tmpDir, gitRevision, version, isSigned, keysFile):
checkAllJARs("%s/maven/org/apache/lucene" % tmpDir, gitRevision, version)
-def getBinaryDistFiles(tmpDir, version, baseURL):
+def getBinaryDistFiles(tmpDir: str, version: str, baseURL: str):
distribution = "lucene-%s.tgz" % version
if not os.path.exists("%s/%s" % (tmpDir, distribution)):
distURL = "%s/lucene/%s" % (baseURL, distribution)
@@ -767,13 +767,13 @@ def getBinaryDistFiles(tmpDir, version, baseURL):
print(" unpack %s..." % distribution)
unpackLogFile = "%s/unpack-%s-getBinaryDistFiles.log" % (tmpDir, distribution)
run("tar xzf %s/%s" % (tmpDir, distribution), unpackLogFile)
- distributionFiles = []
+ distributionFiles: list[str] = []
for root, _, files in os.walk(destDir):
distributionFiles.extend([os.path.join(root, file) for file in files])
return distributionFiles
-def checkJavadocAndSourceArtifacts(artifacts, version):
+def checkJavadocAndSourceArtifacts(artifacts: list[str], version: str):
print(" check for javadoc and sources artifacts...")
for artifact in artifacts:
if artifact.endswith(version + ".jar"):
@@ -785,8 +785,8 @@ def checkJavadocAndSourceArtifacts(artifacts, version):
raise RuntimeError("missing: %s" % sourcesJar)
-def getZipFileEntries(fileName):
- entries = []
+def getZipFileEntries(fileName: str):
+ entries: list[str] = []
with zipfile.ZipFile(fileName) as zf:
for zi in zf.infolist():
entries.append(zi.filename)
@@ -795,10 +795,10 @@ def getZipFileEntries(fileName):
return entries
-def checkIdenticalMavenArtifacts(distFiles, artifacts, version):
+def checkIdenticalMavenArtifacts(distFiles: list[str], artifacts: list[str], version: str):
print(" verify that Maven artifacts are same as in the binary distribution...")
reJarWar = re.compile(r"%s\.[wj]ar$" % version) # exclude *-javadoc.jar and *-sources.jar
- distFilenames = dict()
+ distFilenames: dict[str, str] = dict()
for file in distFiles:
baseName = os.path.basename(file)
distFilenames[baseName] = file
@@ -807,13 +807,12 @@ def checkIdenticalMavenArtifacts(distFiles, artifacts, version):
artifactFilename = os.path.basename(artifact)
if artifactFilename not in distFilenames:
raise RuntimeError("Maven artifact %s is not present in lucene binary distribution" % artifact)
- else:
- identical = filecmp.cmp(artifact, distFilenames[artifactFilename], shallow=False)
- if not identical:
- raise RuntimeError("Maven artifact %s is not identical to %s in lucene binary distribution" % (artifact, distFilenames[artifactFilename]))
+ identical = filecmp.cmp(artifact, distFilenames[artifactFilename], shallow=False)
+ if not identical:
+ raise RuntimeError("Maven artifact %s is not identical to %s in lucene binary distribution" % (artifact, distFilenames[artifactFilename]))
-def verifyMavenDigests(artifacts):
+def verifyMavenDigests(artifacts: list[str]):
print(" verify Maven artifacts' md5/sha1 digests...")
reJarWarPom = re.compile(r"\.(?:[wj]ar|pom)$")
for artifactFile in [a for a in artifacts if reJarWarPom.search(a)]:
@@ -843,23 +842,37 @@ def verifyMavenDigests(artifacts):
raise RuntimeError("SHA1 digest mismatch for %s: expected %s but got %s" % (artifactFile, sha1Expected, sha1Actual))
-def getPOMcoordinate(treeRoot):
+def getPOMcoordinate(treeRoot: ET.Element):
namespace = "{http://maven.apache.org/POM/4.0.0}"
groupId = treeRoot.find("%sgroupId" % namespace)
if groupId is None:
- groupId = treeRoot.find("{0}parent/{0}groupId".format(namespace))
+ groupId = treeRoot.find(f"{namespace}parent/{namespace}groupId")
+ assert groupId is not None
+ assert groupId.text is not None
groupId = groupId.text.strip()
- artifactId = treeRoot.find("%sartifactId" % namespace).text.strip()
+
+ artifactId = treeRoot.find("%sartifactId" % namespace)
+ assert artifactId is not None
+ assert artifactId.text is not None
+ artifactId = artifactId.text.strip()
+
version = treeRoot.find("%sversion" % namespace)
if version is None:
- version = treeRoot.find("{0}parent/{0}version".format(namespace))
+ version = treeRoot.find(f"{namespace}parent/{namespace}version")
+ assert version is not None
+ assert version.text is not None
version = version.text.strip()
+
packaging = treeRoot.find("%spackaging" % namespace)
- packaging = "jar" if packaging is None else packaging.text.strip()
+ if packaging is None:
+ packaging = "jar"
+ else:
+ assert packaging.text is not None
+ packaging = packaging.text.strip()
return groupId, artifactId, packaging, version
-def verifyMavenSigs(tmpDir, artifacts, keysFile):
+def verifyMavenSigs(tmpDir: str, artifacts: list[str], keysFile: str):
print(" verify maven artifact sigs", end=" ")
# Set up clean gpg world; import keys file:
@@ -891,14 +904,14 @@ def verifyMavenSigs(tmpDir, artifacts, keysFile):
print()
-def print_warnings_in_file(file):
+def print_warnings_in_file(file: str):
with open(file) as f:
for line in f.readlines():
if line.lower().find("warning") != -1 and line.find("WARNING: This key is not certified with a trusted signature") == -1 and line.find("WARNING: using insecure memory") == -1:
print(" GPG: %s" % line.strip())
-def verifyPOMperBinaryArtifact(artifacts, version):
+def verifyPOMperBinaryArtifact(artifacts: list[str], version: str):
print(" verify that each binary artifact has a deployed POM...")
reBinaryJarWar = re.compile(r"%s\.[jw]ar$" % re.escape(version))
for artifact in [a for a in artifacts if reBinaryJarWar.search(a)]:
@@ -907,9 +920,8 @@ def verifyPOMperBinaryArtifact(artifacts, version):
raise RuntimeError("missing: POM for %s" % artifact)
-def verifyDeployedPOMsCoordinates(artifacts, version):
- """
- verify that each POM's coordinate (drawn from its content) matches
+def verifyDeployedPOMsCoordinates(artifacts: list[str], version: str):
+ """Verify that each POM's coordinate (drawn from its content) matches
its filepath, and verify that the corresponding artifact exists.
"""
print(" verify deployed POMs' coordinates...")
@@ -925,9 +937,10 @@ def verifyDeployedPOMsCoordinates(artifacts, version):
raise RuntimeError("Missing corresponding .%s artifact for POM %s" % (packaging, POM))
-def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
+def crawl(downloadedFiles: list[str], urlString: str, targetDir: str, exclusions: set[str] | None = None):
+ exclude: set[str] = exclusions if exclusions else set()
for text, subURL in getDirEntries(urlString):
- if text not in exclusions:
+ if text not in exclude:
path = os.path.join(targetDir, text)
if text.endswith("/"):
if not os.path.exists(path):
@@ -940,8 +953,8 @@ def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
sys.stdout.write(".")
-def make_java_config(parser, alt_java_homes):
- def _make_runner(java_home, is_base_version=False):
+def make_java_config(parser: argparse.ArgumentParser, alt_java_homes: list[str]):
+ def _make_runner(java_home: str, is_base_version: bool = False):
if cygwin:
java_home = subprocess.check_output('cygpath -u "%s"' % java_home, shell=True).decode("utf-8").strip()
cmd_prefix = 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (java_home, java_home, java_home)
@@ -954,13 +967,12 @@ def make_java_config(parser, alt_java_homes):
# validate Java version
if is_base_version:
- if BASE_JAVA_VERSION != actual_version:
+ if actual_version != BASE_JAVA_VERSION:
parser.error("got wrong base version for java %s:\n%s" % (BASE_JAVA_VERSION, s))
- else:
- if int(actual_version) < int(BASE_JAVA_VERSION):
- parser.error("got wrong version for java %s, less than base version %s:\n%s" % (actual_version, BASE_JAVA_VERSION, s))
+ elif int(actual_version) < int(BASE_JAVA_VERSION):
+ parser.error("got wrong version for java %s, less than base version %s:\n%s" % (actual_version, BASE_JAVA_VERSION, s))
- def run_java(cmd, logfile):
+ def run_java(cmd: str, logfile: str):
run("%s; %s" % (cmd_prefix, cmd), logfile)
return run_java, actual_version
@@ -969,8 +981,8 @@ def make_java_config(parser, alt_java_homes):
if java_home is None:
parser.error("JAVA_HOME must be set")
run_java, _ = _make_runner(java_home, True)
- run_alt_javas = []
- alt_java_versions = []
+ run_alt_javas: list[Callable[[str, str], None]] = []
+ alt_java_versions: list[str] = []
if alt_java_homes:
for alt_java_home in alt_java_homes:
run_alt_java, version = _make_runner(alt_java_home)
@@ -1044,7 +1056,7 @@ reVersion2 = re.compile(r"-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.", re.IGNORECASE)
def getAllLuceneReleases():
s = load("https://archive.apache.org/dist/lucene/java")
- releases = set()
+ releases: set[tuple[int, ...]] = set()
for r in reVersion1, reVersion2:
for tup in r.findall(s):
if tup[-1].lower() == "-alpha":
@@ -1062,14 +1074,14 @@ def getAllLuceneReleases():
return releaseList
-def confirmAllReleasesAreTestedForBackCompat(smokeVersion, unpackPath):
+def confirmAllReleasesAreTestedForBackCompat(smokeVersion: str, unpackPath: str):
print(" find all past Lucene releases...")
allReleases = getAllLuceneReleases()
# for tup in allReleases:
# print(' %s' % '.'.join(str(x) for x in tup))
testedIndicesPaths = glob.glob("%s/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/*-cfs.zip" % unpackPath)
- testedIndices = set()
+ testedIndices: set[tuple[int, ...]] = set()
reIndexName = re.compile(r"^[^.]*.(.*?)-cfs.zip")
for name in testedIndicesPaths:
@@ -1112,7 +1124,7 @@ def confirmAllReleasesAreTestedForBackCompat(smokeVersion, unpackPath):
if x != (1, 9, 0):
raise RuntimeError("tested version=%s but it was not released?" % ".".join(str(y) for y in x))
- notTested = []
+ notTested: list[tuple[int, ...]] = []
for x in allReleases:
if x not in testedIndices:
releaseVersion = ".".join(str(y) for y in x)
@@ -1131,8 +1143,7 @@ def confirmAllReleasesAreTestedForBackCompat(smokeVersion, unpackPath):
for x in notTested:
print(" %s" % ".".join(str(y) for y in x))
raise RuntimeError("some releases are not tested by TestBackwardsCompatibility?")
- else:
- print(" success!")
+ print(" success!")
def main():
@@ -1149,7 +1160,7 @@ def main():
smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, " ".join(c.test_args), downloadOnly=c.download_only)
-def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs, downloadOnly=False):
+def smokeTest(java: Any, baseURL: str, gitRevision: str, version: str, tmpDir: str, isSigned: bool, local_keys: str | None, testArgs: str, downloadOnly: bool = False):
startTime = datetime.datetime.now()
# Tests annotated @Nightly are more resource-intensive but often cover