You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2025/03/06 22:46:56 UTC
(lucene) branch python_linting created (now fc0167b6fa6)
This is an automated email from the ASF dual-hosted git repository.
rmuir pushed a change to branch python_linting
in repository https://gitbox.apache.org/repos/asf/lucene.git
at fc0167b6fa6 python: enable all linting checks and type-hint the code
This branch includes the following new commits:
new fc0167b6fa6 python: enable all linting checks and type-hint the code
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(lucene) 01/01: python: enable all linting checks and type-hint the code
Posted by rm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rmuir pushed a commit to branch python_linting
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit fc0167b6fa6712f596f0c0668ee03f020c62324a
Author: Robert Muir <rm...@apache.org>
AuthorDate: Thu Mar 6 17:44:45 2025 -0500
python: enable all linting checks and type-hint the code
enable all type checking
type-hint the code to get checker flowing
disable a few remaining checks with a TODO list
enable all linter checks
autofix all safe fixes (e.g. api migrations)
disable just the remaining checks with a massive TODO list
---
dev-tools/scripts/Makefile | 10 +-
dev-tools/scripts/addBackcompatIndexes.py | 17 +-
dev-tools/scripts/addVersion.py | 35 +-
dev-tools/scripts/buildAndPushRelease.py | 41 ++-
dev-tools/scripts/create_line_file_docs.py | 30 +-
dev-tools/scripts/diff_lucene_changes.py | 7 +-
dev-tools/scripts/githubPRs.py | 23 +-
dev-tools/scripts/pyproject.toml | 152 ++++++++-
dev-tools/scripts/releaseWizard.py | 472 +++++++++++++-------------
dev-tools/scripts/releasedJirasRegex.py | 35 +-
dev-tools/scripts/reproduceJenkinsFailures.py | 46 ++-
dev-tools/scripts/scriptutil.py | 37 +-
dev-tools/scripts/smokeTestRelease.py | 201 +++++------
13 files changed, 632 insertions(+), 474 deletions(-)
diff --git a/dev-tools/scripts/Makefile b/dev-tools/scripts/Makefile
index 97ed7aaed24..f2786ba142c 100644
--- a/dev-tools/scripts/Makefile
+++ b/dev-tools/scripts/Makefile
@@ -17,7 +17,7 @@ PIP_INSTALL_ARGS=--disable-pip-version-check --no-input --upgrade
VENV=${PWD}/.venv
# don't behave strangely if these files exist
-.PHONY: lint format reformat ruff pyright env clean
+.PHONY: lint format reformat autofix ruff ruff-fix pyright env clean
# list of directories we check
SOURCES=$(wildcard *.py)
@@ -39,11 +39,19 @@ reformat: env
# reformat sources
$(VENV)/bin/ruff format $(SOURCES)
+# fixes all code problems that are safe to automatically fix (including formatting)
+autofix: ruff-fix reformat
+
# lints sources
ruff: env
# validate sources with ruff linter
$(VENV)/bin/ruff check $(SOURCES)
+# fixes (safe) issues that are autofixable such as deprecated/renamed APIs
+ruff-fix: env
+ # (safe) fixes sources with ruff linter
+ $(VENV)/bin/ruff check --fix $(SOURCES)
+
# checks types
pyright: env
# type-check sources with basedpyright
diff --git a/dev-tools/scripts/addBackcompatIndexes.py b/dev-tools/scripts/addBackcompatIndexes.py
index 3ae0ea62d4c..417ea785b12 100755
--- a/dev-tools/scripts/addBackcompatIndexes.py
+++ b/dev-tools/scripts/addBackcompatIndexes.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -34,7 +33,7 @@ import urllib.request
import scriptutil
-def create_and_add_index(source, indextype, index_version, current_version, temp_dir):
+def create_and_add_index(source: str, indextype: str, index_version: scriptutil.Version, current_version: scriptutil.Version, temp_dir: str):
if not current_version.is_back_compat_with(index_version):
prefix = "unsupported"
else:
@@ -79,7 +78,7 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
print("done")
-def update_backcompat_tests(index_version, current_version):
+def update_backcompat_tests(index_version: scriptutil.Version, current_version: scriptutil.Version):
print(" adding new indexes to backcompat tests...", end="", flush=True)
module = "lucene/backward-codecs"
@@ -91,12 +90,12 @@ def update_backcompat_tests(index_version, current_version):
strip_dash_suffix_re = re.compile(r"-.*")
- def find_version(x):
+ def find_version(x: str):
x = x.strip()
x = re.sub(strip_dash_suffix_re, "", x) # remove the -suffix if any
return scriptutil.Version.parse(x)
- def edit(buffer, match, line):
+ def edit(buffer: list[str], _match: re.Match[str], line: str):
v = find_version(line)
changed = False
if v.on_or_after(index_version):
@@ -106,7 +105,7 @@ def update_backcompat_tests(index_version, current_version):
buffer.append(line)
return changed
- def append(buffer, changed):
+ def append(buffer: list[str], changed: bool):
if changed:
return changed
if not buffer[len(buffer) - 1].endswith("\n"):
@@ -124,7 +123,7 @@ def check_backcompat_tests():
print("ok")
-def download_from_cdn(version, remotename, localname):
+def download_from_cdn(version: scriptutil.Version, remotename: str, localname: str):
url = "http://dlcdn.apache.org/lucene/java/%s/%s" % (version, remotename)
try:
urllib.request.urlretrieve(url, localname)
@@ -135,7 +134,7 @@ def download_from_cdn(version, remotename, localname):
raise e
-def download_from_archives(version, remotename, localname):
+def download_from_archives(version: scriptutil.Version, remotename: str, localname: str):
url = "http://archive.apache.org/dist/lucene/java/%s/%s" % (version, remotename)
try:
urllib.request.urlretrieve(url, localname)
@@ -146,7 +145,7 @@ def download_from_archives(version, remotename, localname):
raise e
-def download_release(version, temp_dir, force):
+def download_release(version: scriptutil.Version, temp_dir: str, force: bool):
print(" downloading %s source release..." % version, end="", flush=True)
source = os.path.join(temp_dir, "lucene-%s" % version)
if os.path.exists(source):
diff --git a/dev-tools/scripts/addVersion.py b/dev-tools/scripts/addVersion.py
index 75632d6a6dd..952177ca0c1 100755
--- a/dev-tools/scripts/addVersion.py
+++ b/dev-tools/scripts/addVersion.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -26,11 +25,11 @@ from configparser import ConfigParser, ExtendedInterpolation
from scriptutil import Version, find_branch_type, find_current_version, run, update_file
-def update_changes(filename, new_version, init_changes, headers):
+def update_changes(filename: str, new_version: Version, init_changes: str, headers: list[str]):
print(" adding new section to %s..." % filename, end="", flush=True)
matcher = re.compile(r"\d+\.\d+\.\d+\s+===")
- def edit(buffer, match, line):
+ def edit(buffer: list[str], _match: re.Match[str], line: str):
if new_version.dot in line:
return None
match = new_version.previous_dot_matcher.search(line)
@@ -46,14 +45,14 @@ def update_changes(filename, new_version, init_changes, headers):
print("done" if changed else "uptodate")
-def add_constant(new_version, deprecate):
+def add_constant(new_version: Version, deprecate: bool):
filename = "lucene/core/src/java/org/apache/lucene/util/Version.java"
print(" adding constant %s..." % new_version.constant, end="", flush=True)
constant_prefix = "public static final Version LUCENE_"
matcher = re.compile(constant_prefix)
prev_matcher = new_version.make_previous_matcher(prefix=constant_prefix, sep="_")
- def ensure_deprecated(buffer):
+ def ensure_deprecated(buffer: list[str]):
last = buffer[-1]
if last.strip() != "@Deprecated":
spaces = " " * (len(last) - len(last.lstrip()) - 1)
@@ -62,22 +61,22 @@ def add_constant(new_version, deprecate):
del buffer[-3:] # drop the trailing lines '<p> / Use this to get the latest ... / ... for Lucene.'
buffer.append(("{0} * @deprecated ({1}) Use latest\n" + "{0} */\n" + "{0}@Deprecated\n").format(spaces, new_version))
- def buffer_constant(buffer, line):
+ def buffer_constant(buffer: list[str], line: str):
spaces = " " * (len(line) - len(line.lstrip()))
buffer.append(("\n{0}/**\n" + "{0} * Match settings and bugs in Lucene's {1} release.\n").format(spaces, new_version))
if deprecate:
buffer.append("%s * @deprecated Use latest\n" % spaces)
else:
- buffer.append(("{0} * <p>Use this to get the latest & greatest settings, bug fixes, etc, for Lucene.\n").format(spaces))
+ buffer.append(f"{spaces} * <p>Use this to get the latest & greatest settings, bug fixes, etc, for Lucene.\n")
buffer.append("%s */\n" % spaces)
if deprecate:
buffer.append("%s@Deprecated\n" % spaces)
- buffer.append("{0}public static final Version {1} = new Version({2}, {3}, {4});\n".format(spaces, new_version.constant, new_version.major, new_version.minor, new_version.bugfix))
+ buffer.append(f"{spaces}public static final Version {new_version.constant} = new Version({new_version.major}, {new_version.minor}, {new_version.bugfix});\n")
- class Edit(object):
+ class Edit:
found = -1
- def __call__(self, buffer, match, line):
+ def __call__(self, buffer: list[str], _match: re.Match[str], line: str):
if new_version.constant in line:
return None # constant already exists
# outer match is just to find lines declaring version constants
@@ -88,7 +87,7 @@ def add_constant(new_version, deprecate):
elif self.found != -1:
# we didn't match, but we previously had a match, so insert new version here
# first find where to insert (first empty line before current constant)
- c = []
+ c: list[str] = []
buffer_constant(c, line)
tmp = buffer[self.found :]
buffer[self.found :] = c
@@ -103,11 +102,11 @@ def add_constant(new_version, deprecate):
print("done" if changed else "uptodate")
-def update_build_version(new_version):
+def update_build_version(new_version: Version):
print(" changing baseVersion...", end="", flush=True)
filename = "build.gradle"
- def edit(buffer, _, line):
+ def edit(buffer: list[str], _match: re.Match[str], line: str):
if new_version.dot in line:
return None
buffer.append(" String baseVersion = '" + new_version.dot + "'\n")
@@ -118,12 +117,12 @@ def update_build_version(new_version):
print("done" if changed else "uptodate")
-def update_latest_constant(new_version):
+def update_latest_constant(new_version: Version):
print(" changing Version.LATEST to %s..." % new_version.constant, end="", flush=True)
filename = "lucene/core/src/java/org/apache/lucene/util/Version.java"
matcher = re.compile("public static final Version LATEST")
- def edit(buffer, _, line):
+ def edit(buffer: list[str], _match: re.Match[str], line: str):
if new_version.constant in line:
return None
buffer.append(line.rpartition("=")[0] + ("= %s;\n" % new_version.constant))
@@ -133,7 +132,7 @@ def update_latest_constant(new_version):
print("done" if changed else "uptodate")
-def onerror(x):
+def onerror(x: Exception):
raise x
@@ -143,7 +142,7 @@ def check_lucene_version_tests():
print("ok")
-def read_config(current_version):
+def read_config(current_version: Version):
parser = argparse.ArgumentParser(description="Add a new version to CHANGES, to Version.java and build.gradle files")
parser.add_argument("version", type=Version.parse)
newconf = parser.parse_args()
@@ -157,7 +156,7 @@ def read_config(current_version):
# Hack ConfigParser, designed to parse INI files, to parse & interpolate Java .properties files
-def parse_properties_file(filename):
+def parse_properties_file(filename: str):
contents = open(filename, encoding="ISO-8859-1").read().replace("%", "%%") # Escape interpolation metachar
parser = ConfigParser(interpolation=ExtendedInterpolation()) # Handle ${property-name} interpolation
parser.read_string("[DUMMY_SECTION]\n" + contents) # Add required section
diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py
index 85952be2dd0..b000d180605 100755
--- a/dev-tools/scripts/buildAndPushRelease.py
+++ b/dev-tools/scripts/buildAndPushRelease.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -35,13 +34,13 @@ LOG = "/tmp/release.log"
dev_mode = False
-def log(msg):
+def log(msg: str):
f = open(LOG, mode="ab")
f.write(msg.encode("utf-8"))
f.close()
-def run(command):
+def run(command: str):
log("\n\n%s: RUN: %s\n" % (datetime.datetime.now(), command))
if os.system("%s >> %s 2>&1" % (command, LOG)):
msg = " FAILED: %s [see log %s]" % (command, LOG)
@@ -49,7 +48,7 @@ def run(command):
raise RuntimeError(msg)
-def runAndSendGPGPassword(command, password):
+def runAndSendGPGPassword(command: str, password: str):
p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
f = open(LOG, "ab")
while True:
@@ -63,7 +62,7 @@ def runAndSendGPGPassword(command, password):
if line.find(b"Enter GPG keystore password:") != -1:
time.sleep(1.0)
p.stdin.write((password + "\n").encode("UTF-8"))
- p.stdin.write("\n".encode("UTF-8"))
+ p.stdin.write(b"\n")
try:
result = p.wait(timeout=120)
@@ -77,7 +76,7 @@ def runAndSendGPGPassword(command, password):
raise RuntimeError(msg)
-def load(urlString, encoding="utf-8"):
+def load(urlString: str, encoding: str = "utf-8"):
try:
content = urllib.request.urlopen(urlString).read().decode(encoding)
except Exception as e:
@@ -99,7 +98,7 @@ def getGitRev():
return os.popen("git rev-parse HEAD").read().strip()
-def prepare(root, version, pause_before_sign, gpg_key_id, gpg_password, gpg_home=None, sign_gradle=False):
+def prepare(root: str, version: str, pause_before_sign: bool, gpg_key_id: str | None, gpg_password: str | None, gpg_home: str | None = None, sign_gradle: bool = False):
print()
print("Prepare release...")
if os.path.exists(LOG):
@@ -166,30 +165,30 @@ reVersion2 = re.compile(r"-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.zip<", re.IGNOREC
reDoapRevision = re.compile(r"(\d+)\.(\d+)(?:\.(\d+))?(-alpha|-beta)?", re.IGNORECASE)
-def checkDOAPfiles(version):
+def checkDOAPfiles(version: str):
# In Lucene DOAP file, verify presence of all releases less than the one being produced.
- errorMessages = []
+ errorMessages: list[str] = []
for product in ["lucene"]:
url = "https://archive.apache.org/dist/lucene/%s" % ("java" if product == "lucene" else product)
distpage = load(url)
- releases = set()
+ releases: set[str] = set()
for regex in reVersion1, reVersion2:
for tup in regex.findall(distpage):
if tup[0] in ("1", "2"): # Ignore 1.X and 2.X releases
continue
releases.add(normalizeVersion(tup))
doapNS = "{http://usefulinc.com/ns/doap#}"
- xpathRevision = "{0}Project/{0}release/{0}Version/{0}revision".format(doapNS)
+ xpathRevision = f"{doapNS}Project/{doapNS}release/{doapNS}Version/{doapNS}revision"
doapFile = "dev-tools/doap/%s.rdf" % product
treeRoot = ET.parse(doapFile).getroot()
- doapRevisions = set()
+ doapRevisions: set[str] = set()
for revision in treeRoot.findall(xpathRevision):
if revision.text and (match := reDoapRevision.match(revision.text)):
if match.group(1) not in ("0", "1", "2"): # Ignore 0.X, 1.X and 2.X revisions
doapRevisions.add(normalizeVersion(match.groups()))
else:
errorMessages.append("ERROR: Failed to parse revision: %s in %s" % (revision.text, doapFile))
- missingDoapRevisions = set()
+ missingDoapRevisions: set[str] = set()
for release in releases:
if release not in doapRevisions and release < version: # Ignore releases greater than the one being produced
missingDoapRevisions.add(release)
@@ -199,7 +198,7 @@ def checkDOAPfiles(version):
raise RuntimeError("\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)" % "\n".join(errorMessages))
-def normalizeVersion(tup):
+def normalizeVersion(tup: tuple[str, ...]):
suffix = ""
if tup[-1] is not None and tup[-1].lower() == "-alpha":
tup = tup[: (len(tup) - 1)]
@@ -214,7 +213,7 @@ def normalizeVersion(tup):
return ".".join(tup) + suffix
-def pushLocal(version, root, rcNum, localDir):
+def pushLocal(version: str, root: str, rcNum: int, localDir: str):
print("Push local [%s]..." % localDir)
os.makedirs(localDir)
@@ -243,7 +242,7 @@ def pushLocal(version, root, rcNum, localDir):
return "file://%s/%s" % (os.path.abspath(localDir), dir)
-def read_version(path):
+def read_version(_path: str):
return scriptutil.find_current_version()
@@ -271,9 +270,9 @@ def parse_config():
dest="gpg_pass_noprompt",
default=False,
action="store_true",
- help="Do not prompt for gpg passphrase. For the default gnupg method, this means your gpg-agent"
- " needs a non-TTY pin-entry program. For gradle signing method, passphrase must be provided"
- " in gradle.properties or by env.var/sysprop. See ./gradlew helpPublishing for more info",
+ help="""Do not prompt for gpg passphrase. For the default gnupg method, this means your gpg-agent
+ needs a non-TTY pin-entry program. For gradle signing method, passphrase must be provided
+ in gradle.properties or by env.var/sysprop. See ./gradlew helpPublishing for more info""",
)
parser.add_argument("--gpg-home", metavar="PATH", help="Path to gpg home containing your secring.gpg Optional, will use $HOME/.gnupg/secring.gpg by default")
parser.add_argument("--rc-num", metavar="NUM", type=int, default=1, help="Release Candidate number. Default: 1")
@@ -301,7 +300,7 @@ def parse_config():
cwd = os.getcwd()
os.chdir(config.root)
config.root = os.getcwd() # Absolutize root dir
- if os.system("git rev-parse") or 2 != len([d for d in ("dev-tools", "lucene") if os.path.isdir(d)]):
+ if os.system("git rev-parse") or len([d for d in ("dev-tools", "lucene") if os.path.isdir(d)]) != 2:
parser.error('Root path "%s" is not a valid lucene checkout' % config.root)
os.chdir(cwd)
global LOG
@@ -320,7 +319,7 @@ def check_cmdline_tools(): # Fail fast if there are cmdline tool problems
raise RuntimeError('"git --version" returned a non-zero exit code.')
-def check_key_in_keys(gpgKeyID, local_keys):
+def check_key_in_keys(gpgKeyID: str | None, local_keys: str | None):
if gpgKeyID is not None:
print(" Verify your gpg key is in the main KEYS file")
if local_keys is not None:
diff --git a/dev-tools/scripts/create_line_file_docs.py b/dev-tools/scripts/create_line_file_docs.py
index 9c753f22f79..1dd9963ecdc 100644
--- a/dev-tools/scripts/create_line_file_docs.py
+++ b/dev-tools/scripts/create_line_file_docs.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -24,16 +23,17 @@ import subprocess
import tempfile
import time
import urllib.request
+from io import TextIOWrapper
DEBUG = False
TARGET_DOC_CHARS = 1024
-def compress_with_seek_points(file_name_in, file_name_out, num_seek_points):
+def compress_with_seek_points(file_name_in: str, file_name_out: str, num_seek_points: int):
bytes_per_chunk = os.path.getsize(file_name_in) / num_seek_points
- seek_points = []
+ seek_points: list[int] = []
if os.path.exists(file_name_out):
os.remove(file_name_out)
@@ -60,7 +60,7 @@ def compress_with_seek_points(file_name_in, file_name_out, num_seek_points):
break
bytes_in_chunk += len(line)
- f_out.write(line) # false positive in python's crazy typing # pyright: ignore[reportArgumentType]
+ f_out.write(line)
if bytes_in_chunk > bytes_per_chunk and chunk_count < num_seek_points:
f_out.close()
@@ -77,16 +77,13 @@ re_newlines = re.compile(r"\n+")
re_space = re.compile(r"\s")
# used to find word break, for splitting docs into ~1 KB sized smaller docs:
-re_next_non_word_character = re.compile(r"\W", re.U)
+re_next_non_word_character = re.compile(r"\W", re.UNICODE)
EUROPARL_V7_URL = "https://www.statmt.org/europarl/v7/europarl.tgz"
-def split_docs(all_out, title_string, date_string, body_string):
- """
- Splits docs into smallish (~1 KB) sized docs, repeating same title and date
- """
-
+def split_docs(all_out: TextIOWrapper, title_string: str, date_string: str, body_string: str):
+ """Splits docs into smallish (~1 KB) sized docs, repeating same title and date"""
doc_count = 0
while len(body_string) > 0:
char_count = int(random.gauss(TARGET_DOC_CHARS, TARGET_DOC_CHARS / 4))
@@ -129,7 +126,7 @@ def sample_europarl():
if not DEBUG:
cmd = "tar xzf %s -C %s" % (file_name, tmp_dir_path)
print("Run: %s" % cmd)
- subprocess.run(cmd, shell=True)
+ subprocess.run(cmd, shell=True, check=False)
doc_count = 0
skip_count = 0
@@ -158,8 +155,8 @@ def sample_europarl():
# unfortunately we need errors='ignore' since in Europarl v7, one file (pl/ep-09-10-22-009.txt) has invalid utf-8:
chapter_count = 0
- with open("%s/%s" % (dir_path, file_name), "r", encoding="utf-8", errors="ignore") as f_in:
- last_text = []
+ with open("%s/%s" % (dir_path, file_name), encoding="utf-8", errors="ignore") as f_in:
+ last_text: list[str] = []
last_title = None
while True:
line = f_in.readline()
@@ -188,8 +185,7 @@ def sample_europarl():
if len(last_title) > 0:
break
continue
- else:
- last_text.append(line)
+ last_text.append(line)
if last_title is not None:
s = " ".join(last_text)
@@ -232,7 +228,7 @@ def sample_europarl():
)
print("Shuffle...")
- subprocess.run("shuf %s > %s.shuffled" % (all_txt_file_name, all_txt_file_name), shell=True)
+ subprocess.run("shuf %s > %s.shuffled" % (all_txt_file_name, all_txt_file_name), shell=True, check=False)
for mb in (20, 200, 2000):
print("Sample %d MB file..." % mb)
@@ -240,7 +236,7 @@ def sample_europarl():
with open(file_name_out, "w", encoding="utf-8") as f_out:
chance = mb / total_mb
- with open(all_txt_file_name + ".shuffled", "r", encoding="utf-8") as f:
+ with open(all_txt_file_name + ".shuffled", encoding="utf-8") as f:
while True:
line = f.readline()
if len(line) == 0:
diff --git a/dev-tools/scripts/diff_lucene_changes.py b/dev-tools/scripts/diff_lucene_changes.py
index d443ff32bce..4d8e5121f39 100644
--- a/dev-tools/scripts/diff_lucene_changes.py
+++ b/dev-tools/scripts/diff_lucene_changes.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -31,10 +30,8 @@ to users and avoid future annoying conflicts on backport.
# e.g. python3 -u diff_lucene_changes.py branch_9_9 main 9.9.0
-#
-
-def get_changes_url(branch_name):
+def get_changes_url(branch_name: str):
if os.path.isdir(branch_name):
url = f"file://{branch_name}/lucene/CHANGES.txt"
else:
@@ -43,7 +40,7 @@ def get_changes_url(branch_name):
return url
-def extract_release_section(changes_txt, release_name):
+def extract_release_section(changes_txt: bytes, release_name: str):
match = re.search(f"=======+ Lucene {re.escape(release_name)} =======+(.*?)=======+ Lucene .*? =======+$", changes_txt.decode("utf-8"), re.MULTILINE | re.DOTALL)
assert match
return match.group(1).encode("utf-8")
diff --git a/dev-tools/scripts/githubPRs.py b/dev-tools/scripts/githubPRs.py
index 335c05e6aa7..66a2ae30713 100755
--- a/dev-tools/scripts/githubPRs.py
+++ b/dev-tools/scripts/githubPRs.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -15,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""
-Simple script that queries GitHub for all open PRs, then finds the ones without
+"""Simple script that queries GitHub for all open PRs, then finds the ones without
issue number in title, and the ones where the linked JIRA is already closed
"""
@@ -27,13 +25,16 @@ sys.path.append(os.path.dirname(__file__))
import argparse
import json
import re
-from typing import cast
+from typing import TYPE_CHECKING, Any, cast
from github import Github
from jinja2 import BaseLoader, Environment
from jira import JIRA, Issue
from jira.client import ResultList
+if TYPE_CHECKING:
+ from github.PullRequest import PullRequest
+
def read_config():
parser = argparse.ArgumentParser(description="Find open Pull Requests that need attention")
@@ -44,13 +45,13 @@ def read_config():
return newconf
-def out(text):
+def out(text: str):
global conf
if not (conf.json or conf.html):
print(text)
-def make_html(dict):
+def make_html(dict: dict[Any, Any]):
global conf
template = Environment(loader=BaseLoader()).from_string("""
<h1>Lucene Github PR report</h1>
@@ -83,7 +84,7 @@ def main():
else:
gh = Github()
jira = JIRA("https://issues.apache.org/jira") # this ctor has broken types in jira library. # pyright: ignore[reportArgumentType]
- result = {}
+ result: dict[str, Any] = {}
repo = gh.get_repo("apache/lucene")
open_prs = repo.get_pulls(state="open")
out("Lucene Github PR report")
@@ -93,7 +94,7 @@ def main():
lack_jira = list(filter(lambda x: not re.match(r".*\b(LUCENE)-\d{3,6}\b", x.title), open_prs))
result["no_jira_count"] = len(lack_jira)
- lack_jira_list = []
+ lack_jira_list: list[dict[str, Any]] = []
for pr in lack_jira:
lack_jira_list.append({"title": pr.title, "number": pr.number, "user": pr.user.login, "created": pr.created_at.strftime("%Y-%m-%d")})
result["no_jira"] = lack_jira_list
@@ -104,8 +105,8 @@ def main():
out("\nOpen PRs with a resolved JIRA")
has_jira = list(filter(lambda x: re.match(r".*\b(LUCENE)-\d{3,6}\b", x.title), open_prs))
- issue_ids = []
- issue_to_pr = {}
+ issue_ids: list[str] = []
+ issue_to_pr: dict[str, PullRequest] = {}
for pr in has_jira:
match = re.match(r".*\b((LUCENE)-\d{3,6})\b", pr.title)
assert match
@@ -114,7 +115,7 @@ def main():
issue_to_pr[jira_issue_str] = pr
resolved_jiras = cast(ResultList[Issue], jira.search_issues(jql_str="key in (%s) AND status in ('Closed', 'Resolved')" % ", ".join(issue_ids)))
- closed_jiras = []
+ closed_jiras: list[dict[str, Any]] = []
for issue in resolved_jiras:
pr_title = issue_to_pr[issue.key].title
pr_number = issue_to_pr[issue.key].number
diff --git a/dev-tools/scripts/pyproject.toml b/dev-tools/scripts/pyproject.toml
index bc9a970c12f..24e5c507280 100644
--- a/dev-tools/scripts/pyproject.toml
+++ b/dev-tools/scripts/pyproject.toml
@@ -1,21 +1,163 @@
[tool.pyright]
+pythonVersion = "3.12"
venvPath = "."
venv = ".venv"
-# TODO: improve!
-# typeCheckingMode = "strict"
+typeCheckingMode = "strict"
+reportCallInDefaultInitializer = "error"
+reportImplicitStringConcatenation = "error"
+reportImportCycles = "error"
+reportImplicitOverride = "error"
+reportPropertyTypeMismatch = "error"
+reportUninitializedInstanceVariable = "error"
reportUnnecessaryTypeIgnoreComment = "error"
-typeCheckingMode = "standard"
+
+# This is a big TODO list of current typing problems
+# Pick an item off the list, change to "error", and send a pull request to fix it!
+# You can find more information on a rule at https://github.com/microsoft/pyright/blob/main/docs/configuration.md
+reportConstantRedefinition = "hint" # Requires fixing nonstandard variable conventions
+reportMissingSuperCall = "hint" # Requires adding missing super().__init__() calls
+reportMissingTypeStubs = "hint" # Can be fixed by switching 'ics' to alternative such as 'icalendar'
+reportPrivateUsage = "hint" # Can be fixed by using TLS properly (Ruff S323)
+reportUnnecessaryComparison = "hint" # Requires cleaning up some dead code
+reportUntypedNamedTuple = "hint" # Requires moving to different type of named tuple
+reportUnusedCallResult = "none" # Forces you to assign unused retvals to '_', very noisy.
+
+# Intentionally disabled because it slows pyright by 2x
+reportShadowedImports = "none" # Extremely slow check
[tool.ruff]
+target-version = "py312"
line-length = 200
indent-width = 2
[tool.ruff.lint]
-# TODO: improve!
-# select = ["ALL"]
+select = ["ALL"]
# disabling/enabling of rules
ignore = [
+ # This is a big TODO list of current linter problems.
+ # Pick an item off the list, remove it, and send a pull request to fix it!
+ # You can find more information on a rule at https://docs.astral.sh/ruff/rules/
+ "A001", # variable name is shadowing a python builtin
+ "A002", # function argument name is shadowing a python builtin
+ "ANN001", # missing type annotation for function argument
+ "ANN201", # missing return annotation for public function
+ "ANN202", # missing return annotation for private function
+ "ANN204", # missing return annotation for special method
+ "ANN206", # missing return annotation for classmethod
+ "ANN401", # use of Any as function parameter
+ "ARG001", # unused function argument
+ "ARG002", # unused method argument
+ "ARG005", # unused lambda argument
+ "B006", # do not use mutable datastructures for arg defaults
+ "B904", # within except clause, raise exceptions with .. from
+ "BLE001", # do not catch blind exception
+ "C408", # unnecessary dict() call
+ "C416", # unnecessary list comprehension
+ "C417", # unnecessary map() usage
+ "C901", # function is too complex
+ "D100", # missing docstring in public module
+ "D101", # missing docstring in public class
+ "D102", # missing docstring in public method
+ "D103", # missing docstring in public function
+ "D105", # missing docstring in magic method
+ "D107", # missing docstring in ctor
+ "D205", # blank line required between summary and description
+ "D400", # first docstring line should end with a period
+ "D401", # first docstring line should be in imperative mood
+ "D415", # first docstring line should end with punctuation
+ "DTZ005", # datetime.datetime.now called without tz argument
+ "EM101", # exception must not use a string literal, assign to variable first
+ "ERA001", # found commented-out code
+ "EXE001", # shebang present but file is not executable
+ "FBT001", # boolean-typed positional argument in function definition
+ "FBT002", # boolean default positional argument in function definition
+ "FBT003", # boolean positional value in function call
+ "FIX002", # line contains TODO, consider resolving issue
+ "FIX004", # line contains HACK, consider resolving issue
+ "FURB129", # instead of calling readlines(), iterate over file object directly
+ "N802", # function name should be lowercase
+ "N803", # argument name should be lowercase
+ "N806", # variable should be lowercase
+ "N816", # variable in global scope should not be mixedCase
+ "PERF203", # try-except within a loop incurs performance overhead
+ "PERF401", # use list.extend to create a transformed list
+ "PIE810", # call endswith once with a tuple
+ "PLC0206", # extracting value from dictionary without calling .items
+ "PLR0911", # too many return statements
+ "PLR0912", # too many branches
+ "PLR0913", # too many arguments in function definition
+ "PLR0915", # too many statements
+ "PLR1704", # redefining argument with local name
+ "PLR1714", # consider merging multiple comparisons
+ "PLR1722", # use sys.exit() instead of exit
+ "PLR2004", # magic value used in comparison
+ "PLW0602", # using global for variable but no assignment is done
+ "PLW0603", # using global statement to update variable is discouraged
+ "PLW2901", # loop variable overwritten by assignment target
+ "PTH100", # os.path.abspath should be replaced by Path.resolve
+ "PTH103", # os.mkdirs should be replaced by Path.mkdir(parents=True)
+ "PTH104", # os.rename should be replaced by Path.rename
+ "PTH107", # os.remove shoudl be replaced by Path.unlink
+ "PTH109", # os.getcwd should be replaced by Path.cwd
+ "PTH110", # os.path.exists should be replaced by Path.exists
+ "PTH111", # os.path.expanduser should be replaced by Path.expanduser
+ "PTH112", # os.path.isdir should be replaced by Path.is_dir
+ "PTH113", # os.path.isfile should be replaced by Path.is_file
+ "PTH116", # os.stat should be replaced by Path.stat, Path.owner, or Path.group
+ "PTH117", # os.path.isabs should be replaced by Path.is_absolute
+ "PTH118", # os.path.join should be replaced by Path with '/' operator
+ "PTH119", # os.path.basename should be replaced by Path.name
+ "PTH120", # os.path.dirname should be replaced by Path.parent
+ "PTH123", # open should be replaced by Path.open
+ "PTH202", # os.path.getsize should be replaced by Path.stat.st_size
+ "PTH207", # glob should be replaced by Path.glob or Path.rglob
+ "PYI024", # use typing.NamedTuple instead of collections.namedtuple
+ "RET503", # missing explicit return at end of function able to return non-None
+ "RET504", # unnecessary assignment before return
+ "RUF005", # unnecessary concatenation
+ "RUF012", # mutable class attributes should be annotated with typing.ClassVar
+ "S101", # use of 'assert' detected
+ "S108", # probable insecure usage of temp file/directory
+ "S110", # try-except-pass detected, consider logging the exception
+ "S307", # use of insecure eval, consider using ast.literal_eval
+ "S310", # audit url open for permitted schemes. allowing use of file: is often unexpected
+ "S311", # standard prng not suitable for cryptographic purposes
+ "S314", # using xml to parse untrusted data, use defusedxml equivalents
+ "S323", # use of insecure TLS context
+ "S324", # use of insecure hash function: sha1
+ "S506", # probable use of unsafe yaml loader, consider yaml.safe_load
+ "S602", # subprocess call with shell=True identified, security issue
+ "S603", # subprocess call: check for execution of untrusted input
+ "S605", # starting a process wtih a shell, seems safe, but may be changed in future
+ "S607", # starting a process with a partial executable path
+ "S701", # consider using jinja autoescape=True to mitigate XSS
+ "SIM102", # use single if statement instead of nested if statement
+ "SIM108", # use ternary operator instead of if-else-block
+ "SIM112", # use capitalized environment variable
+ "SIM113", # use enumerate for index variable in loop
+ "SIM115", # use a context manager for opening files
+ "SIM118", # use 'key in dict' instead of 'key in dict.keys'
+ "SIM201", # use returncode != 0 instead of not returncode == 0
+ "SLF001", # private member accessed
+ "T201", # print function found
+ "TD002", # missing author for this TODO
+ "TD003", # missing issue link for this TODO
+ "TD004", # missing colon for this TODO
+ "TD005", # missing issue description for this TODO
+ "TRY002", # create your own exception
+ "TRY003", # avoid specifying long messages outside exception class
+ "TRY201", # use raise without specifying exception name
+ "TRY300", # consider moving this statement to else block
+ "UP008", # use super() instead of super(__class__, self)
+ "UP036", # version block is outdated for minimum python version
+ "UP031", # use format specifiers instead of percent format
+
+ # These rules are always disabled, conflict with other rules
+ # don't enable! (switch to the alternative instead)
+ "D203", # disabled in favor of D211 (no-black-line-before-class)
+ "D213", # disabled in favor of D212 (multi-line-summary-first-line)
+
# These rules are always disabled: conflict with the formatter
# don't enable! https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
"W191", "E111", "E114", "E117", "D206", "D300", "Q000", "Q001",
diff --git a/dev-tools/scripts/releaseWizard.py b/dev-tools/scripts/releaseWizard.py
index fa64491d77a..bfc83fdc000 100755
--- a/dev-tools/scripts/releaseWizard.py
+++ b/dev-tools/scripts/releaseWizard.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -47,8 +46,9 @@ import textwrap
import time
import urllib.request
from collections import OrderedDict
-from datetime import datetime, timedelta, timezone
-from typing import cast
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+from typing import Any, Self, TextIO, cast, override
import yaml
from consolemenu import ConsoleMenu
@@ -66,8 +66,8 @@ editor = None
# Edit this to add other global jinja2 variables or filters
-def expand_jinja(text, vars=None):
- global_vars = OrderedDict(
+def expand_jinja(text: str, vars: dict[str, Any] | None = None):
+ global_vars: dict[str, Any] = OrderedDict(
{
"script_version": state.script_version,
"release_version": state.release_version,
@@ -96,7 +96,7 @@ def expand_jinja(text, vars=None):
"state": state,
"gpg_key": state.get_gpg_key(),
"gradle_cmd": "gradlew.bat" if is_windows() else "./gradlew",
- "epoch": unix_time_millis(datetime.now(tz=timezone.utc)),
+ "epoch": unix_time_millis(datetime.now(tz=UTC)),
"get_next_version": state.get_next_version(),
"current_git_rev": state.get_current_git_rev(),
"keys_downloaded": keys_downloaded(),
@@ -124,9 +124,9 @@ def expand_jinja(text, vars=None):
try:
env = Environment(lstrip_blocks=True, keep_trailing_newline=False, trim_blocks=True)
- env.filters["path_join"] = lambda paths: os.path.join(*paths)
- env.filters["expanduser"] = lambda path: os.path.expanduser(path)
- env.filters["formatdate"] = lambda date: (datetime.strftime(date, "%-d %B %Y") if date else "<date>")
+ env.filters["path_join"] = lambda paths: os.path.join(*(cast(list[str], paths)))
+ env.filters["expanduser"] = lambda path: os.path.expanduser(str(path))
+ env.filters["formatdate"] = lambda date: (datetime.strftime(date, "%-d %B %Y") if date and isinstance(date, datetime) else "<date>")
template = env.from_string(str(filled), globals=global_vars)
filled = template.render()
except Exception as e:
@@ -134,8 +134,8 @@ def expand_jinja(text, vars=None):
return filled
-def replace_templates(text):
- tpl_lines = []
+def replace_templates(text: str):
+ tpl_lines: list[str] = []
for line in text.splitlines():
if match := re.search(r"^\(\( template=(.+?) \)\)", line):
name = match.group(1)
@@ -165,7 +165,7 @@ def get_editor():
return editor
-def check_prerequisites(todo=None):
+def check_prerequisites(todo: Any = None):
if sys.version_info < (3, 4):
sys.exit("Script requires Python v3.4 or later")
try:
@@ -197,14 +197,14 @@ def check_prerequisites(todo=None):
return True
-epoch = datetime.fromtimestamp(timestamp=0, tz=timezone.utc)
+epoch = datetime.fromtimestamp(timestamp=0, tz=UTC)
-def unix_time_millis(dt):
+def unix_time_millis(dt: datetime):
return int((dt - epoch).total_seconds() * 1000.0)
-def bootstrap_todos(todo_list):
+def bootstrap_todos(todo_list: list[Any]):
# Establish links from commands to to_do for finding todo vars
for tg in todo_list:
if dry_run:
@@ -248,10 +248,11 @@ def maybe_remove_rc_from_svn():
# To be able to hide fields when dumping Yaml
class SecretYamlObject(yaml.YAMLObject):
- hidden_fields = []
+ hidden_fields: list[str] = []
@classmethod
- def to_yaml(cls, dumper, data):
+ @override
+ def to_yaml(cls: type[Self], dumper: yaml.Dumper, data: Any):
print("Dumping object %s" % type(data))
new_data = copy.deepcopy(data)
@@ -261,27 +262,27 @@ class SecretYamlObject(yaml.YAMLObject):
for item in data.__dict__:
if item in new_data.__dict__ and new_data.__dict__[item] is None:
del new_data.__dict__[item]
- return dumper.represent_yaml_object(cls.yaml_tag, new_data, cls, flow_style=cls.yaml_flow_style)
+ return dumper.represent_yaml_object(cls.yaml_tag, new_data, cls, flow_style=cls.yaml_flow_style) # TODO: fix me # pyright: ignore[reportUnknownMemberType]
-def str_presenter(dumper, data):
+def str_presenter(dumper: yaml.Dumper, data: str):
if len(data.split("\n")) > 1: # check for multiline string
- return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
- return dumper.represent_scalar("tag:yaml.org,2002:str", data)
+ return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") # TODO: fix me # pyright: ignore[reportUnknownMemberType]
+ return dumper.represent_scalar("tag:yaml.org,2002:str", data) # TODO: fix me # pyright: ignore[reportUnknownMemberType]
class ReleaseState:
- def __init__(self, config_path, release_version, script_version):
+ def __init__(self, config_path: str, release_version: str, script_version: str):
self.script_version = script_version
self.config_path = config_path
self.todo_groups: list[TodoGroup] = []
self.todos: dict[str, Todo] = {}
- self.latest_version = None
- self.previous_rcs = {}
- self.rc_number = 1
- self.start_date = unix_time_millis(datetime.now(tz=timezone.utc))
- self.script_branch = run("git rev-parse --abbrev-ref HEAD").strip()
- self.mirrored_versions = None
+ self.latest_version: str | None = None
+ self.previous_rcs: dict[str, Any] = {}
+ self.rc_number: int = 1
+ self.start_date = unix_time_millis(datetime.now(tz=UTC))
+ self.script_branch: str = run("git rev-parse --abbrev-ref HEAD").strip()
+ self.mirrored_versions: list[str] | None = None
try:
self.script_branch_type = scriptutil.find_branch_type()
except Exception:
@@ -289,7 +290,7 @@ class ReleaseState:
self.script_branch_type = "feature"
self.set_release_version(release_version)
- def set_release_version(self, version):
+ def set_release_version(self, version: str):
self.validate_release_version(self.script_branch_type, self.script_branch, version)
self.release_version = version
v = Version.parse(version)
@@ -314,23 +315,20 @@ class ReleaseState:
assert gpg_task
if gpg_task.is_done():
return gpg_task.get_state()["gpg_key"]
- else:
- return None
+ return None
def get_release_date(self):
publish_task = self.get_todo_by_id("publish_maven")
assert publish_task
if publish_task.is_done():
return unix_to_datetime(publish_task.get_state()["done_date"])
- else:
- return None
+ return None
def get_release_date_iso(self):
release_date = self.get_release_date()
if release_date is None:
return "yyyy-mm-dd"
- else:
- return release_date.isoformat()[:10]
+ return release_date.isoformat()[:10]
def get_latest_version(self):
if self.latest_version is None:
@@ -341,12 +339,13 @@ class ReleaseState:
latest = ver
self.latest_version = latest
self.save()
+ assert state.latest_version
return state.latest_version
def get_mirrored_versions(self):
if state.mirrored_versions is None:
releases_str = load("https://projects.apache.org/json/foundation/releases.json", "utf-8")
- releases = json.loads(releases_str)["lucene"]
+ releases: dict[str, str] = json.loads(releases_str)["lucene"]
state.mirrored_versions = [r for r in list(map(lambda y: y[7:], filter(lambda x: x.startswith("lucene-"), list(releases.keys()))))]
return state.mirrored_versions
@@ -381,7 +380,7 @@ class ReleaseState:
latest_lts = ver
return latest_lts
- def validate_release_version(self, branch_type, branch, release_version):
+ def validate_release_version(self, branch_type: BranchType | str, branch: str, release_version: str):
ver = Version.parse(release_version)
# print("release_version=%s, ver=%s" % (release_version, ver))
if branch_type == BranchType.release:
@@ -406,12 +405,11 @@ class ReleaseState:
v = Version.parse(self.release_version)
if v.is_major_release():
return "main"
- elif v.is_minor_release():
+ if v.is_minor_release():
return self.get_stable_branch_name()
- elif v.major == Version.parse(self.get_latest_version()).major:
+ if v.major == Version.parse(self.get_latest_version()).major:
return self.get_minor_branch_name()
- else:
- return self.release_branch
+ return self.release_branch
def clear_rc(self):
if ask_yes_no("Are you sure? This will clear and restart RC%s" % self.rc_number):
@@ -445,7 +443,7 @@ class ReleaseState:
for todo_id in self.todos:
t = self.todos[todo_id]
tmp_todos[todo_id] = copy.deepcopy(t.state)
- dict = {
+ dictionary: dict[str, Any] = {
"script_version": self.script_version,
"release_version": self.release_version,
"start_date": self.start_date,
@@ -455,33 +453,30 @@ class ReleaseState:
"previous_rcs": self.previous_rcs,
}
if self.latest_version:
- dict["latest_version"] = self.latest_version
- return dict
-
- def restore_from_dict(self, dict):
- self.script_version = dict["script_version"]
- assert dict["release_version"] == self.release_version
- if "start_date" in dict:
- self.start_date = dict["start_date"]
- if "latest_version" in dict:
- self.latest_version = dict["latest_version"]
- else:
- self.latest_version = None
- self.rc_number = dict["rc_number"]
- self.script_branch = dict["script_branch"]
- self.previous_rcs = copy.deepcopy(dict["previous_rcs"])
- for todo_id in dict["todos"]:
+ dictionary["latest_version"] = self.latest_version
+ return dictionary
+
+ def restore_from_dict(self, dictionary: dict[str, Any]):
+ self.script_version = dictionary["script_version"]
+ assert dictionary["release_version"] == self.release_version
+ if "start_date" in dictionary:
+ self.start_date = dictionary["start_date"]
+ self.latest_version = dictionary.get("latest_version")
+ self.rc_number = dictionary["rc_number"]
+ self.script_branch = dictionary["script_branch"]
+ self.previous_rcs = copy.deepcopy(dictionary["previous_rcs"])
+ for todo_id in dictionary["todos"]:
if todo_id in self.todos:
t = self.todos[todo_id]
- for k in dict["todos"][todo_id]:
- t.state[k] = dict["todos"][todo_id][k]
+ for k in dictionary["todos"][todo_id]:
+ t.state[k] = dictionary["todos"][todo_id][k]
else:
print("Warning: Could not restore state for %s, Todo definition not found" % todo_id)
def load(self):
if os.path.exists(os.path.join(self.config_path, self.release_version, "state.yaml")):
state_file = os.path.join(self.config_path, self.release_version, "state.yaml")
- with open(state_file, "r") as fp:
+ with open(state_file) as fp:
try:
dict = yaml.load(fp, Loader=yaml.Loader)
self.restore_from_dict(dict)
@@ -515,26 +510,23 @@ class ReleaseState:
except Exception:
return "<git-rev>"
- def get_group_by_id(self, id):
- lst = list(filter(lambda x: x.id == id, self.todo_groups))
+ def get_group_by_id(self, id: str):
+ lst: list[TodoGroup] = list(filter(lambda x: x.id == id, self.todo_groups))
if len(lst) == 1:
return lst[0]
- else:
- return None
+ return None
- def get_todo_by_id(self, id):
- lst = list(filter(lambda x: x.id == id, self.todos.values()))
+ def get_todo_by_id(self, id: str):
+ lst: list[Todo] = list(filter(lambda x: x.id == id, self.todos.values()))
if len(lst) == 1:
return lst[0]
- else:
- return None
+ return None
- def get_todo_state_by_id(self, id):
- lst = list(filter(lambda x: x.id == id, self.todos.values()))
+ def get_todo_state_by_id(self, id: str):
+ lst: list[Todo] = list(filter(lambda x: x.id == id, self.todos.values()))
if len(lst) == 1:
return lst[0].state
- else:
- return {}
+ return cast(dict[Any, Any], {})
def get_release_folder(self):
folder = os.path.join(self.config_path, self.release_version)
@@ -564,11 +556,8 @@ class ReleaseState:
def get_minor_branch_name(self):
latest = state.get_latest_version()
- if latest is not None:
- v = Version.parse(latest)
- return "branch_%s_%s" % (v.major, v.minor)
- else:
- raise Exception("Cannot find latest version")
+ v = Version.parse(latest)
+ return "branch_%s_%s" % (v.major, v.minor)
def get_stable_branch_name(self):
if self.release_type == "major":
@@ -589,30 +578,29 @@ class ReleaseState:
def get_java_home(self):
return self.get_java_home_for_version(self.release_version)
- def get_java_home_for_version(self, version):
+ def get_java_home_for_version(self, version: str):
v = Version.parse(version)
java_ver = java_versions[v.major]
java_home_var = "JAVA%s_HOME" % java_ver
if java_home_var in os.environ:
return os.environ[java_home_var]
- else:
- raise Exception("Script needs environment variable %s" % java_home_var)
+ raise Exception("Script needs environment variable %s" % java_home_var)
- def get_java_cmd_for_version(self, version):
+ def get_java_cmd_for_version(self, version: str):
return os.path.join(self.get_java_home_for_version(version), "bin", "java")
def get_java_cmd(self):
return os.path.join(self.get_java_home(), "bin", "java")
def get_todo_states(self):
- states = {}
+ states: dict[str, dict[Any, Any]] = {}
if self.todos:
for todo_id in self.todos:
t = self.todos[todo_id]
states[todo_id] = copy.deepcopy(t.state)
return states
- def init_todos(self, groups):
+ def init_todos(self, groups: list[Any]):
self.todo_groups = groups
self.todos = {}
for g in self.todo_groups:
@@ -624,7 +612,7 @@ class TodoGroup(SecretYamlObject):
yaml_tag = "!TodoGroup"
hidden_fields = []
- def __init__(self, id, title, description, todos, is_in_rc_loop=None, depends=None):
+ def __init__(self, id: str, title: str, description: str, todos: list[Any], is_in_rc_loop: bool | None = None, depends: str | list[str] | None = None):
self.id = id
self.title = title
self.description = description
@@ -633,9 +621,10 @@ class TodoGroup(SecretYamlObject):
self.todos = todos
@classmethod
- def from_yaml(cls, loader, node):
+ @override
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
fields = loader.construct_mapping(node, deep=True)
- return TodoGroup(**fields)
+ return TodoGroup(**cast(dict[str, Any], fields))
def num_done(self):
return sum(1 for x in self.todos if x.is_done() > 0)
@@ -678,8 +667,7 @@ class TodoGroup(SecretYamlObject):
desc = self.description
if desc:
return expand_jinja(desc)
- else:
- return None
+ return None
def get_subtitle(self) -> str:
if self.depends:
@@ -700,26 +688,25 @@ class Todo(SecretYamlObject):
def __init__(
self,
- id,
- title,
- description=None,
- post_description=None,
- done=None,
- types=None,
- links=None,
- commands=None,
- user_input=None,
- depends=None,
- vars=None,
- asciidoc=None,
- persist_vars=None,
- function=None,
+ id: str,
+ title: str,
+ description: str | None = None,
+ post_description: str | None = None,
+ done: bool | None = None,
+ types: list[str] | str | None = None,
+ links: list[str] | None = None,
+ commands: Any = None,
+ user_input: Any = None,
+ depends: str | list[str] | None = None,
+ vars: dict[str, Any] | None = None,
+ asciidoc: str | None = None,
+ persist_vars: dict[str, str] | None = None,
+ function: str | None = None,
):
self.id = id
self.title = title
self.description = description
self.asciidoc = asciidoc
- self.types = types
self.depends = depends
self.vars = vars
self.persist_vars = persist_vars
@@ -728,38 +715,40 @@ class Todo(SecretYamlObject):
self.commands = commands
self.post_description = post_description
self.links = links
- self.state = {}
+ self.state: dict[Any, Any] = {}
self.set_done(done)
- if self.types:
- self.types = ensure_list(self.types)
- for t in self.types:
- if t not in ["minor", "major", "bugfix"]:
- sys.exit("Wrong Todo config for '%s'. Type needs to be either 'minor', 'major' or 'bugfix'" % self.id)
+ self.types: list[str] = ensure_list(types) if types else []
+
+ for t in self.types:
+ if t not in ["minor", "major", "bugfix"]:
+ sys.exit("Wrong Todo config for '%s'. Type needs to be either 'minor', 'major' or 'bugfix'" % self.id)
+
if self.commands:
self.commands.todo_id = self.id
for c in self.commands.commands:
c.todo_id = self.id
@classmethod
- def from_yaml(cls, loader, node):
+ @override
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
fields = loader.construct_mapping(node, deep=True)
- return Todo(**fields)
+ return Todo(**cast(dict[str, Any], fields))
def get_vars(self):
- myvars = {}
+ myvars: dict[str, str] = {}
if self.vars:
for k in self.vars:
val = self.vars[k]
if callable(val):
- myvars[k] = expand_jinja(val(), vars=myvars)
+ myvars[k] = expand_jinja(str(val()), vars=myvars)
else:
myvars[k] = expand_jinja(val, vars=myvars)
return myvars
- def set_done(self, is_done):
+ def set_done(self, is_done: bool | None):
if is_done:
- self.state["done_date"] = unix_time_millis(datetime.now(tz=timezone.utc))
+ self.state["done_date"] = unix_time_millis(datetime.now(tz=UTC))
if self.persist_vars:
for k in self.persist_vars:
self.state[k] = self.get_vars()[k]
@@ -767,7 +756,7 @@ class Todo(SecretYamlObject):
self.state.clear()
self.state["done"] = is_done
- def applies(self, type):
+ def applies(self, type: str):
if self.types:
return type in self.types
return True
@@ -856,14 +845,12 @@ class Todo(SecretYamlObject):
desc = self.description
if desc:
return expand_jinja(desc, vars=self.get_vars_and_state())
- else:
- return None
+ return None
def get_post_description(self):
if self.post_description:
return expand_jinja(self.post_description, vars=self.get_vars_and_state())
- else:
- return None
+ return None
def get_commands(self):
cmds = self.commands
@@ -872,8 +859,7 @@ class Todo(SecretYamlObject):
def get_asciidoc(self):
if self.asciidoc:
return expand_jinja(self.asciidoc, vars=self.get_vars_and_state())
- else:
- return None
+ return None
def get_vars_and_state(self):
d = self.get_vars().copy()
@@ -887,7 +873,7 @@ def get_release_version():
version = Version.parse(v)
except Exception:
print("Not a valid version %s" % v)
- return get_release_version()
+ raise
return str(version)
@@ -923,7 +909,7 @@ def reset_state():
state.clear()
-def template(name, vars=None):
+def template(name: str, vars: dict[str, Any] | None = None):
return expand_jinja(templates[name], vars=vars)
@@ -932,16 +918,15 @@ def help():
pause()
-def ensure_list(o):
+def ensure_list(o: Any):
if o is None:
- return []
+ return cast(list[Any], [])
if not isinstance(o, list):
return [o]
- else:
- return o
+ return cast(list[Any], o)
-def open_file(filename):
+def open_file(filename: str):
print("Opening file %s" % filename)
if platform.system().startswith("Win"):
run("start %s" % filename)
@@ -949,12 +934,12 @@ def open_file(filename):
run("open %s" % filename)
-def expand_multiline(cmd_txt, indent=0):
+def expand_multiline(cmd_txt: str, indent: int = 0):
return re.sub(r" +", " %s\n %s" % (Commands.cmd_continuation_char, " " * indent), cmd_txt)
-def unix_to_datetime(unix_stamp):
- return datetime.fromtimestamp(timestamp=unix_stamp / 1000, tz=timezone.utc)
+def unix_to_datetime(unix_stamp: int):
+ return datetime.fromtimestamp(timestamp=unix_stamp / 1000, tz=UTC)
def generate_asciidoc():
@@ -965,7 +950,7 @@ def generate_asciidoc():
fh = open(filename_adoc, "w")
fh.write("= Lucene Release %s\n\n" % state.release_version)
- fh.write("(_Generated by releaseWizard.py v%s at %s_)\n\n" % (getScriptVersion(), datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC")))
+ fh.write("(_Generated by releaseWizard.py v%s at %s_)\n\n" % (getScriptVersion(), datetime.now(tz=UTC).strftime("%Y-%m-%d %H:%M UTC")))
fh.write(":numbered:\n\n")
fh.write("%s\n\n" % template("help"))
for group in state.todo_groups:
@@ -992,8 +977,9 @@ def generate_asciidoc():
fh.write(".Variables collected in this step\n")
fh.write("|===\n")
fh.write("|Variable |Value\n")
- mykeys = set()
- for e in ensure_list(todo.user_input):
+ mykeys: set[str] = set()
+ myinputs: list[UserInput] = ensure_list(todo.user_input)
+ for e in myinputs:
mykeys.add(e.name)
for e in state_copy.keys():
mykeys.add(e)
@@ -1016,7 +1002,7 @@ def generate_asciidoc():
else:
fh.write("export %s=%s\n" % (key, val))
fh.write(abbreviate_homedir("cd %s\n" % cmds.get_root_folder()))
- cmds2 = ensure_list(cmds.commands)
+ cmds2: list[Command] = ensure_list(cmds.commands)
for c in cmds2:
for line in c.display_cmd():
fh.write("%s\n" % line)
@@ -1043,13 +1029,13 @@ def generate_asciidoc():
def load_rc():
lucenerc = os.path.expanduser("~/.lucenerc")
try:
- with open(lucenerc, "r") as fp:
- return json.load(fp)
+ with open(lucenerc) as fp:
+ return cast(dict[str, Any], json.load(fp))
except Exception:
- return {}
+ return cast(dict[str, Any], {})
-def store_rc(release_root, release_version=None):
+def store_rc(release_root: str, release_version: str | None = None):
lucenerc = os.path.expanduser("~/.lucenerc")
dict = {}
dict["root"] = release_root
@@ -1067,7 +1053,7 @@ def release_other_version():
sys.exit(0)
-def file_to_string(filename):
+def file_to_string(filename: str):
with open(filename, encoding="utf8") as f:
return f.read().strip()
@@ -1083,8 +1069,8 @@ def keys_downloaded():
def dump_yaml():
file = open(os.path.join(script_path, "releaseWizard.yaml"), "w")
yaml.add_representer(str, str_presenter)
- yaml.Dumper.ignore_aliases = lambda self, data: True
- dump_obj = {"templates": templates, "groups": state.todo_groups}
+ yaml.Dumper.ignore_aliases = lambda self, data: True # TODO: fix me # pyright: ignore[reportUnknownLambdaType]
+ dump_obj: dict[str, Any] = {"templates": templates, "groups": state.todo_groups}
yaml.dump(dump_obj, width=180, stream=file, sort_keys=False, default_flow_style=False)
@@ -1098,7 +1084,7 @@ def parse_config():
return config
-def load(urlString, encoding="utf-8"):
+def load(urlString: str, encoding: str = "utf-8"):
try:
content = urllib.request.urlopen(urlString).read().decode(encoding)
except Exception as e:
@@ -1107,7 +1093,7 @@ def load(urlString, encoding="utf-8"):
return content
-def configure_pgp(gpg_todo):
+def configure_pgp(gpg_todo: Todo):
print("Based on your Apache ID we'll lookup your key online\nand through this complete the 'gpg' prerequisite task.\n")
gpg_state = gpg_todo.get_state()
id = str(input("Please enter your Apache id: (ENTER=skip) "))
@@ -1137,7 +1123,7 @@ def configure_pgp(gpg_todo):
gpg_fingerprint = str(input("Enter your key fingerprint manually, all 40 characters (ENTER=skip): "))
if gpg_fingerprint.strip() == "":
return False
- elif len(gpg_fingerprint) != 40:
+ if len(gpg_fingerprint) != 40:
print("gpg fingerprint must be 40 characters long, do not just input the last 8")
gpg_fingerprint = gpg_fingerprint.upper()
gpg_id = gpg_fingerprint[-8:]
@@ -1252,16 +1238,18 @@ def configure_pgp(gpg_todo):
return True
-def pause(fun=None):
+def pause(fun: Callable[[], None] | None = None):
if fun:
fun()
input("\nPress ENTER to continue...")
class CustomExitItem(ExitItem):
- def show(self, index, available_width=None):
+ @override
+ def show(self, index: int, available_width: int | None = None):
return super(CustomExitItem, self).show(index)
+ @override
def get_return(self):
return ""
@@ -1314,7 +1302,7 @@ def main():
check_prerequisites()
try:
- y = yaml.load(open(os.path.join(script_path, "releaseWizard.yaml"), "r"), Loader=yaml.Loader)
+ y = yaml.load(open(os.path.join(script_path, "releaseWizard.yaml")), Loader=yaml.Loader)
templates = y.get("templates")
todo_list = y.get("groups")
state = ReleaseState(release_root, release_version, getScriptVersion())
@@ -1335,8 +1323,8 @@ def main():
main_menu = ConsoleMenu(
title="Lucene ReleaseWizard",
subtitle=get_releasing_text,
- prologue_text="Welcome to the release wizard. From here you can manage the process including creating new RCs. "
- "All changes are persisted, so you can exit any time and continue later. Make sure to read the Help section.",
+ prologue_text="""Welcome to the release wizard. From here you can manage the process including creating new RCs.
+ All changes are persisted, so you can exit any time and continue later. Make sure to read the Help section.""",
epilogue_text="® 2022 The Lucene project. Licensed under the Apache License 2.0\nScript version v%s)" % getScriptVersion(),
clear_screen=False,
)
@@ -1376,20 +1364,18 @@ def git_checkout_folder():
return state.get_git_checkout_folder()
-def tail_file(file, lines):
+def tail_file(file: str, lines: int):
bufsize = 8192
fsize = os.stat(file).st_size
with open(file) as f:
- if bufsize >= fsize:
- bufsize = fsize
+ bufsize = min(fsize, bufsize)
idx = 0
while True:
idx += 1
seek_pos = fsize - bufsize * idx
- if seek_pos < 0:
- seek_pos = 0
+ seek_pos = max(seek_pos, 0)
f.seek(seek_pos)
- data = []
+ data: list[str] = []
data.extend(f.readlines())
if len(data) >= lines or f.tell() == 0 or seek_pos == 0:
if not seek_pos == 0:
@@ -1398,7 +1384,7 @@ def tail_file(file, lines):
break
-def run_with_log_tail(command, cwd, logfile=None, tail_lines=10, tee=False, live=False, shell=None):
+def run_with_log_tail(command: str | list[str], cwd: str | None, logfile: str | None = None, tail_lines: int = 10, tee: bool | None = False, live: bool | None = False, shell: bool | None = None):
fh = sys.stdout
if logfile:
logdir = os.path.dirname(logfile)
@@ -1413,7 +1399,7 @@ def run_with_log_tail(command, cwd, logfile=None, tail_lines=10, tee=False, live
return rc
-def ask_yes_no(text):
+def ask_yes_no(text: str):
answer = None
while answer not in ["y", "n"]:
answer = str(input("\nQ: %s (y/n): " % text))
@@ -1421,7 +1407,7 @@ def ask_yes_no(text):
return answer == "y"
-def abbreviate_line(line, width):
+def abbreviate_line(line: str, width: int):
line = line.rstrip()
if len(line) > width:
line = "%s.....%s" % (line[: (width / 2 - 5)], line[-(width / 2) :])
@@ -1430,18 +1416,17 @@ def abbreviate_line(line, width):
return line
-def print_line_cr(line, linenum, stdout=True, tee=False):
+def print_line_cr(line: str, linenum: int, stdout: bool = True, tee: bool | None = False):
if not tee:
if not stdout:
print("[line %s] %s" % (linenum, abbreviate_line(line, 80)), end="\r")
+ elif line.endswith("\r"):
+ print(line.rstrip(), end="\r")
else:
- if line.endswith("\r"):
- print(line.rstrip(), end="\r")
- else:
- print(line.rstrip())
+ print(line.rstrip())
-def run_follow(command, cwd=None, fh=sys.stdout, tee=False, live=False, shell=None):
+def run_follow(command: str | list[str], cwd: str | None = None, fh: TextIO = sys.stdout, tee: bool | None = False, live: bool | None = False, shell: bool | None = None):
doShell = "&&" in command or "&" in command or shell is not None
if not doShell and not isinstance(command, list):
command = shlex.split(command)
@@ -1457,7 +1442,7 @@ def run_follow(command, cwd=None, fh=sys.stdout, tee=False, live=False, shell=No
fcntl.fcntl(process.stderr, fcntl.F_SETFL, flerr | os.O_NONBLOCK)
endstdout = endstderr = False
- errlines = []
+ errlines: list[str] = []
while not (endstderr and endstdout):
lines_before = lines_written
if not endstdout:
@@ -1536,7 +1521,18 @@ class Commands(SecretYamlObject):
cmd_continuation_char = "^" if is_windows() else "\\"
def __init__(
- self, root_folder, commands_text=None, commands=[], logs_prefix=None, run_text=None, enable_execute=None, confirm_each_command=None, env=None, vars=None, todo_id=None, remove_files=None
+ self,
+ root_folder: str,
+ commands_text: str | None = None,
+ commands: list[Any] | None = None,
+ logs_prefix: str | None = None,
+ run_text: str | None = None,
+ enable_execute: bool | None = None,
+ confirm_each_command: bool | None = None,
+ env: dict[str, str] | None = None,
+ vars: dict[str, Any] | None = None,
+ todo_id: str | None = None,
+ remove_files: list[str] | None = None,
):
self.root_folder = root_folder
self.commands_text = commands_text
@@ -1548,14 +1544,15 @@ class Commands(SecretYamlObject):
self.logs_prefix = logs_prefix
self.enable_execute = enable_execute
self.confirm_each_command = confirm_each_command
- self.commands = commands
+ self.commands: list[Command] = commands if commands else []
for c in self.commands:
c.todo_id = todo_id
@classmethod
- def from_yaml(cls, loader, node):
+ @override
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
fields = loader.construct_mapping(node, deep=True)
- return Commands(**fields)
+ return Commands(**cast(dict[str, Any], fields))
def run(self): # pylint: disable=inconsistent-return-statements # TODO
root = self.get_root_folder()
@@ -1571,7 +1568,7 @@ class Commands(SecretYamlObject):
else:
print("\n export %s=%s" % (key, val))
print(abbreviate_homedir("\n cd %s" % root))
- commands = ensure_list(self.commands)
+ commands: list[Command] = ensure_list(self.commands)
for cmd in commands:
for line in cmd.display_cmd():
print(" %s" % line)
@@ -1661,14 +1658,12 @@ class Commands(SecretYamlObject):
print("WARN: Command %s returned with error" % cmd.get_cmd())
success = False
break
- else:
- if cmd.should_fail and not dry_run:
- print("Expected command to fail, but it succeeded.")
- success = False
- break
- else:
- if elapsed > 30:
- print("Command completed in %s seconds" % elapsed)
+ elif cmd.should_fail and not dry_run:
+ print("Expected command to fail, but it succeeded.")
+ success = False
+ break
+ elif elapsed > 30:
+ print("Command completed in %s seconds" % elapsed)
if not success:
print("WARNING: One or more commands failed, you may want to check the logs")
return success
@@ -1686,17 +1681,17 @@ class Commands(SecretYamlObject):
return self.jinjaify(self.remove_files)
def get_vars(self):
- myvars = {}
+ myvars: dict[str, str] = {}
if self.vars:
for k in self.vars:
val = self.vars[k]
if callable(val):
- myvars[k] = expand_jinja(val(), vars=myvars)
+ myvars[k] = expand_jinja(str(val()), vars=myvars)
else:
myvars[k] = expand_jinja(val, vars=myvars)
return myvars
- def jinjaify(self, data, join=False):
+ def jinjaify(self, data: str | list[str] | None, join: bool = False):
if not data:
return None
v = self.get_vars()
@@ -1707,32 +1702,43 @@ class Commands(SecretYamlObject):
if isinstance(data, list):
if join:
return expand_jinja(" ".join(data), v)
- else:
- res = []
- for rf in data:
- res.append(expand_jinja(rf, v))
- return res
- else:
- return expand_jinja(data, v)
+ res: list[str] = []
+ for rf in data:
+ res.append(expand_jinja(rf, v))
+ return res
+ return expand_jinja(data, v)
-def abbreviate_homedir(line):
+def abbreviate_homedir(line: str):
if is_windows():
if "HOME" in os.environ:
return re.sub(r"([^/]|\b)%s" % os.path.expanduser("~"), "\\1%HOME%", line)
- elif "USERPROFILE" in os.environ:
+ if "USERPROFILE" in os.environ:
return re.sub(r"([^/]|\b)%s" % os.path.expanduser("~"), "\\1%USERPROFILE%", line)
- else:
- return line
- else:
- return re.sub(r"([^/]|\b)%s" % os.path.expanduser("~"), "\\1~", line)
+ return line
+ return re.sub(r"([^/]|\b)%s" % os.path.expanduser("~"), "\\1~", line)
class Command(SecretYamlObject):
yaml_tag = "!Command"
hidden_fields = ["todo_id"]
- def __init__(self, cmd, cwd=None, stdout=None, logfile=None, tee=None, live=None, comment=None, vars=None, todo_id=None, should_fail=None, redirect=None, redirect_append=None, shell=None):
+ def __init__(
+ self,
+ cmd: str,
+ cwd: str | None = None,
+ stdout: bool | None = None,
+ logfile: str | None = None,
+ tee: bool | None = None,
+ live: bool | None = None,
+ comment: str | None = None,
+ vars: dict[str, Any] | None = None,
+ todo_id: str | None = None,
+ should_fail: bool | None = None,
+ redirect: str | None = None,
+ redirect_append: bool | None = None,
+ shell: bool | None = None,
+ ):
self.cmd = cmd
self.cwd = cwd
self.comment = comment
@@ -1760,34 +1766,38 @@ class Command(SecretYamlObject):
print("Command %s specifies 'redirect' and other out options at the same time. Using redirect only" % self.cmd)
@classmethod
- def from_yaml(cls, loader, node):
+ @override
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
fields = loader.construct_mapping(node, deep=True)
- return Command(**fields)
+ return Command(**cast(dict[str, Any], fields))
def get_comment(self):
- return self.jinjaify(self.comment)
+ return str(self.jinjaify(self.comment))
def get_redirect(self):
- return self.jinjaify(self.redirect)
+ return str(self.jinjaify(self.redirect))
def get_cmd(self):
- return cast(str, self.jinjaify(self.cmd, join=True))
+ return str(self.jinjaify(self.cmd, join=True))
def get_vars(self):
- myvars = {}
+ myvars: dict[str, str] = {}
if self.vars:
for k in self.vars:
val = self.vars[k]
if callable(val):
- myvars[k] = expand_jinja(val(), vars=myvars)
+ myvars[k] = expand_jinja(str(val()), vars=myvars)
else:
myvars[k] = expand_jinja(val, vars=myvars)
return myvars
+ @override
def __str__(self):
return self.get_cmd()
- def jinjaify(self, data, join=False):
+ def jinjaify(self, data: str | list[str] | None, join: bool = False):
+ if data is None:
+ return None
v = self.get_vars()
if self.todo_id:
todo = state.get_todo_by_id(self.todo_id)
@@ -1796,16 +1806,14 @@ class Command(SecretYamlObject):
if isinstance(data, list):
if join:
return expand_jinja(" ".join(data), v)
- else:
- res = []
- for rf in data:
- res.append(expand_jinja(rf, v))
- return res
- else:
- return expand_jinja(data, v)
+ res: list[str] = []
+ for rf in data:
+ res.append(expand_jinja(rf, v))
+ return res
+ return expand_jinja(data, v)
def display_cmd(self):
- lines = []
+ lines: list[str] = []
if self.comment:
if is_windows():
lines.append("REM %s" % self.get_comment())
@@ -1826,17 +1834,18 @@ class Command(SecretYamlObject):
class UserInput(SecretYamlObject):
yaml_tag = "!UserInput"
- def __init__(self, name, prompt, type=None):
+ def __init__(self, name: str, prompt: str, type: str | None = None):
self.type = type
self.prompt = prompt
self.name = name
@classmethod
- def from_yaml(cls, loader, node):
+ @override
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
fields = loader.construct_mapping(node, deep=True)
- return UserInput(**fields)
+ return UserInput(**cast(dict[str, Any], fields))
- def run(self, dict=None):
+ def run(self, dict: dict[str, Any] | None = None):
correct = False
while not correct:
try:
@@ -1852,7 +1861,7 @@ class UserInput(SecretYamlObject):
return result
-def create_ical(_todo): # pylint: disable=unused-argument
+def create_ical(_todo: Todo): # pylint: disable=unused-argument
if ask_yes_no("Do you want to add a Calendar reminder for the close vote time?"):
# TODO: this library has broken typing and seems unmaintained, replace?
c = Calendar() # pyright: ignore[reportArgumentType]
@@ -1868,9 +1877,9 @@ def create_ical(_todo): # pylint: disable=unused-argument
return True
-today = datetime.now(tz=timezone.utc).date()
+today = datetime.now(tz=UTC).date()
sundays = {(today + timedelta(days=x)): "Sunday" for x in range(10) if (today + timedelta(days=x)).weekday() == 6}
-y = datetime.now(tz=timezone.utc).year
+y = datetime.now(tz=UTC).year
years = [y, y + 1]
non_working = (
country_holidays("CA", years=years)
@@ -1885,13 +1894,13 @@ non_working = (
def vote_close_72h_date():
# Voting open at least 72 hours according to ASF policy
- return datetime.now(tz=timezone.utc) + timedelta(hours=73)
+ return datetime.now(tz=UTC) + timedelta(hours=73)
def vote_close_72h_holidays():
days = 0
day_offset = -1
- holidays = []
+ holidays: list[str] = []
# Warn RM about major holidays coming up that should perhaps extend the voting deadline
# Warning will be given for Sunday or a public holiday observed by 3 or more [CA, US, EN, DE, NO, IND, RU]
while days < 3:
@@ -1899,15 +1908,14 @@ def vote_close_72h_holidays():
d = today + timedelta(days=day_offset)
if not (d in sundays or (d in non_working and len(non_working[d]) >= 2)):
days += 1
+ elif d in sundays:
+ holidays.append("%s (Sunday)" % d)
else:
- if d in sundays:
- holidays.append("%s (Sunday)" % d)
- else:
- holidays.append("%s (%s)" % (d, non_working[d]))
+ holidays.append("%s (%s)" % (d, non_working[d]))
return holidays if len(holidays) > 0 else None
-def prepare_announce_lucene(_todo): # pylint: disable=unused-argument
+def prepare_announce_lucene(_todo: Todo): # pylint: disable=unused-argument
if not os.path.exists(lucene_news_file):
lucene_text = expand_jinja("(( template=announce_lucene ))")
with open(lucene_news_file, "w") as fp:
@@ -1918,7 +1926,7 @@ def prepare_announce_lucene(_todo): # pylint: disable=unused-argument
return True
-def check_artifacts_available(_todo): # pylint: disable=unused-argument
+def check_artifacts_available(_todo: Todo): # pylint: disable=unused-argument
cdnUrl = expand_jinja("https://dlcdn.apache.org/lucene/java/{{ release_version }}/lucene-{{ release_version }}-src.tgz.asc")
try:
load(cdnUrl)
@@ -1938,14 +1946,14 @@ def check_artifacts_available(_todo): # pylint: disable=unused-argument
return True
-def set_java_home(version):
+def set_java_home(version: str):
os.environ["JAVA_HOME"] = state.get_java_home_for_version(version)
os.environ["JAVACMD"] = state.get_java_cmd_for_version(version)
-def load_lines(file, from_line=0):
+def load_lines(file: str, from_line: int = 0):
if os.path.exists(file):
- with open(file, "r") as fp:
+ with open(file) as fp:
return fp.readlines()[from_line:]
else:
return ["<Please paste the announcement text here>\n"]
diff --git a/dev-tools/scripts/releasedJirasRegex.py b/dev-tools/scripts/releasedJirasRegex.py
index f6887bc9923..0dd5ccd6360 100755
--- a/dev-tools/scripts/releasedJirasRegex.py
+++ b/dev-tools/scripts/releasedJirasRegex.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -33,7 +32,7 @@ from scriptutil import Version
# does not find Bugzilla bugs or JIRAs not mentioned at the beginning of
# bullets or numbered entries.
#
-def print_released_jiras_regex(version, filename):
+def print_released_jiras_regex(version: str, filename: str):
release_boundary_re = re.compile(r"\s*====*\s+(.*)\s+===")
version_re = re.compile(r"%s(?:$|[^-])" % version)
bullet_re = re.compile(r"\s*(?:[-*]|\d+\.(?=(?:\s|(?:LUCENE)-)))(.*)")
@@ -45,28 +44,26 @@ def print_released_jiras_regex(version, filename):
under_requested_version = False
requested_version_found = False
more_jiras_on_next_line = False
- lucene_jiras = []
- with open(filename, "r") as changes:
+ lucene_jiras: list[str] = []
+ with open(filename) as changes:
for line in changes:
version_boundary = release_boundary_re.match(line)
if version_boundary is not None:
if under_requested_version:
break # No longer under the requested version - stop looking for JIRAs
- else:
- if version_re.search(version_boundary.group(1)):
- under_requested_version = True # Start looking for JIRAs
- requested_version_found = True
- else:
- if under_requested_version:
- bullet_match = bullet_re.match(line)
- if more_jiras_on_next_line or bullet_match is not None:
- content = line if bullet_match is None else bullet_match.group(1)
- jira_list_match = jira_list_re.match(content)
- if jira_list_match is not None:
- jira_match = jira_re.findall(jira_list_match.group(0))
- for jira in jira_match:
- lucene_jiras.append(jira.rsplit("-", 1)[-1])
- more_jiras_on_next_line = more_jiras_on_next_line_re.match(content)
+ if version_re.search(version_boundary.group(1)):
+ under_requested_version = True # Start looking for JIRAs
+ requested_version_found = True
+ elif under_requested_version:
+ bullet_match = bullet_re.match(line)
+ if more_jiras_on_next_line or bullet_match is not None:
+ content = line if bullet_match is None else bullet_match.group(1)
+ jira_list_match = jira_list_re.match(content)
+ if jira_list_match is not None:
+ jira_match = jira_re.findall(jira_list_match.group(0))
+ for jira in jira_match:
+ lucene_jiras.append(jira.rsplit("-", 1)[-1])
+ more_jiras_on_next_line = more_jiras_on_next_line_re.match(content)
if not requested_version_found:
raise Exception("Could not find %s in %s" % (version, filename))
print()
diff --git a/dev-tools/scripts/reproduceJenkinsFailures.py b/dev-tools/scripts/reproduceJenkinsFailures.py
index 372091869ef..37b85c971a0 100644
--- a/dev-tools/scripts/reproduceJenkinsFailures.py
+++ b/dev-tools/scripts/reproduceJenkinsFailures.py
@@ -79,7 +79,7 @@ def readConfig():
return parser.parse_args()
-def runOutput(cmd):
+def runOutput(cmd: str):
print("[repro] %s" % cmd)
try:
return subprocess.check_output(cmd.split(" "), universal_newlines=True).strip()
@@ -88,23 +88,23 @@ def runOutput(cmd):
# Remembers non-zero exit code in lastFailureCode unless rememberFailure==False
-def run(cmd, rememberFailure=True):
+def run(cmd: str, rememberFailure: bool = True):
global lastFailureCode
print("[repro] %s" % cmd)
code = os.system(cmd)
- if 0 != code and rememberFailure:
+ if code != 0 and rememberFailure:
print("\n[repro] Setting last failure code to %d\n" % code)
lastFailureCode = code
return code
-def fetchAndParseJenkinsLog(url, numRetries):
+def fetchAndParseJenkinsLog(url: str, numRetries: int) -> dict[str, str]:
global revisionFromLog
global branchFromLog
global antOptions
revisionFromLog = None
antOptions = ""
- tests = {}
+ tests: dict[str, str] = {}
print("[repro] Jenkins log URL: %s\n" % url)
try:
# HTTPS fails at certificate validation, see LUCENE-9412, PEP-476
@@ -137,50 +137,48 @@ def fetchAndParseJenkinsLog(url, numRetries):
print("[repro] Encountered IncompleteRead exception, pausing and then retrying...")
time.sleep(2) # pause for 2 seconds
return fetchAndParseJenkinsLog(url, numRetries - 1)
- else:
- print("[repro] Encountered IncompleteRead exception, aborting after too many retries.")
- raise RuntimeError("ERROR: fetching %s : %s" % (url, e))
+ print("[repro] Encountered IncompleteRead exception, aborting after too many retries.")
+ raise RuntimeError("ERROR: fetching %s : %s" % (url, e))
if revisionFromLog is None:
if reJenkinsURLWithoutConsoleText.match(url):
print('[repro] Not a Jenkins log. Appending "/consoleText" and retrying ...\n')
return fetchAndParseJenkinsLog(url + "/consoleText", numRetries)
- else:
- raise RuntimeError("ERROR: %s does not appear to be a Jenkins log." % url)
- if 0 == len(tests):
+ raise RuntimeError("ERROR: %s does not appear to be a Jenkins log." % url)
+ if len(tests) == 0:
print('[repro] No "reproduce with" lines found; exiting.')
sys.exit(0)
return tests
-def prepareWorkspace(useGit, gitRef):
+def prepareWorkspace(useGit: bool, gitRef: str | None):
global gitCheckoutSucceeded
if useGit:
code = run("git fetch")
- if 0 != code:
+ if code != 0:
raise RuntimeError('ERROR: "git fetch" failed. See above.')
checkoutCmd = "git checkout %s" % gitRef
code = run(checkoutCmd)
- if 0 != code:
+ if code != 0:
addWantedBranchCmd = "git remote set-branches --add origin %s" % gitRef
checkoutBranchCmd = "git checkout -t -b %s origin/%s" % (gitRef, gitRef) # Checkout remote branch as new local branch
print('"%s" failed. Trying "%s" and "%s".' % (checkoutCmd, addWantedBranchCmd, checkoutBranchCmd))
code = run(addWantedBranchCmd)
- if 0 != code:
+ if code != 0:
raise RuntimeError('ERROR: "%s" failed. See above.' % addWantedBranchCmd)
code = run(checkoutBranchCmd)
- if 0 != code:
+ if code != 0:
raise RuntimeError('ERROR: "%s" failed. See above.' % checkoutBranchCmd)
gitCheckoutSucceeded = True
run("git merge --ff-only", rememberFailure=False) # Ignore failure on non-branch ref
code = run("ant clean")
- if 0 != code:
+ if code != 0:
raise RuntimeError('ERROR: "ant clean" failed. See above.')
-def groupTestsByModule(tests):
- modules = {}
+def groupTestsByModule(tests: dict[str, str]):
+ modules: dict[str, set[str]] = {}
for dir, _, files in os.walk("."):
for file in files:
match = reJavaFile.search(file)
@@ -201,7 +199,7 @@ def groupTestsByModule(tests):
return modules
-def runTests(testIters, modules, tests):
+def runTests(testIters: int, modules: dict[str, set[str]], tests: dict[str, str]):
cwd = os.getcwd()
testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s %s'
for module in modules:
@@ -212,15 +210,15 @@ def runTests(testIters, modules, tests):
os.chdir(module)
code = run("ant compile-test")
try:
- if 0 != code:
+ if code != 0:
raise RuntimeError("ERROR: Compile failed in %s/ with code %d. See above." % (module, code))
run(testCmdline % (testIters, testIters * numTests, testList, antOptions, params))
finally:
os.chdir(cwd)
-def printAndMoveReports(testIters, newSubDir, location):
- failures = {}
+def printAndMoveReports(testIters: int, newSubDir: str, location: str):
+ failures: dict[str, int] = {}
for start in ("lucene/build", "solr/build"):
for dir, _, files in os.walk(start):
for file in files:
@@ -275,7 +273,7 @@ def main():
if config.useGit:
# Retest 100% failures at the tip of the branch
oldTests = tests
- tests = {}
+ tests: dict[str, str] = {}
for fullClass in failures:
testcase = fullClass[(fullClass.rindex(".") + 1) :]
if failures[fullClass] == config.testIters:
diff --git a/dev-tools/scripts/scriptutil.py b/dev-tools/scripts/scriptutil.py
index 4b054dfaa0c..01c7e9d81ba 100644
--- a/dev-tools/scripts/scriptutil.py
+++ b/dev-tools/scripts/scriptutil.py
@@ -22,11 +22,15 @@ import time
import urllib.error
import urllib.parse
import urllib.request
+from collections.abc import Callable
from enum import Enum
+from re import Match, Pattern
+from typing import Self, override
-class Version(object):
- def __init__(self, major, minor, bugfix, prerelease):
+class Version:
+ def __init__(self, major: int, minor: int, bugfix: int, prerelease: int):
+ super().__init__()
self.major = major
self.minor = minor
self.bugfix = bugfix
@@ -36,7 +40,7 @@ class Version(object):
self.constant = "LUCENE_%d_%d_%d" % (self.major, self.minor, self.bugfix)
@classmethod
- def parse(cls, value):
+ def parse(cls, value: str):
match = re.search(r"(\d+)\.(\d+).(\d+)(.1|.2)?", value)
if match is None:
raise argparse.ArgumentTypeError("Version argument must be of format x.y.z(.1|.2)?")
@@ -44,10 +48,11 @@ class Version(object):
parts.append({None: 0, ".1": 1, ".2": 2}[match.groups()[-1]])
return Version(*parts)
+ @override
def __str__(self):
return self.dot
- def make_previous_matcher(self, prefix="", suffix="", sep="\\."):
+ def make_previous_matcher(self, prefix: str = "", suffix: str = "", sep: str = "\\."):
if self.is_bugfix_release():
pattern = "%s%s%s%s%d" % (self.major, sep, self.minor, sep, self.bugfix - 1)
elif self.is_minor_release():
@@ -66,23 +71,21 @@ class Version(object):
def is_major_release(self):
return self.bugfix == 0 and self.minor == 0
- def on_or_after(self, other):
- return (
- self.major > other.major
- or self.major == other.major
- and (self.minor > other.minor or self.minor == other.minor and (self.bugfix > other.bugfix or self.bugfix == other.bugfix and self.prerelease >= other.prerelease))
+ def on_or_after(self, other: Self):
+ return self.major > other.major or (
+ self.major == other.major and (self.minor > other.minor or (self.minor == other.minor and (self.bugfix > other.bugfix or (self.bugfix == other.bugfix and self.prerelease >= other.prerelease))))
)
- def gt(self, other):
+ def gt(self, other: Self):
return self.major > other.major or (self.major == other.major and self.minor > other.minor) or (self.major == other.major and self.minor == other.minor and self.bugfix > other.bugfix)
- def is_back_compat_with(self, other):
+ def is_back_compat_with(self, other: Self):
if not self.on_or_after(other):
raise Exception("Back compat check disallowed for newer version: %s < %s" % (self, other))
return other.major + 1 >= self.major
-def run(cmd, cwd=None):
+def run(cmd: str, cwd: str | None = None):
try:
output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, cwd=cwd)
except subprocess.CalledProcessError as e:
@@ -91,9 +94,9 @@ def run(cmd, cwd=None):
return output.decode("utf-8")
-def update_file(filename, line_re, edit, append=None):
- infile = open(filename, "r")
- buffer = []
+def update_file(filename: str, line_re: Pattern[str], edit: Callable[[list[str], Match[str], str], bool | None], append: Callable[[list[str], bool], bool] | None = None):
+ infile = open(filename)
+ buffer: list[str] = []
changed = False
for line in infile:
@@ -139,7 +142,7 @@ def find_branch_type():
raise Exception("Cannot run %s on feature branch" % sys.argv[0].rsplit("/", 1)[-1])
-def download(name, urlString, tmpDir, quiet=False, force_clean=True):
+def download(name: str, urlString: str, tmpDir: str, quiet: bool = False, force_clean: bool = True):
if not quiet:
print("Downloading %s" % urlString)
startTime = time.time()
@@ -162,7 +165,7 @@ def download(name, urlString, tmpDir, quiet=False, force_clean=True):
print(" %.1f MB in %.2f sec (%.1f MB/sec)" % (sizeMB, t, sizeMB / t))
-def attemptDownload(urlString, fileName):
+def attemptDownload(urlString: str, fileName: str):
fIn = urllib.request.urlopen(urlString)
fOut = open(fileName, "wb")
success = False
diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py
index e29d95ca4ab..05f479c97a3 100755
--- a/dev-tools/scripts/smokeTestRelease.py
+++ b/dev-tools/scripts/smokeTestRelease.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -36,6 +35,8 @@ import urllib.request
import xml.etree.ElementTree as ET
import zipfile
from collections import namedtuple
+from collections.abc import Callable
+from typing import Any
import scriptutil
@@ -49,14 +50,14 @@ cygwin = platform.system().lower().startswith("cygwin")
cygwinWindowsRoot = os.popen("cygpath -w /").read().strip().replace("\\", "/") if cygwin else ""
-def unshortenURL(url):
+def unshortenURL(url: str):
parsed = urllib.parse.urlparse(url)
if parsed[0] in ("http", "https"):
h = http.client.HTTPConnection(parsed.netloc)
h.request("HEAD", parsed.path)
response = h.getresponse()
if int(response.status / 100) == 3 and response.getheader("Location"):
- return response.getheader("Location")
+ return str(response.getheader("Location"))
return url
@@ -70,7 +71,7 @@ reHREF = re.compile('<a href="(.*?)">(.*?)</a>')
FORCE_CLEAN = True
-def getHREFs(urlString):
+def getHREFs(urlString: str):
# Deref any redirects
while True:
url = urllib.parse.urlparse(urlString)
@@ -88,7 +89,7 @@ def getHREFs(urlString):
else:
break
- links = []
+ links: list[tuple[str, str]] = []
try:
html = load(urlString)
except:
@@ -102,7 +103,7 @@ def getHREFs(urlString):
return links
-def load(urlString):
+def load(urlString: str):
try:
content = urllib.request.urlopen(urlString).read().decode("utf-8")
except Exception as e:
@@ -111,14 +112,14 @@ def load(urlString):
return content
-def noJavaPackageClasses(desc, file):
+def noJavaPackageClasses(desc: str, file: str):
with zipfile.ZipFile(file) as z2:
for name2 in z2.namelist():
if name2.endswith(".class") and (name2.startswith("java/") or name2.startswith("javax/")):
raise RuntimeError('%s contains sheisty class "%s"' % (desc, name2))
-def decodeUTF8(bytes):
+def decodeUTF8(bytes: bytes):
return codecs.getdecoder("UTF-8")(bytes)[0]
@@ -127,7 +128,7 @@ NOTICE_FILE_NAME = "META-INF/NOTICE.txt"
LICENSE_FILE_NAME = "META-INF/LICENSE.txt"
-def checkJARMetaData(desc, jarFile, gitRevision, version):
+def checkJARMetaData(desc: str, jarFile: str, gitRevision: str, version: str):
with zipfile.ZipFile(jarFile, "r") as z:
for name in (MANIFEST_FILE_NAME, NOTICE_FILE_NAME, LICENSE_FILE_NAME):
try:
@@ -159,8 +160,7 @@ def checkJARMetaData(desc, jarFile, gitRevision, version):
else:
if len(verify) == 1:
raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFEST.MF: %s' % (desc, verify[0], s))
- else:
- raise RuntimeError('%s is missing one of "%s" inside its META-INF/MANIFEST.MF: %s' % (desc, verify, s))
+ raise RuntimeError('%s is missing one of "%s" inside its META-INF/MANIFEST.MF: %s' % (desc, verify, s))
if gitRevision != "skip":
# Make sure this matches the version and git revision we think we are releasing:
@@ -186,11 +186,11 @@ def checkJARMetaData(desc, jarFile, gitRevision, version):
raise RuntimeError("%s: %s contents doesn't match main LICENSE.txt" % (desc, LICENSE_FILE_NAME))
-def normSlashes(path):
+def normSlashes(path: str):
return path.replace(os.sep, "/")
-def checkAllJARs(topDir, gitRevision, version):
+def checkAllJARs(topDir: str, gitRevision: str, version: str):
print(" verify JAR metadata/identity/no javax.* or java.* classes...")
for root, _, files in os.walk(topDir):
normRoot = normSlashes(root)
@@ -205,24 +205,24 @@ def checkAllJARs(topDir, gitRevision, version):
checkJARMetaData('JAR file "%s"' % fullPath, fullPath, gitRevision, version)
-def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
+def checkSigs(urlString: str, version: str, tmpDir: str, isSigned: bool, keysFile: str):
print(" test basics...")
ents = getDirEntries(urlString)
artifact = None
changesURL = None
mavenURL = None
artifactURL = None
- expectedSigs = []
+ expectedSigs: list[str] = []
if isSigned:
expectedSigs.append("asc")
expectedSigs.extend(["sha512"])
- sigs = []
- artifacts = []
+ sigs: list[str] = []
+ artifacts: list[tuple[str, str]] = []
for text, subURL in ents:
if text == "KEYS":
raise RuntimeError("lucene: release dir should not contain a KEYS file - only toplevel /dist/lucene/KEYS is used")
- elif text == "maven/":
+ if text == "maven/":
mavenURL = subURL
elif text.startswith("changes"):
if text not in ("changes/", "changes-%s/" % version):
@@ -240,12 +240,15 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
else:
if sigs != expectedSigs:
raise RuntimeError("lucene: artifact %s has wrong sigs: expected %s but got %s" % (artifact, expectedSigs, sigs))
+ assert artifactURL is not None
artifacts.append((artifact, artifactURL))
artifact = text
artifactURL = subURL
sigs = []
if sigs != []:
+ assert artifact is not None
+ assert artifactURL is not None
artifacts.append((artifact, artifactURL))
if sigs != expectedSigs:
raise RuntimeError("lucene: artifact %s has wrong sigs: expected %s but got %s" % (artifact, expectedSigs, sigs))
@@ -302,7 +305,7 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
print(" GPG: %s" % line.strip())
-def testChanges(version, changesURLString):
+def testChanges(version: str, changesURLString: str):
print(" check changes HTML...")
changesURL = None
for text, subURL in getDirEntries(changesURLString):
@@ -316,7 +319,7 @@ def testChanges(version, changesURLString):
checkChangesContent(s, version, changesURL, True)
-def testChangesText(dir, version):
+def testChangesText(dir: str, version: str):
"Checks all CHANGES.txt under this dir."
for root, _, files in os.walk(dir):
# NOTE: O(N) but N should be smallish:
@@ -331,7 +334,7 @@ reUnderbarNotDashHTML = re.compile(r"<li>(\s*(LUCENE)_\d\d\d\d+)")
reUnderbarNotDashTXT = re.compile(r"\s+((LUCENE)_\d\d\d\d+)", re.MULTILINE)
-def checkChangesContent(s, version, name, isHTML):
+def checkChangesContent(s: str, version: str, name: str, isHTML: bool):
currentVersionTuple = versionToTuple(version, name)
if isHTML and s.find("Release %s" % version) == -1:
@@ -359,8 +362,8 @@ def checkChangesContent(s, version, name, isHTML):
if isHTML:
# Make sure that a section only appears once under each release,
# and that each release is not greater than the current version
- seenIDs = set()
- seenText = set()
+ seenIDs: set[str] = set()
+ seenText: set[str] = set()
release = None
for id, text in reChangesSectionHREF.findall(s):
@@ -381,7 +384,7 @@ def checkChangesContent(s, version, name, isHTML):
reVersion = re.compile(r"(\d+)\.(\d+)(?:\.(\d+))?\s*(-alpha|-beta|final|RC\d+)?\s*(?:\[.*\])?", re.IGNORECASE)
-def versionToTuple(version, name):
+def versionToTuple(version: str, name: str):
versionMatch = reVersion.match(version)
if versionMatch is None:
raise RuntimeError("Version %s in %s cannot be parsed" % (version, name))
@@ -402,7 +405,7 @@ def versionToTuple(version, name):
reUnixPath = re.compile(r'\b[a-zA-Z_]+=(?:"(?:\\"|[^"])*"' + "|(?:\\\\.|[^\"'\\s])*" + r"|'(?:\\'|[^'])*')" + r'|(/(?:\\.|[^"\'\s])*)' + r'|("/(?:\\.|[^"])*")' + r"|('/(?:\\.|[^'])*')")
-def unix2win(matchobj):
+def unix2win(matchobj: re.Match[str]):
if matchobj.group(1) is not None:
return cygwinWindowsRoot + matchobj.group()
if matchobj.group(2) is not None:
@@ -412,7 +415,7 @@ def unix2win(matchobj):
return matchobj.group()
-def cygwinifyPaths(command):
+def cygwinifyPaths(command: str):
# The problem: Native Windows applications running under Cygwin can't
# handle Cygwin's Unix-style paths. However, environment variable
# values are automatically converted, so only paths outside of
@@ -423,7 +426,7 @@ def cygwinifyPaths(command):
return command
-def printFileContents(fileName):
+def printFileContents(fileName: str):
# Assume log file was written in system's default encoding, but
# even if we are wrong, we replace errors ... the ASCII chars
# (which is what we mostly care about eg for the test seed) should
@@ -440,7 +443,7 @@ def printFileContents(fileName):
print()
-def run(command, logFile):
+def run(command: str, logFile: str):
if cygwin:
command = cygwinifyPaths(command)
if os.system("%s > %s 2>&1" % (command, logFile)):
@@ -450,7 +453,7 @@ def run(command, logFile):
raise RuntimeError('command "%s" failed; see log file %s' % (command, logPath))
-def verifyDigests(artifact, urlString, tmpDir):
+def verifyDigests(artifact: str, urlString: str, tmpDir: str):
print(" verify sha512 digest")
sha512Expected, t = load(urlString + ".sha512").strip().split()
if t != "*" + artifact:
@@ -469,18 +472,17 @@ def verifyDigests(artifact, urlString, tmpDir):
raise RuntimeError("SHA512 digest mismatch for %s: expected %s but got %s" % (artifact, sha512Expected, sha512Actual))
-def getDirEntries(urlString):
+def getDirEntries(urlString: str):
if urlString.startswith("file:/") and not urlString.startswith("file://"):
# stupid bogus ant URI
urlString = "file:///" + urlString[6:]
if urlString.startswith("file://"):
path = urlString[7:]
- if path.endswith("/"):
- path = path[:-1]
+ path = path.removesuffix("/")
if cygwin: # Convert Windows path to Cygwin path
path = re.sub(r"^/([A-Za-z]):/", r"/cygdrive/\1/", path)
- files = []
+ files: list[tuple[str, str]] = []
for ent in os.listdir(path):
entPath = "%s/%s" % (path, ent)
if os.path.isdir(entPath):
@@ -489,15 +491,14 @@ def getDirEntries(urlString):
files.append((ent, "file://%s" % entPath))
files.sort()
return files
- else:
- links = getHREFs(urlString)
- for i, (text, _) in enumerate(links):
- if text == "Parent Directory" or text == "..":
- return links[(i + 1) :]
- raise RuntimeError("could not enumerate %s" % (urlString))
+ links = getHREFs(urlString)
+ for i, (text, _) in enumerate(links):
+ if text == "Parent Directory" or text == "..":
+ return links[(i + 1) :]
+ raise RuntimeError("could not enumerate %s" % (urlString))
-def unpackAndVerify(java, tmpDir, artifact, gitRevision, version, testArgs):
+def unpackAndVerify(java: Any, tmpDir: str, artifact: str, gitRevision: str, version: str, testArgs: str):
destDir = "%s/unpack" % tmpDir
if os.path.exists(destDir):
shutil.rmtree(destDir)
@@ -525,7 +526,7 @@ LUCENE_NOTICE = None
LUCENE_LICENSE = None
-def is_in_list(in_folder, files, indent=4):
+def is_in_list(in_folder: list[str], files: list[str], indent: int = 4):
for fileName in files:
print("%sChecking %s" % (" " * indent, fileName))
found = False
@@ -537,7 +538,7 @@ def is_in_list(in_folder, files, indent=4):
raise RuntimeError('file "%s" is missing' % fileName)
-def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
+def verifyUnpacked(java: Any, artifact: str, unpackPath: str, gitRevision: str, version: str, testArgs: str):
global LUCENE_NOTICE
global LUCENE_LICENSE
@@ -547,7 +548,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
# Check text files in release
print(" %s" % artifact)
in_root_folder = list(filter(lambda x: x[0] != ".", os.listdir(unpackPath)))
- in_lucene_folder = []
+ in_lucene_folder: list[str] = []
if isSrc:
in_lucene_folder.extend(os.listdir(os.path.join(unpackPath, "lucene")))
is_in_list(in_root_folder, ["LICENSE", "NOTICE", "README"])
@@ -656,7 +657,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
testDemo(java.run_java, isSrc, version, BASE_JAVA_VERSION)
if java.run_alt_javas:
- for run_alt_java, alt_java_version in zip(java.run_alt_javas, java.alt_java_versions):
+ for run_alt_java, alt_java_version in zip(java.run_alt_javas, java.alt_java_versions, strict=False):
print(" run tests w/ Java %s and testArgs='%s'..." % (alt_java_version, testArgs))
run_alt_java("./gradlew --no-daemon test %s" % testArgs, "%s/test.log" % unpackPath)
print(" compile jars w/ Java %s" % alt_java_version)
@@ -671,13 +672,13 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
testDemo(java.run_java, isSrc, version, BASE_JAVA_VERSION)
if java.run_alt_javas:
- for run_alt_java, alt_java_version in zip(java.run_alt_javas, java.alt_java_versions):
+ for run_alt_java, alt_java_version in zip(java.run_alt_javas, java.alt_java_versions, strict=False):
testDemo(run_alt_java, isSrc, version, alt_java_version)
testChangesText(".", version)
-def testDemo(run_java, isSrc, version, jdk):
+def testDemo(run_java: Callable[[str, str], None], isSrc: bool, version: str, jdk: str):
if os.path.exists("index"):
shutil.rmtree("index") # nuke any index from any previous iteration
@@ -710,11 +711,10 @@ def testDemo(run_java, isSrc, version, jdk):
m = reMatchingDocs.search(open("search.log", encoding="UTF-8").read())
if m is None:
raise RuntimeError("lucene demo's SearchFiles found no results")
- else:
- numHits = int(m.group(1))
- if numHits < 100:
- raise RuntimeError("lucene demo's SearchFiles found too few results: %s" % numHits)
- print(' got %d hits for query "lucene"' % numHits)
+ numHits = int(m.group(1))
+ if numHits < 100:
+ raise RuntimeError("lucene demo's SearchFiles found too few results: %s" % numHits)
+ print(' got %d hits for query "lucene"' % numHits)
print(" checkindex with %s..." % jdk)
run_java(checkIndexCmd, "checkindex.log")
@@ -727,13 +727,13 @@ def testDemo(run_java, isSrc, version, jdk):
raise RuntimeError('wrong version from CheckIndex: got "%s" but expected "%s"' % (actualVersion, version))
-def removeTrailingZeros(version):
+def removeTrailingZeros(version: str):
return re.sub(r"(\.0)*$", "", version)
-def checkMaven(baseURL, tmpDir, gitRevision, version, isSigned, keysFile):
+def checkMaven(baseURL: str, tmpDir: str, gitRevision: str, version: str, isSigned: bool, keysFile: str):
print(" download artifacts")
- artifacts = []
+ artifacts: list[str] = []
artifactsURL = "%s/lucene/maven/org/apache/lucene/" % baseURL
targetDir = "%s/maven/org/apache/lucene" % tmpDir
if not os.path.exists(targetDir):
@@ -753,7 +753,7 @@ def checkMaven(baseURL, tmpDir, gitRevision, version, isSigned, keysFile):
checkAllJARs("%s/maven/org/apache/lucene" % tmpDir, gitRevision, version)
-def getBinaryDistFiles(tmpDir, version, baseURL):
+def getBinaryDistFiles(tmpDir: str, version: str, baseURL: str):
distribution = "lucene-%s.tgz" % version
if not os.path.exists("%s/%s" % (tmpDir, distribution)):
distURL = "%s/lucene/%s" % (baseURL, distribution)
@@ -767,13 +767,13 @@ def getBinaryDistFiles(tmpDir, version, baseURL):
print(" unpack %s..." % distribution)
unpackLogFile = "%s/unpack-%s-getBinaryDistFiles.log" % (tmpDir, distribution)
run("tar xzf %s/%s" % (tmpDir, distribution), unpackLogFile)
- distributionFiles = []
+ distributionFiles: list[str] = []
for root, _, files in os.walk(destDir):
distributionFiles.extend([os.path.join(root, file) for file in files])
return distributionFiles
-def checkJavadocAndSourceArtifacts(artifacts, version):
+def checkJavadocAndSourceArtifacts(artifacts: list[str], version: str):
print(" check for javadoc and sources artifacts...")
for artifact in artifacts:
if artifact.endswith(version + ".jar"):
@@ -785,8 +785,8 @@ def checkJavadocAndSourceArtifacts(artifacts, version):
raise RuntimeError("missing: %s" % sourcesJar)
-def getZipFileEntries(fileName):
- entries = []
+def getZipFileEntries(fileName: str):
+ entries: list[str] = []
with zipfile.ZipFile(fileName) as zf:
for zi in zf.infolist():
entries.append(zi.filename)
@@ -795,10 +795,10 @@ def getZipFileEntries(fileName):
return entries
-def checkIdenticalMavenArtifacts(distFiles, artifacts, version):
+def checkIdenticalMavenArtifacts(distFiles: list[str], artifacts: list[str], version: str):
print(" verify that Maven artifacts are same as in the binary distribution...")
reJarWar = re.compile(r"%s\.[wj]ar$" % version) # exclude *-javadoc.jar and *-sources.jar
- distFilenames = dict()
+ distFilenames: dict[str, str] = dict()
for file in distFiles:
baseName = os.path.basename(file)
distFilenames[baseName] = file
@@ -807,13 +807,12 @@ def checkIdenticalMavenArtifacts(distFiles, artifacts, version):
artifactFilename = os.path.basename(artifact)
if artifactFilename not in distFilenames:
raise RuntimeError("Maven artifact %s is not present in lucene binary distribution" % artifact)
- else:
- identical = filecmp.cmp(artifact, distFilenames[artifactFilename], shallow=False)
- if not identical:
- raise RuntimeError("Maven artifact %s is not identical to %s in lucene binary distribution" % (artifact, distFilenames[artifactFilename]))
+ identical = filecmp.cmp(artifact, distFilenames[artifactFilename], shallow=False)
+ if not identical:
+ raise RuntimeError("Maven artifact %s is not identical to %s in lucene binary distribution" % (artifact, distFilenames[artifactFilename]))
-def verifyMavenDigests(artifacts):
+def verifyMavenDigests(artifacts: list[str]):
print(" verify Maven artifacts' md5/sha1 digests...")
reJarWarPom = re.compile(r"\.(?:[wj]ar|pom)$")
for artifactFile in [a for a in artifacts if reJarWarPom.search(a)]:
@@ -843,23 +842,37 @@ def verifyMavenDigests(artifacts):
raise RuntimeError("SHA1 digest mismatch for %s: expected %s but got %s" % (artifactFile, sha1Expected, sha1Actual))
-def getPOMcoordinate(treeRoot):
+def getPOMcoordinate(treeRoot: ET.Element):
namespace = "{http://maven.apache.org/POM/4.0.0}"
groupId = treeRoot.find("%sgroupId" % namespace)
if groupId is None:
- groupId = treeRoot.find("{0}parent/{0}groupId".format(namespace))
+ groupId = treeRoot.find(f"{namespace}parent/{namespace}groupId")
+ assert groupId is not None
+ assert groupId.text is not None
groupId = groupId.text.strip()
- artifactId = treeRoot.find("%sartifactId" % namespace).text.strip()
+
+ artifactId = treeRoot.find("%sartifactId" % namespace)
+ assert artifactId is not None
+ assert artifactId.text is not None
+ artifactId = artifactId.text.strip()
+
version = treeRoot.find("%sversion" % namespace)
if version is None:
- version = treeRoot.find("{0}parent/{0}version".format(namespace))
+ version = treeRoot.find(f"{namespace}parent/{namespace}version")
+ assert version is not None
+ assert version.text is not None
version = version.text.strip()
+
packaging = treeRoot.find("%spackaging" % namespace)
- packaging = "jar" if packaging is None else packaging.text.strip()
+ if packaging is None:
+ packaging = "jar"
+ else:
+ assert packaging.text is not None
+ packaging = packaging.text.strip()
return groupId, artifactId, packaging, version
-def verifyMavenSigs(tmpDir, artifacts, keysFile):
+def verifyMavenSigs(tmpDir: str, artifacts: list[str], keysFile: str):
print(" verify maven artifact sigs", end=" ")
# Set up clean gpg world; import keys file:
@@ -891,14 +904,14 @@ def verifyMavenSigs(tmpDir, artifacts, keysFile):
print()
-def print_warnings_in_file(file):
+def print_warnings_in_file(file: str):
with open(file) as f:
for line in f.readlines():
if line.lower().find("warning") != -1 and line.find("WARNING: This key is not certified with a trusted signature") == -1 and line.find("WARNING: using insecure memory") == -1:
print(" GPG: %s" % line.strip())
-def verifyPOMperBinaryArtifact(artifacts, version):
+def verifyPOMperBinaryArtifact(artifacts: list[str], version: str):
print(" verify that each binary artifact has a deployed POM...")
reBinaryJarWar = re.compile(r"%s\.[jw]ar$" % re.escape(version))
for artifact in [a for a in artifacts if reBinaryJarWar.search(a)]:
@@ -907,9 +920,8 @@ def verifyPOMperBinaryArtifact(artifacts, version):
raise RuntimeError("missing: POM for %s" % artifact)
-def verifyDeployedPOMsCoordinates(artifacts, version):
- """
- verify that each POM's coordinate (drawn from its content) matches
+def verifyDeployedPOMsCoordinates(artifacts: list[str], version: str):
+ """Verify that each POM's coordinate (drawn from its content) matches
its filepath, and verify that the corresponding artifact exists.
"""
print(" verify deployed POMs' coordinates...")
@@ -925,9 +937,10 @@ def verifyDeployedPOMsCoordinates(artifacts, version):
raise RuntimeError("Missing corresponding .%s artifact for POM %s" % (packaging, POM))
-def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
+def crawl(downloadedFiles: list[str], urlString: str, targetDir: str, exclusions: set[str] | None = None):
+ exclude: set[str] = exclusions if exclusions else set()
for text, subURL in getDirEntries(urlString):
- if text not in exclusions:
+ if text not in exclude:
path = os.path.join(targetDir, text)
if text.endswith("/"):
if not os.path.exists(path):
@@ -940,8 +953,8 @@ def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
sys.stdout.write(".")
-def make_java_config(parser, alt_java_homes):
- def _make_runner(java_home, is_base_version=False):
+def make_java_config(parser: argparse.ArgumentParser, alt_java_homes: list[str]):
+ def _make_runner(java_home: str, is_base_version: bool = False):
if cygwin:
java_home = subprocess.check_output('cygpath -u "%s"' % java_home, shell=True).decode("utf-8").strip()
cmd_prefix = 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (java_home, java_home, java_home)
@@ -954,13 +967,12 @@ def make_java_config(parser, alt_java_homes):
# validate Java version
if is_base_version:
- if BASE_JAVA_VERSION != actual_version:
+ if actual_version != BASE_JAVA_VERSION:
parser.error("got wrong base version for java %s:\n%s" % (BASE_JAVA_VERSION, s))
- else:
- if int(actual_version) < int(BASE_JAVA_VERSION):
- parser.error("got wrong version for java %s, less than base version %s:\n%s" % (actual_version, BASE_JAVA_VERSION, s))
+ elif int(actual_version) < int(BASE_JAVA_VERSION):
+ parser.error("got wrong version for java %s, less than base version %s:\n%s" % (actual_version, BASE_JAVA_VERSION, s))
- def run_java(cmd, logfile):
+ def run_java(cmd: str, logfile: str):
run("%s; %s" % (cmd_prefix, cmd), logfile)
return run_java, actual_version
@@ -969,8 +981,8 @@ def make_java_config(parser, alt_java_homes):
if java_home is None:
parser.error("JAVA_HOME must be set")
run_java, _ = _make_runner(java_home, True)
- run_alt_javas = []
- alt_java_versions = []
+ run_alt_javas: list[Callable[[str, str], None]] = []
+ alt_java_versions: list[str] = []
if alt_java_homes:
for alt_java_home in alt_java_homes:
run_alt_java, version = _make_runner(alt_java_home)
@@ -1044,7 +1056,7 @@ reVersion2 = re.compile(r"-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.", re.IGNORECASE)
def getAllLuceneReleases():
s = load("https://archive.apache.org/dist/lucene/java")
- releases = set()
+ releases: set[tuple[int, ...]] = set()
for r in reVersion1, reVersion2:
for tup in r.findall(s):
if tup[-1].lower() == "-alpha":
@@ -1062,14 +1074,14 @@ def getAllLuceneReleases():
return releaseList
-def confirmAllReleasesAreTestedForBackCompat(smokeVersion, unpackPath):
+def confirmAllReleasesAreTestedForBackCompat(smokeVersion: str, unpackPath: str):
print(" find all past Lucene releases...")
allReleases = getAllLuceneReleases()
# for tup in allReleases:
# print(' %s' % '.'.join(str(x) for x in tup))
testedIndicesPaths = glob.glob("%s/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/*-cfs.zip" % unpackPath)
- testedIndices = set()
+ testedIndices: set[tuple[int, ...]] = set()
reIndexName = re.compile(r"^[^.]*.(.*?)-cfs.zip")
for name in testedIndicesPaths:
@@ -1112,7 +1124,7 @@ def confirmAllReleasesAreTestedForBackCompat(smokeVersion, unpackPath):
if x != (1, 9, 0):
raise RuntimeError("tested version=%s but it was not released?" % ".".join(str(y) for y in x))
- notTested = []
+ notTested: list[tuple[int, ...]] = []
for x in allReleases:
if x not in testedIndices:
releaseVersion = ".".join(str(y) for y in x)
@@ -1131,8 +1143,7 @@ def confirmAllReleasesAreTestedForBackCompat(smokeVersion, unpackPath):
for x in notTested:
print(" %s" % ".".join(str(y) for y in x))
raise RuntimeError("some releases are not tested by TestBackwardsCompatibility?")
- else:
- print(" success!")
+ print(" success!")
def main():
@@ -1149,7 +1160,7 @@ def main():
smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, " ".join(c.test_args), downloadOnly=c.download_only)
-def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs, downloadOnly=False):
+def smokeTest(java: Any, baseURL: str, gitRevision: str, version: str, tmpDir: str, isSigned: bool, local_keys: str | None, testArgs: str, downloadOnly: bool = False):
startTime = datetime.datetime.now()
# Tests annotated @Nightly are more resource-intensive but often cover