You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/07/23 15:50:35 UTC
(tooling-trusted-releases) 01/01: Preserve and act upon LDAP connection errors
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 590d6e8d56c665ceb29d2a563ccdf703b54ee119
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Thu Jul 23 16:35:27 2026 +0100
Preserve and act upon LDAP connection errors
---
atr/ldap.py | 56 ++++++++++++++++++++------
atr/principal.py | 6 +++
atr/server.py | 24 +-----------
atr/sessions.py | 30 ++++++++++++++
atr/worker.py | 31 ++++++++++++++-
tests/unit/test_ldap.py | 87 +++++++++++++++++++++++++++++++++++++++++
tests/unit/test_sessions.py | 56 ++++++++++++++++++++++++++
tests/unit/test_worker_defer.py | 57 +++++++++++++++++++++++++++
8 files changed, 310 insertions(+), 37 deletions(-)
diff --git a/atr/ldap.py b/atr/ldap.py
index 81f89871..1edae78b 100644
--- a/atr/ldap.py
+++ b/atr/ldap.py
@@ -22,6 +22,8 @@ import ssl
from typing import Final, Literal
import ldap3
+import ldap3.core.exceptions
+import ldap3.core.results
import ldap3.utils.conv as conv
import ldap3.utils.dn as dn
@@ -43,6 +45,11 @@ RESULT_ATTRIBUTES: Final[list[str]] = [
"uid",
]
+SEARCH_OK_RESULT_CODES: Final[tuple[int, int]] = (
+ ldap3.core.results.RESULT_SUCCESS,
+ ldap3.core.results.RESULT_NO_SUCH_OBJECT,
+)
+
class PubSubAttributes(schema.Subset):
"""LDAP attributes as they appear in pubsub old_attributes/new_attributes."""
@@ -92,13 +99,16 @@ class Search:
def __enter__(self):
server = ldap3.Server(LDAP_SERVER_HOST, use_ssl=True, tls=_tls_config)
- self._conn = ldap3.Connection(
- server,
- user=self._bind_dn,
- password=self._bind_password,
- auto_bind=True,
- check_names=False,
- )
+ try:
+ self._conn = ldap3.Connection(
+ server,
+ user=self._bind_dn,
+ password=self._bind_password,
+ auto_bind=True,
+ check_names=False,
+ )
+ except ldap3.core.exceptions.LDAPException as e:
+ raise UnavailableError(str(e)) from e
return self
def __exit__(self, exc_type, exc_val, exc_tb):
@@ -116,12 +126,17 @@ class Search:
raise RuntimeError("LDAP connection not available")
attributes = ldap_attrs if ldap_attrs else RESULT_ATTRIBUTES
- self._conn.search(
- search_base=ldap_base,
- search_filter=ldap_query,
- search_scope=ldap_scope,
- attributes=attributes,
- )
+ try:
+ self._conn.search(
+ search_base=ldap_base,
+ search_filter=ldap_query,
+ search_scope=ldap_scope,
+ attributes=attributes,
+ )
+ except ldap3.core.exceptions.LDAPException as e:
+ raise UnavailableError(str(e)) from e
+ if self._conn.result["result"] not in SEARCH_OK_RESULT_CODES:
+ raise UnavailableError(f"LDAP search failed: {self._conn.result}")
results: list[Result] = []
for entry in self._conn.entries:
results.append(Result.model_validate({"dn": entry.entry_dn, **entry.entry_attributes_as_dict}))
@@ -132,6 +147,10 @@ class LookupError(Exception):
pass
+class UnavailableError(Exception):
+ pass
+
+
# We use a dataclass to support ldap3.Connection objects
@dataclasses.dataclass
class SearchParameters:
@@ -145,6 +164,7 @@ class SearchParameters:
err_msg: str | None = None
srv_info: str | None = None
detail_err: str | None = None
+ failed: bool = False
connection: ldap3.Connection | None = None
@@ -167,6 +187,8 @@ async def account_lookup(asf_uid: str) -> Result | None:
)
await asyncio.to_thread(search, params)
+ if params.failed:
+ raise UnavailableError(params.err_msg or "LDAP search failed")
if not params.results_list:
return None
@@ -225,6 +247,8 @@ async def github_to_apache(github_numeric_uid: int) -> str:
github_nid_query=github_numeric_uid,
)
await asyncio.to_thread(search, ldap_params)
+ if ldap_params.failed:
+ raise UnavailableError(ldap_params.err_msg or "LDAP search failed")
if not (ldap_params.results_list and ldap_params.results_list[0].uid):
raise LookupError(f"GitHub NID {github_numeric_uid} not registered with the ATR")
return ldap_params.results_list[0].uid[0]
@@ -304,6 +328,7 @@ def search(params: SearchParameters) -> None:
try:
_search_core(params)
except Exception as e:
+ params.failed = True
params.err_msg = f"An unexpected error occurred: {e!s}"
params.detail_err = f"Details: {e.args}"
finally:
@@ -386,6 +411,7 @@ def _search_core(params: SearchParameters) -> None:
params.err_msg = None
params.srv_info = None
params.detail_err = None
+ params.failed = False
params.connection = None
server = ldap3.Server(LDAP_SERVER_HOST, use_ssl=True, tls=_tls_config, get_info=ldap3.ALL)
@@ -441,6 +467,10 @@ def _search_core_2(params: SearchParameters, filters: list[str]) -> None:
search_filter=search_filter,
attributes=RESULT_ATTRIBUTES,
)
+ if params.connection.result["result"] != ldap3.core.results.RESULT_SUCCESS:
+ params.failed = True
+ params.err_msg = f"LDAP search failed: {params.connection.result}"
+ return
for entry in params.connection.entries:
params.results_list.append(Result.model_validate({"dn": entry.entry_dn, **entry.entry_attributes_as_dict}))
diff --git a/atr/principal.py b/atr/principal.py
index 8074f234..cf259591 100644
--- a/atr/principal.py
+++ b/atr/principal.py
@@ -123,6 +123,8 @@ class Committer:
raise CommitterError("Authentication failed")
except CommitterError:
raise
+ except ldap.UnavailableError:
+ raise
except Exception as ex:
log.exception(f"An unknown error occurred while fetching user details: {ex!s}")
raise CommitterError("An unknown error occurred while fetching user details.") from ex
@@ -151,6 +153,8 @@ class Committer:
raise CommitterError("Common backend assertions failed, LDAP corruption?")
except CommitterError:
raise
+ except ldap.UnavailableError:
+ raise
except Exception as ex:
log.exception(f"An unknown error occurred while fetching group memberships from {ldap_base}: {ex!s}")
raise CommitterError(
@@ -170,6 +174,8 @@ class Committer:
ldap_query=ldap_filter % (self.user,),
ldap_attrs=["cn"],
)
+ except ldap.UnavailableError:
+ raise
except Exception as ex:
log.exception(f"An unknown error occurred while fetching project memberships: {ex!s}")
raise CommitterError("An unknown error occurred while fetching project memberships.") from ex
diff --git a/atr/server.py b/atr/server.py
index 4058a213..93cfd869 100644
--- a/atr/server.py
+++ b/atr/server.py
@@ -59,7 +59,6 @@ import atr.errors as errors
import atr.filters as filters
import atr.form as form
import atr.jwtoken as jwtoken
-import atr.ldap as ldap
import atr.log as log
import atr.manager as manager
import atr.models.sql as sql
@@ -539,30 +538,9 @@ def _app_setup_request_lifecycle(app: base.QuartApp) -> None:
# Check if session has a check timestamp
last_check = session.last_account_check
current_time = time.time()
- uid = session.uid
if last_check is None or (current_time - last_check > account_check_interval):
- admin_uid = session.admin_uid
-
- if isinstance(admin_uid, str) and bool(admin_uid):
- user_active, admin_active = await asyncio.gather(
- ldap.is_active(uid),
- ldap.is_active(admin_uid),
- )
- if not admin_active:
- await sessions.deleted_or_banned(admin_uid)
- raise base.ASFQuartException("Account is disabled", errorcode=401)
- else:
- user_active = await ldap.is_active(uid)
- if not user_active:
- await sessions.deleted_or_banned(uid)
- raise base.ASFQuartException("Account is disabled", errorcode=401)
-
- session.last_account_check = current_time
- await asfquart.APP.sessions.save(session, {"last_account_check"})
-
- if last_check is None:
- log.auth_success("oauth", uid)
+ await sessions.account_check(session, current_time)
@app.after_request
async def log_request(response: quart.Response) -> quart.Response:
diff --git a/atr/sessions.py b/atr/sessions.py
index 459580d9..05987bd1 100644
--- a/atr/sessions.py
+++ b/atr/sessions.py
@@ -15,9 +15,11 @@
# specific language governing permissions and limitations
# under the License.
+import asyncio
import time
from typing import Any, Final, Literal, get_args
+import asfquart.base as base
import asfquart.session
import quart
import sqlalchemy.dialects.sqlite
@@ -25,6 +27,7 @@ import sqlmodel
import atr.config as config
import atr.db as db
+import atr.ldap as ldap
import atr.log as log
import atr.models.sql as sql
@@ -40,6 +43,24 @@ if not _MUTABLE_SESSION_FIELDS.issubset(sql.UserSession.model_fields):
)
+async def account_check(session: sql.UserSession, current_time: float) -> None:
+ uid = session.uid
+ admin_uid = session.admin_uid
+ if isinstance(admin_uid, str) and bool(admin_uid):
+ user_result, admin_result = await asyncio.gather(_account_active(uid), _account_active(admin_uid))
+ if admin_result is False:
+ await deleted_or_banned(admin_uid)
+ raise base.ASFQuartException("Account is disabled", errorcode=401)
+ else:
+ user_result = await _account_active(uid)
+ if user_result is False:
+ await deleted_or_banned(uid)
+ raise base.ASFQuartException("Account is disabled", errorcode=401)
+
+ session.last_account_check = current_time
+ await asfquart.APP.sessions.save(session, {"last_account_check"})
+
+
async def deleted_or_banned(uid: str) -> None:
log.auth_failure("oauth", "account_deleted_or_banned", uid)
await asfquart.APP.sessions.revoke_by_uid(uid)
@@ -104,6 +125,14 @@ async def terminate_current_users_sessions(uid: str) -> int:
return count
+async def _account_active(uid: str) -> bool | ldap.UnavailableError:
+ try:
+ return await ldap.is_active(uid)
+ except ldap.UnavailableError as e:
+ log.warning(f"Skipping the account check for {uid} because LDAP is unavailable: {e}")
+ return e
+
+
async def _form_error_lookup(data: db.Session, sid_hash: str, path: str) -> sql.SessionFormError | None:
statement = sqlmodel.select(sql.SessionFormError).where(
sql.SessionFormError.sid_hash == sid_hash,
@@ -150,6 +179,7 @@ class Store:
async with db.session() as data:
data.add(user_session)
await data.commit()
+ log.auth_success("oauth", user_session.uid)
async def destroy(self, hsid: str) -> None:
async with db.session() as data:
diff --git a/atr/worker.py b/atr/worker.py
index 4057cee0..ea4d497d 100644
--- a/atr/worker.py
+++ b/atr/worker.py
@@ -52,6 +52,8 @@ import atr.tasks.task as task
_CPU_LIMIT_SECONDS: Final = 300
_MEMORY_LIMIT_BYTES: Final = 3 * 1024 * 1024 * 1024
+_DEFER_SECONDS: Final = 120
+
# Successful recurring tasks are appended here as JSON lines instead of kept in the database
_TASK_LOG_LOGGER: Final = "atr.tasks.log"
@@ -227,6 +229,28 @@ def _task_completed_record(
}
+async def _task_defer(task_id: int) -> None:
+ via = sql.validate_instrumented_attribute
+ scheduled = datetime.datetime.now(datetime.UTC) + datetime.timedelta(seconds=_DEFER_SECONDS)
+ async with db.session() as data:
+ async with data.begin():
+ update_stmt = (
+ sqlmodel.update(sql.Task)
+ .where(
+ sqlmodel.and_(
+ via(sql.Task.id) == task_id,
+ sql.Task.status == task.ACTIVE,
+ via(sql.Task.pid) == os.getpid(),
+ )
+ )
+ .values(status=task.QUEUED, started=None, pid=None, scheduled=scheduled)
+ .returning(via(sql.Task.id))
+ )
+ result = await data.execute(update_stmt)
+ if result.first() is None:
+ log.warning(f"Task {task_id} was not deferred because it is no longer active")
+
+
def _task_failure_message(
task_type: sql.TaskType,
project_key: str | None,
@@ -321,7 +345,12 @@ async def _task_process(task_id: int, task_type: str, task_args: list[str] | dic
and not (config.is_test_mode() and asf_uid == "test")
and (config.is_production_mode() or config.is_ldap_configured())
):
- user_account = await ldap.account_lookup(asf_uid)
+ try:
+ user_account = await ldap.account_lookup(asf_uid)
+ except ldap.UnavailableError as e:
+ log.warning(f"Deferring task {task_id} ({task_type}) because LDAP is unavailable: {e}")
+ await _task_defer(task_id)
+ return
# We check here to see if the account is banned - in the case of running tasks,
# we don't really need to worry about admin/membership status as that wouldn't
# materially affect outstanding worker tasks and is rare anyway.
diff --git a/tests/unit/test_ldap.py b/tests/unit/test_ldap.py
index b4994d1e..73fa55fd 100644
--- a/tests/unit/test_ldap.py
+++ b/tests/unit/test_ldap.py
@@ -19,6 +19,7 @@ import pathlib
import unittest.mock as mock
from typing import TYPE_CHECKING
+import ldap3.core.exceptions
import pytest
import atr.cache as cache
@@ -45,11 +46,37 @@ class MockConfig:
self.LDAP_BIND_PASSWORD = ldap_bind_password
+class MockRaisingConnection:
+ def search(self, **kwargs: object) -> bool:
+ raise ldap3.core.exceptions.LDAPSocketReceiveError("connection reset")
+
+
+class MockResultConnection:
+ def __init__(self, result_code: int):
+ self.result = {"result": result_code}
+ self.entries: list[object] = []
+
+ def search(self, search_base: str, search_filter: str, attributes: list[str]) -> bool:
+ return self.result["result"] == 0
+
+
@pytest.fixture
def ldap_configured() -> bool:
return ldap.get_bind_credentials() is not None
+@pytest.mark.asyncio
+async def test_account_lookup_raises_when_ldap_unavailable(monkeypatch: "MonkeyPatch"):
+ def failed_search(params: ldap.SearchParameters) -> None:
+ params.failed = True
+ params.err_msg = "An unexpected error occurred: connection refused"
+
+ monkeypatch.setattr("atr.ldap.get_bind_credentials", lambda: ("dn", "pw"))
+ monkeypatch.setattr("atr.ldap.search", failed_search)
+ with pytest.raises(ldap.UnavailableError):
+ await ldap.account_lookup("alice")
+
+
@pytest.mark.asyncio
async def test_admins_startup_load_fetches_real_admins(
ldap_configured: bool, tmp_path: pathlib.Path, monkeypatch: "MonkeyPatch"
@@ -144,6 +171,17 @@ async def test_fetch_admin_users_returns_reasonable_count(ldap_configured: bool)
assert len(admins) < 100
+@pytest.mark.asyncio
+async def test_github_to_apache_raises_when_ldap_unavailable(monkeypatch: "MonkeyPatch"):
+ def failed_search(params: ldap.SearchParameters) -> None:
+ params.failed = True
+ params.err_msg = "An unexpected error occurred: connection refused"
+
+ monkeypatch.setattr("atr.ldap.search", failed_search)
+ with pytest.raises(ldap.UnavailableError):
+ await ldap.github_to_apache(12345)
+
+
@pytest.mark.asyncio
async def test_handle_update_handles_invalid_payload(monkeypatch: "MonkeyPatch"):
warnings: list[str] = []
@@ -223,6 +261,15 @@ async def test_handle_update_skips_revoke_logs_when_nothing_to_revoke(monkeypatc
assert "ssh_keys_revoked" not in auth_events
+@pytest.mark.asyncio
+async def test_is_active_raises_when_ldap_unavailable(monkeypatch: "MonkeyPatch"):
+ monkeypatch.setattr("atr.ldap.get_bind_credentials", lambda: ("dn", "pw"))
+ monkeypatch.setattr("atr.config.is_test_mode", lambda: True)
+ monkeypatch.setattr("atr.ldap.account_lookup", mock.AsyncMock(side_effect=ldap.UnavailableError("down")))
+ with pytest.raises(ldap.UnavailableError):
+ await ldap.is_active("alice")
+
+
@pytest.mark.asyncio
async def test_is_active_returns_false_for_banned_account(monkeypatch: "MonkeyPatch"):
account = ldap.Result.model_validate(
@@ -321,6 +368,46 @@ def test_pubsub_payload_parses_valid_event():
assert parsed.old_attributes.asf_banned == []
+def test_search_class_wraps_transport_errors():
+ ldap_search = ldap.Search("dn", "pw")
+ ldap_search._conn = MockRaisingConnection()
+ with pytest.raises(ldap.UnavailableError):
+ ldap_search.search(ldap_base="ou=people,dc=apache,dc=org", ldap_scope="BASE")
+
+
+def test_search_core_2_reports_no_results_on_empty_success():
+ params = ldap.SearchParameters(uid_query="alice")
+ params.connection = MockResultConnection(0)
+ ldap._search_core_2(params, ["(uid=alice)"])
+ assert params.failed is False
+ assert params.err_msg == "No results found for the given criteria."
+
+
+def test_search_core_2_sets_failed_on_error_result():
+ params = ldap.SearchParameters(uid_query="alice")
+ params.connection = MockResultConnection(52)
+ ldap._search_core_2(params, ["(uid=alice)"])
+ assert params.failed is True
+
+
+def test_search_core_2_sets_failed_on_no_such_object():
+ params = ldap.SearchParameters(uid_query="alice")
+ params.connection = MockResultConnection(32)
+ ldap._search_core_2(params, ["(uid=alice)"])
+ assert params.failed is True
+
+
+def test_search_sets_failed_on_exception(monkeypatch: "MonkeyPatch"):
+ def broken_search_core(params: ldap.SearchParameters) -> None:
+ raise ConnectionRefusedError("connection refused")
+
+ monkeypatch.setattr("atr.ldap._search_core", broken_search_core)
+ params = ldap.SearchParameters(uid_query="alice")
+ ldap.search(params)
+ assert params.failed is True
+ assert params.err_msg is not None
+
+
def _make_pubsub_payload(
uid: str = "testuser",
old_banned: list[str] | None = None,
diff --git a/tests/unit/test_sessions.py b/tests/unit/test_sessions.py
index f09db116..59d8c065 100644
--- a/tests/unit/test_sessions.py
+++ b/tests/unit/test_sessions.py
@@ -18,11 +18,13 @@
import types
import unittest.mock as mock
+import asfquart.base as base
import pytest
import sqlalchemy.ext.asyncio
import sqlmodel
import atr.db as db
+import atr.ldap as ldap
import atr.models.sql as sql
import atr.sessions as sessions
@@ -56,6 +58,60 @@ async def store(monkeypatch):
db._global_atr_sessionmaker = None
+async def test_account_check_fails_open_when_ldap_unavailable(monkeypatch):
+ user_session = sql.UserSession(uid="alice")
+ monkeypatch.setattr("atr.ldap.is_active", mock.AsyncMock(side_effect=ldap.UnavailableError("down")))
+ deleted = mock.AsyncMock()
+ monkeypatch.setattr("atr.sessions.deleted_or_banned", deleted)
+ save = mock.AsyncMock()
+ monkeypatch.setattr("asfquart.APP", types.SimpleNamespace(sessions=types.SimpleNamespace(save=save)))
+
+ await sessions.account_check(user_session, 123.0)
+
+ deleted.assert_not_awaited()
+ save.assert_awaited_once()
+ assert user_session.last_account_check == 123.0
+
+
+async def test_account_check_revokes_user_when_admin_check_unavailable(monkeypatch):
+ user_session = sql.UserSession(uid="alice", admin_uid="admin")
+
+ async def is_active(uid: str) -> bool:
+ if uid == "admin":
+ raise ldap.UnavailableError("down")
+ return False
+
+ monkeypatch.setattr("atr.ldap.is_active", is_active)
+ deleted = mock.AsyncMock()
+ monkeypatch.setattr("atr.sessions.deleted_or_banned", deleted)
+
+ with pytest.raises(base.ASFQuartException):
+ await sessions.account_check(user_session, 123.0)
+
+ deleted.assert_awaited_once_with("alice")
+
+
+async def test_account_check_saves_timestamp_when_active(monkeypatch):
+ user_session = sql.UserSession(uid="alice")
+ monkeypatch.setattr("atr.ldap.is_active", mock.AsyncMock(return_value=True))
+ save = mock.AsyncMock()
+ monkeypatch.setattr("asfquart.APP", types.SimpleNamespace(sessions=types.SimpleNamespace(save=save)))
+
+ await sessions.account_check(user_session, 123.0)
+
+ save.assert_awaited_once()
+ assert user_session.last_account_check == 123.0
+
+
+async def test_create_logs_auth_success(store, monkeypatch):
+ events: list[tuple[str, str | None]] = []
+ monkeypatch.setattr("atr.log.auth_success", lambda type, asfuid=None: events.append((type, asfuid)))
+
+ await store.create(_hsid(), {"uid": "alice"})
+
+ assert events == [("oauth", "alice")]
+
+
async def test_create_maps_pmcs_to_member_committees(store):
hsid = _hsid()
await store.create(hsid, _session_data())
diff --git a/tests/unit/test_worker_defer.py b/tests/unit/test_worker_defer.py
new file mode 100644
index 00000000..44015f5a
--- /dev/null
+++ b/tests/unit/test_worker_defer.py
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest.mock as mock
+
+import pytest
+
+import atr.ldap as ldap
+import atr.models.sql as sql
+import atr.tasks.task as task
+import atr.worker as worker
+
+
+async def test_task_process_defers_when_ldap_unavailable(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("atr.config.is_production_mode", lambda: True)
+ monkeypatch.setattr("atr.ldap.account_lookup", mock.AsyncMock(side_effect=ldap.UnavailableError("down")))
+ defer = mock.AsyncMock()
+ monkeypatch.setattr("atr.worker._task_defer", defer)
+ result_process = mock.AsyncMock()
+ monkeypatch.setattr("atr.worker._task_result_process", result_process)
+
+ await worker._task_process(1, sql.TaskType.MESSAGE_SEND.value, {}, "alice")
+
+ defer.assert_awaited_once_with(1)
+ result_process.assert_not_awaited()
+
+
+async def test_task_process_fails_when_handler_ldap_unavailable(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def handler(args: object) -> None:
+ raise ldap.UnavailableError("down")
+
+ monkeypatch.setattr("atr.config.is_production_mode", lambda: False)
+ monkeypatch.setattr("atr.config.is_ldap_configured", lambda: False)
+ monkeypatch.setattr("atr.tasks.resolve", lambda task_type: handler)
+ defer = mock.AsyncMock()
+ monkeypatch.setattr("atr.worker._task_defer", defer)
+ result_process = mock.AsyncMock()
+ monkeypatch.setattr("atr.worker._task_result_process", result_process)
+
+ await worker._task_process(1, sql.TaskType.MESSAGE_SEND.value, {}, "alice")
+
+ defer.assert_not_awaited()
+ result_process.assert_awaited_once_with(1, None, task.FAILED, "down")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org