You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2026/10/20 19:08:49 UTC
(allura) branch master updated: [#8625] Implemented full field level encryption for audit log messages
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
The following commit(s) were added to refs/heads/master by this push:
new d618571c8 [#8625] Implemented full field level encryption for audit log messages
d618571c8 is described below
commit d618571c8089b2b54963ee838f7d351da6c5974b
Author: Carlos Cruz <ca...@slashdotmedia.com>
AuthorDate: Mon Sep 21 19:37:50 2026 +0000
[#8625] Implemented full field level encryption for audit log messages
---
Allura/allura/lib/plugin.py | 36 --------------
Allura/allura/model/auth.py | 19 +-------
.../scripts/backfill_previous_login_details.py | 57 ----------------------
Allura/allura/tests/decorators.py | 16 +++---
.../allura/tests/functional/test_neighborhood.py | 2 +-
Allura/allura/tests/model/test_auth.py | 39 +++------------
Allura/allura/tests/test_plugin.py | 29 -----------
.../tests/functional/test_forum_admin.py | 3 +-
.../forgegit/tests/functional/test_controllers.py | 6 +--
9 files changed, 23 insertions(+), 184 deletions(-)
diff --git a/Allura/allura/lib/plugin.py b/Allura/allura/lib/plugin.py
index 2c835922a..0eb3c0aba 100644
--- a/Allura/allura/lib/plugin.py
+++ b/Allura/allura/lib/plugin.py
@@ -530,42 +530,6 @@ def details_links(self, user):
def hibp_password_check_enabled(self):
return asbool(tg.config.get('auth.hibp_password_check', False))
- @property
- def trusted_auditlog_line_prefixes(self):
- return [
- "Successful login", # this is the main one
- # all others are to include login activity before mid-2017 when "Successful login" logs were introduced:
- "Primary email changed",
- "New email address:",
- "Display Name changed",
- "Email address verified:",
- "Password changed",
- "Email address deleted:",
- "Account activated",
- "Phone verification succeeded.",
- "Visited multifactor new TOTP page",
- "Set up multifactor TOTP",
- "Viewed multifactor TOTP config page",
- "Viewed multifactor recovery codes",
- "Regenerated multifactor recovery codes",
- ]
-
- def login_details_from_auditlog(self, auditlog):
- from allura import model as M
- ip = ua = None
- matches = re.search(r'^IP Address: (.+)\n', auditlog.message, re.MULTILINE)
- if matches:
- ip = matches.group(1)
- matches = re.search(r'^User-Agent: (.+)\n', auditlog.message, re.MULTILINE)
- if matches:
- ua = matches.group(1)
- if ua or ip:
- return M.UserLoginDetails(
- user_id=auditlog.user_id,
- ip=ip,
- ua=ua,
- )
-
def get_login_detail(self, request, user):
from allura import model as M
return M.UserLoginDetails(
diff --git a/Allura/allura/model/auth.py b/Allura/allura/model/auth.py
index 1c866c113..5b1bedd76 100644
--- a/Allura/allura/model/auth.py
+++ b/Allura/allura/model/auth.py
@@ -457,20 +457,6 @@ def add_login_detail(self, detail):
except DuplicateKeyError:
session(detail).expunge(detail)
- def backfill_login_details(self, auth_provider):
- # ".*" at start of regex and the DOTALL flag is needed only for the test, which uses mim
- # Fixed in ming f9f69d3c, so once we upgrade to 0.6.1+ we can remove it
- msg_regex = re.compile(r'.*^({})'.format('|'.join([re.escape(line_prefix)
- for line_prefix
- in auth_provider.trusted_auditlog_line_prefixes])),
- re.MULTILINE | re.DOTALL)
- for auditlog in AuditLog.for_user(self, message=msg_regex):
- if not msg_regex.search(auditlog.message):
- continue
- login_detail = auth_provider.login_details_from_auditlog(auditlog)
- if login_detail:
- self.add_login_detail(login_detail)
-
def send_password_reset_email(self, email_address=None, subject_tmpl='{site_name} Password recovery'):
if email_address is None:
email_address = self.get_pref('email_address')
@@ -1126,12 +1112,11 @@ def users_with_role(self, project=None):
class AuditLogMapperExtension(MapperExtension):
- """Dual-write encrypted messages while plaintext remains the source of truth."""
+ """Classify untyped audit messages on inserts and updates."""
def before_insert(self, instance, state, sess):
if instance.event_type is None and instance.project_id is not None:
instance.event_type = instance.unclassified_event_type(instance.message, instance.project_id)
- instance.message_encrypted = instance.encr(instance.message)
def before_update(self, instance, state, sess):
self.before_insert(instance, state, sess)
@@ -1157,7 +1142,7 @@ class __mongometa__:
user = RelationProperty('User')
timestamp = FieldProperty(datetime, if_missing=datetime.utcnow)
url = FieldProperty(str)
- message = FieldProperty(str)
+ message = DecryptedProperty(str, 'message_encrypted')
message_encrypted = FieldProperty(S.Binary, if_missing=None)
event_type = FieldProperty(str, if_missing=None)
diff --git a/Allura/allura/scripts/backfill_previous_login_details.py b/Allura/allura/scripts/backfill_previous_login_details.py
deleted file mode 100644
index f1e702c7f..000000000
--- a/Allura/allura/scripts/backfill_previous_login_details.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import logging
-
-import argparse
-
-from allura.model import main_orm_session, main_explicitflush_orm_session
-
-from allura.scripts import ScriptTask
-from allura import model as M
-from allura.lib.plugin import AuthenticationProvider
-from allura.lib.utils import chunked_find
-
-log = logging.getLogger('allura.scripts.backfill_previous_login_details')
-
-
-class BackfillPreviousLoginDetails(ScriptTask):
-
- @classmethod
- def parser(cls):
- return argparse.ArgumentParser(description="Backfill previous login details from audit logs. "
- "Needed if auth.hibp_failure_force_pwd_change is enabled")
-
- @classmethod
- def execute(cls, options):
- auth_provider = AuthenticationProvider.get(None)
- for i, chunk in enumerate(chunked_find(M.User, {}, pagesize=2)):
- log.info('Backfilling login details for chunk #%s', i)
- for u in chunk:
- try:
- u.backfill_login_details(auth_provider)
- except Exception:
- log.exception('Error backfilling on user %s', u)
-
- main_orm_session.clear() # AuditLog and User objs
- main_explicitflush_orm_session.clear() # UserLoginDetails objs, already flushed individually
-
- log.info('Finished backfilling previous login details')
-
-
-if __name__ == '__main__':
- BackfillPreviousLoginDetails.main()
diff --git a/Allura/allura/tests/decorators.py b/Allura/allura/tests/decorators.py
index 2f6e90ad3..aa9cfb334 100644
--- a/Allura/allura/tests/decorators.py
+++ b/Allura/allura/tests/decorators.py
@@ -150,6 +150,12 @@ def __exit__(self, exc_type, exc_val, exc_t):
allura.config.middleware.make_app = self._make_app
+def matching_audits(pattern, **query):
+ """AuditLog entries matching ``query`` whose decrypted message matches the ``pattern`` regex"""
+ pattern = re.compile(pattern)
+ return [entry for entry in M.AuditLog.query.find(query) if pattern.search(entry.message or '')]
+
+
@contextlib.contextmanager
def audits(*messages, user=False, actor=r'.*', ip_addr=r'.*', user_agent=r'.*', event_type=None):
"""
@@ -168,11 +174,8 @@ def audits(*messages, user=False, actor=r'.*', ip_addr=r'.*', user_agent=r'.*',
preamble = ''
for message in messages:
- query = dict(message=re.compile(preamble + message))
- if event_type is not None:
- query['event_type'] = event_type
- found = M.AuditLog.query.find(query).count()
- if not found:
+ query = {'event_type': event_type} if event_type is not None else {}
+ if not matching_audits(preamble + message, **query):
hints = ''
all = M.AuditLog.query.find().all()
if len(all) < 10:
@@ -198,8 +201,7 @@ def out_audits(*messages, user=False, actor=r'.*', ip_addr=r'.*'):
else:
preamble = ''
for message in messages:
- assert not M.AuditLog.query.find(dict(
- message=re.compile(preamble + message))).count(), 'Found unexpected: "%s"' % message
+ assert not matching_audits(preamble + message), 'Found unexpected: "%s"' % message
# not a decorator but use it with LogCapture() context manager
diff --git a/Allura/allura/tests/functional/test_neighborhood.py b/Allura/allura/tests/functional/test_neighborhood.py
index f635e7c89..bc0bd367f 100644
--- a/Allura/allura/tests/functional/test_neighborhood.py
+++ b/Allura/allura/tests/functional/test_neighborhood.py
@@ -97,7 +97,7 @@ def test_admin(self):
def test_admin_overview_audit_log(self):
def check_log(message):
- return M.AuditLog.query.find({'message': message}).count() == 1
+ return M.AuditLog.query.find({'message_encrypted': M.AuditLog.encr(message)}).count() == 1
nbhd = M.Neighborhood.query.get(name='Projects')
nbhd.features['css'] = 'custom'
diff --git a/Allura/allura/tests/model/test_auth.py b/Allura/allura/tests/model/test_auth.py
index 7b5d08a14..e87076249 100644
--- a/Allura/allura/tests/model/test_auth.py
+++ b/Allura/allura/tests/model/test_auth.py
@@ -24,7 +24,7 @@
import pytest
from bson import Binary, ObjectId
-from tg import tmpl_context as c, app_globals as g, request as r
+from tg import tmpl_context as c, app_globals as g
from webob import Request
from mock import patch, Mock
@@ -703,31 +703,6 @@ def test_user_index_none_values(self):
assert idx['telnumbers_t'] == ''
assert idx['webpages_t'] == ''
- def test_user_backfill_login_details(self):
- with h.push_config(r, user_agent='TestBrowser/55'):
- # these shouldn't match
- h.auditlog_user('something happened')
- h.auditlog_user('blah blah Password changed')
- with h.push_config(r, user_agent='TestBrowser/56'):
- # these should all match, but only one entry created for this ip/ua
- h.auditlog_user('Account activated')
- h.auditlog_user('Successful login')
- h.auditlog_user('Password changed')
- with h.push_config(r, user_agent='TestBrowser/57'):
- # this should match too
- h.auditlog_user('Set up multifactor TOTP')
- ThreadLocalODMSession.flush_all()
-
- auth_provider = plugin.AuthenticationProvider.get(None)
- c.user.backfill_login_details(auth_provider)
-
- details = M.UserLoginDetails.query.find({'user_id': c.user._id}).sort('ua').all()
- assert len(details) == 2, details
- assert details[0].ip == '127.0.0.1'
- assert details[0].ua == 'TestBrowser/56'
- assert details[1].ip == '127.0.0.1'
- assert details[1].ua == 'TestBrowser/57'
-
class TestAuditLog:
@@ -736,13 +711,14 @@ def setup_class(cls):
setup_basic_test()
setup_global_objects()
- def test_message_dual_write(self):
+ def test_message_encrypted_storage(self):
entry = M.AuditLog(message='Account disabled', event_type='account.disabled')
for message in ['Account disabled', 'Account disabled by admin']:
entry.message = message
session(entry).flush(entry)
raw = M.main_doc_session.db.audit_log.find_one({'_id': entry._id})
- assert raw['message'] == message
+ assert 'message' not in raw
+ assert entry.message == message
assert isinstance(raw['message_encrypted'], Binary)
assert M.AuditLog.decr(raw['message_encrypted']) == message
assert raw['event_type'] == 'account.disabled'
@@ -760,17 +736,16 @@ def test_log_event_type(self, message, arg, project_scoped, event_type, expected
session(entry).flush(entry)
assert M.main_doc_session.db.audit_log.find_one({'_id': entry._id})['event_type'] == expected
- def test_legacy_message_read_and_update(self):
+ def test_encrypted_message_read_and_update(self):
entry_id = ObjectId()
M.main_doc_session.db.audit_log.insert_one({'_id': entry_id, 'project_id': ObjectId(),
- 'message': 'add user alice to Developer'})
+ 'message_encrypted': M.AuditLog.encr('add user alice to Developer')})
entry = M.AuditLog.query.get(_id=entry_id)
assert entry.message == 'add user alice to Developer'
entry.url = '/updated'
session(entry).flush(entry)
raw = M.main_doc_session.db.audit_log.find_one({'_id': entry_id})
- assert raw['message'] == 'add user alice to Developer'
- assert M.AuditLog.decr(raw['message_encrypted']) == raw['message']
+ assert M.AuditLog.decr(raw['message_encrypted']) == 'add user alice to Developer'
assert raw['event_type'] == 'project.user_mention.unclassified'
def test_message_html(self):
diff --git a/Allura/allura/tests/test_plugin.py b/Allura/allura/tests/test_plugin.py
index b1ed24b59..8f14fd3c5 100644
--- a/Allura/allura/tests/test_plugin.py
+++ b/Allura/allura/tests/test_plugin.py
@@ -784,35 +784,6 @@ def test_disable_user(self):
ThreadLocalODMSession.flush_all()
assert user.disabled is True
- def test_login_details_from_auditlog(self):
- user = M.User(username='asfdasdf')
-
- assert (self.provider.login_details_from_auditlog(M.AuditLog(message='')) is
- None)
-
- detail = self.provider.login_details_from_auditlog(M.AuditLog(message='IP Address: 1.2.3.4\nFoo', user=user))
- assert detail.user_id == user._id
- assert detail.ip == '1.2.3.4'
- assert detail.ua is None
-
- detail = self.provider.login_details_from_auditlog(M.AuditLog(message='Foo\nIP Address: 1.2.3.4\nFoo', user=user))
- assert detail.ip == '1.2.3.4'
- assert detail.ua is None
-
- assert (self.provider.login_details_from_auditlog(M.AuditLog(
- message='blah blah IP Address: 1.2.3.4\nFoo', user=user)) is
- None)
-
- detail = self.provider.login_details_from_auditlog(M.AuditLog(
- message='User-Agent: Mozilla/Firefox\nFoo', user=user))
- assert detail.ip is None
- assert detail.ua == 'Mozilla/Firefox'
-
- detail = self.provider.login_details_from_auditlog(M.AuditLog(
- message='IP Address: 1.2.3.4\nUser-Agent: Mozilla/Firefox\nFoo', user=user))
- assert detail.ip == '1.2.3.4'
- assert detail.ua == 'Mozilla/Firefox'
-
def test_get_login_detail(self):
user = M.User(username='foobarbaz')
detail = self.provider.get_login_detail(Request.blank('/'), user)
diff --git a/ForgeDiscussion/forgediscussion/tests/functional/test_forum_admin.py b/ForgeDiscussion/forgediscussion/tests/functional/test_forum_admin.py
index 459f2c91a..e00f78f10 100644
--- a/ForgeDiscussion/forgediscussion/tests/functional/test_forum_admin.py
+++ b/ForgeDiscussion/forgediscussion/tests/functional/test_forum_admin.py
@@ -17,7 +17,6 @@
import logging
-import re
from ming.odm import ThreadLocalODMSession
from tg import tmpl_context as c
from alluratest.controller import TestController
@@ -55,7 +54,7 @@ def test_forum_CRUD(self):
'forum-0.shortname': 'NewTestForum',
'forum-0.description': 'My desc',
'forum-0.monitoring_email': ''})
- audit_logs = M.AuditLog.query.find({'project_id': project._id, 'message': re.compile(' set option ')}).all()
+ audit_logs = td.matching_audits(' set option ', project_id=project._id)
assert len(audit_logs) == 4
r = self.app.get('/admin/discussion/forums')
assert 'New Test Forum' in r
diff --git a/ForgeGit/forgegit/tests/functional/test_controllers.py b/ForgeGit/forgegit/tests/functional/test_controllers.py
index 10d5c62e0..777454f39 100644
--- a/ForgeGit/forgegit/tests/functional/test_controllers.py
+++ b/ForgeGit/forgegit/tests/functional/test_controllers.py
@@ -601,12 +601,12 @@ def test_force_push_toggle(self):
self.app.post('/p/test/admin/src-git/force_push', params={'allowed': 'on'})
assert GM.Repository.query.get(_id=c.app.repo._id).force_push_allowed is True
assert M.AuditLog.query.find(
- {'message': 'src-git: set "force_push_allowed" False => True'}).count() == 1
+ {'message_encrypted': M.AuditLog.encr('src-git: set "force_push_allowed" False => True')}).count() == 1
self.app.post('/p/test/admin/src-git/force_push', params={})
assert GM.Repository.query.get(_id=c.app.repo._id).force_push_allowed is False
assert M.AuditLog.query.find(
- {'message': 'src-git: set "force_push_allowed" True => False'}).count() == 1
+ {'message_encrypted': M.AuditLog.encr('src-git: set "force_push_allowed" True => False')}).count() == 1
def test_force_push_menu_entry(self):
with h.push_config(tg.config, **{'scm.force_push.git.enabled': 'true'}):
@@ -629,7 +629,7 @@ def test_force_push_save_writes_even_when_unchanged(self):
assert GM.Repository.query.get(_id=c.app.repo._id).force_push_allowed is False
post.assert_called_once_with(c.app.repo.full_fs_path, False)
assert M.AuditLog.query.find(
- {'message': 'src-git: set "force_push_allowed" False => False'}).count() == 0
+ {'message_encrypted': M.AuditLog.encr('src-git: set "force_push_allowed" False => False')}).count() == 0
def test_force_push_hidden_until_repo_ready(self):
with h.push_config(tg.config, **{'scm.force_push.git.enabled': 'true'}):