You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by cc...@apache.org on 2026/08/14 18:55:55 UTC
(allura) 01/02: [#8614] Implement field level encryption for User and UserLoginDetails fields
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch cc/8614-part2
in repository https://gitbox.apache.org/repos/asf/allura.git
commit a9367adfab788042460f4da93a100d812a180176
Author: Carlos Cruz <ca...@slashdotmedia.com>
AuthorDate: Mon Jul 20 15:17:14 2026 -0600
[#8614] Implement field level encryption for User and UserLoginDetails fields
---
Allura/allura/ext/user_profile/user_main.py | 2 +-
Allura/allura/model/auth.py | 52 +++-------------------
.../allura/tests/functional/test_user_profile.py | 19 ++++++++
Allura/allura/tests/model/test_auth.py | 47 ++++++++++++-------
.../tests/scripts/test_convert_encrypted_field.py | 32 ++++++++++---
scripts/convert_encrypted_field.py | 21 ++++-----
6 files changed, 95 insertions(+), 78 deletions(-)
diff --git a/Allura/allura/ext/user_profile/user_main.py b/Allura/allura/ext/user_profile/user_main.py
index 8011958f5..0c9903a54 100644
--- a/Allura/allura/ext/user_profile/user_main.py
+++ b/Allura/allura/ext/user_profile/user_main.py
@@ -306,7 +306,7 @@ def __json__(self):
joined=auth_provider.user_registration_date(self.user),
localization=self.user.get_pref('localization')._deinstrument(),
sex=self.user.get_pref('sex'),
- telnumbers=self.user.get_pref('telnumbers')._deinstrument(),
+ telnumbers=list(self.user.get_pref('telnumbers')),
webpages=self.user.get_pref('webpages')._deinstrument(),
availability=self.user.get_pref('availability')._deinstrument())
diff --git a/Allura/allura/model/auth.py b/Allura/allura/model/auth.py
index 2e2bcc9bd..4b9c34094 100644
--- a/Allura/allura/model/auth.py
+++ b/Allura/allura/model/auth.py
@@ -39,7 +39,7 @@
from tg import request
from ming import schema as S
from ming.encryption import NestedEncryptedProperty
-from ming.odm import session, state, MapperExtension
+from ming.odm import session, state
from ming.odm import FieldProperty, RelationProperty, ForeignIdProperty, DecryptedProperty, DecryptedListProperty
from ming.odm.declarative import MappedClass
from ming.odm.odmsession import ThreadLocalODMSession
@@ -267,15 +267,6 @@ def get_decrypted(self, instance):
return super().__get__(instance, type(instance))
-class UserFieldEncryptionMapperExtension(MapperExtension):
-
- def before_insert(self, obj, state, sess):
- obj.sync_personal_data_encrypted()
-
- def before_update(self, obj, state, sess):
- obj.sync_personal_data_encrypted()
-
-
class User(MappedClass, ActivityNode, ActivityObject, SearchIndexable):
SALT_LEN = 8
@@ -284,7 +275,6 @@ class __mongometa__:
session = main_orm_session
indexes = ['tool_data.AuthPasswordReset.hash']
unique_indexes = ['username']
- extensions = [UserFieldEncryptionMapperExtension]
custom_indexes = [
dict(fields=('tool_data.phone_verification.number_hash',), sparse=True),
]
@@ -342,12 +332,11 @@ class __mongometa__:
end_date=S.DateTime)])
# Additional contacts
- socialnetworks = FieldProperty([dict(
+ socialnetworks = NestedEncryptedProperty([dict(
socialnetwork=str,
- accounturl=str,
accounturl_encrypted=S.Binary,
)])
- telnumbers = FieldProperty([str])
+ telnumbers = DecryptedListProperty('telnumbers_encrypted')
telnumbers_encrypted = FieldProperty([S.Binary], if_missing=[])
webpages = FieldProperty([str])
@@ -359,30 +348,14 @@ class __mongometa__:
# Statistics
stats_id = FieldProperty(S.ObjectId, if_missing=None)
- last_access = FieldProperty(dict(
+ last_access = NestedEncryptedProperty(dict(
login_date=S.DateTime,
- login_ip=str,
login_ip_encrypted=S.Binary,
login_ua=str,
session_date=S.DateTime,
- session_ip=str,
session_ip_encrypted=S.Binary,
session_ua=str))
- def sync_personal_data_encrypted(self):
- last_access = dict(self.last_access or {})
- for field_name in ('login_ip', 'session_ip'):
- value = last_access.get(field_name)
- last_access[f'{field_name}_encrypted'] = self.encr(value)
- self.last_access = last_access
-
- socialnetworks = [dict(socialnetwork) for socialnetwork in self.socialnetworks or []]
- for socialnetwork in socialnetworks:
- socialnetwork['accounturl_encrypted'] = self.encr(socialnetwork.get('accounturl'))
- self.socialnetworks = socialnetworks
-
- self.telnumbers_encrypted = [self.encr(telnumber) for telnumber in self.telnumbers or []]
-
def __repr__(self):
try:
display_name = self.display_name
@@ -1224,15 +1197,6 @@ def comment_user(cls, by, message, *args, **kwargs):
return cls.log_user(message, *args, **kwargs)
-class UserLoginDetailsEncryptionMapperExtension(MapperExtension):
-
- def before_insert(self, obj, state, sess):
- obj.sync_ip_encrypted()
-
- def before_update(self, obj, state, sess):
- obj.sync_ip_encrypted()
-
-
class UserLoginDetails(MappedClass):
"""
Store unique entries for users' previous login details. And other trusted actions like successful password reset
@@ -1244,15 +1208,14 @@ class __mongometa__:
name = 'user_login_details'
session = main_explicitflush_orm_session
indexes = ['user_id']
- unique_indexes = [('user_id', 'ip', 'ua'), # DuplicateKeyError checked in add_login_detail
+ unique_indexes = [('user_id', 'ip_encrypted', 'ua'), # DuplicateKeyError checked in add_login_detail
]
- extensions = [UserLoginDetailsEncryptionMapperExtension]
query: Query[UserLoginDetails]
_id = FieldProperty(S.ObjectId)
user_id: ObjectId = AlluraUserProperty(required=True)
- ip = FieldProperty(str)
+ ip = DecryptedProperty(str, 'ip_encrypted')
ip_encrypted = FieldProperty(S.Binary, if_missing=None)
ua = FieldProperty(str)
extra = FieldProperty({
@@ -1260,6 +1223,3 @@ class __mongometa__:
})
user = RelationProperty('User')
-
- def sync_ip_encrypted(self):
- self.ip_encrypted = self.encr(self.ip)
diff --git a/Allura/allura/tests/functional/test_user_profile.py b/Allura/allura/tests/functional/test_user_profile.py
index 2e9cc7af3..d4573b8f1 100644
--- a/Allura/allura/tests/functional/test_user_profile.py
+++ b/Allura/allura/tests/functional/test_user_profile.py
@@ -17,6 +17,7 @@
import mock
import tg
+from ming.odm import ThreadLocalODMSession
from alluratest.controller import TestRestApiBase
from allura.model import Project, User
@@ -100,6 +101,24 @@ def test_profile_user_card(self):
assert user.get_pref('localization')['country'] in r.html.find('span', attrs={'class': 'subitem-loc'}).getText()
assert user.get_pref('webpages')[0] in str(r.html.find('span', attrs={'class': 'subitem-web'}))
+ @td.with_user_project('test-admin')
+ def test_profile_rest_returns_decrypted_contacts(self):
+ user = User.by_username('test-admin')
+ user.set_pref('telnumbers', ['+1-555-0100', '+1-555-0101'])
+ user.set_pref('socialnetworks', [{
+ 'socialnetwork': 'Mastodon',
+ 'accounturl': 'https://example.com/@test-admin',
+ }])
+ ThreadLocalODMSession.flush_all()
+
+ response = self.app.get('/rest/u/test-admin/profile/')
+
+ assert response.json['telnumbers'] == ['+1-555-0100', '+1-555-0101']
+ assert response.json['socialnetworks'] == [{
+ 'socialnetwork': 'Mastodon',
+ 'accounturl': 'https://example.com/@test-admin',
+ }]
+
def test_wrong_profile(self):
self.app.get('/u/no-such-user/profile/', status=404)
diff --git a/Allura/allura/tests/model/test_auth.py b/Allura/allura/tests/model/test_auth.py
index 0ecf8298a..d5d2161c7 100644
--- a/Allura/allura/tests/model/test_auth.py
+++ b/Allura/allura/tests/model/test_auth.py
@@ -494,7 +494,7 @@ def test_preference_email_address_updates_encrypted_field(self):
assert raw_preferences['email_address_encrypted'] == M.User.encr('new@example.com')
assert M.User.decr(raw_preferences['email_address_encrypted']) == 'new@example.com'
- def test_personal_data_encrypted_fields_are_populated_on_creation(self):
+ def test_personal_data_fields_are_stored_encrypted_on_creation(self):
user = M.User(
username='personal-data-encrypted-create-test',
last_access={
@@ -509,19 +509,26 @@ def test_personal_data_encrypted_fields_are_populated_on_creation(self):
)
ThreadLocalODMSession.flush_all()
+ assert user.last_access['login_ip'] == '192.0.2.1'
+ assert user.last_access['session_ip'] == '198.51.100.1'
+ assert user.socialnetworks[0]['accounturl'] == 'https://example.com/@user'
+ assert user.socialnetworks[1]['accounturl'] is None
+ assert user.telnumbers == ['+1-555-0100', None]
+
document = state(user).document
- assert document['last_access']['login_ip'] == '192.0.2.1'
+ assert 'login_ip' not in document['last_access']
assert document['last_access']['login_ip_encrypted'] == M.User.encr('192.0.2.1')
- assert document['last_access']['session_ip'] == '198.51.100.1'
+ assert 'session_ip' not in document['last_access']
assert document['last_access']['session_ip_encrypted'] == M.User.encr('198.51.100.1')
- assert document['socialnetworks'][0]['accounturl'] == 'https://example.com/@user'
+ assert 'accounturl' not in document['socialnetworks'][0]
assert document['socialnetworks'][0]['accounturl_encrypted'] == M.User.encr(
'https://example.com/@user')
+ assert 'accounturl' not in document['socialnetworks'][1]
assert document['socialnetworks'][1]['accounturl_encrypted'] is None
- assert document['telnumbers'] == ['+1-555-0100', None]
- assert user.telnumbers_encrypted == [M.User.encr('+1-555-0100'), None]
+ assert 'telnumbers' not in document
+ assert document['telnumbers_encrypted'] == [M.User.encr('+1-555-0100'), None]
- def test_personal_data_encrypted_fields_are_synced_on_update(self):
+ def test_personal_data_fields_stay_encrypted_on_update(self):
user = M.User(
username='personal-data-encrypted-update-test',
last_access={
@@ -548,22 +555,27 @@ def test_personal_data_encrypted_fields_are_synced_on_update(self):
user.telnumbers.append('+1-555-0103')
ThreadLocalODMSession.flush_all()
+ assert user.last_access['login_ip'] == '203.0.113.2'
+ assert user.last_access['session_ip'] is None
+ assert user.socialnetworks[0]['accounturl'] == 'https://new.example/@user'
+ assert user.telnumbers == ['+1-555-0102', '+1-555-0103']
+
document = state(user).document
- assert document['last_access']['login_ip'] == '203.0.113.2'
+ assert 'login_ip' not in document['last_access']
assert document['last_access']['login_ip_encrypted'] == M.User.encr('203.0.113.2')
- assert document['last_access']['session_ip'] is None
+ assert 'session_ip' not in document['last_access']
assert document['last_access']['session_ip_encrypted'] is None
assert len(document['socialnetworks']) == 1
- assert document['socialnetworks'][0]['accounturl'] == 'https://new.example/@user'
+ assert 'accounturl' not in document['socialnetworks'][0]
assert document['socialnetworks'][0]['accounturl_encrypted'] == M.User.encr(
'https://new.example/@user')
- assert document['telnumbers'] == ['+1-555-0102', '+1-555-0103']
- assert user.telnumbers_encrypted == [
+ assert 'telnumbers' not in document
+ assert document['telnumbers_encrypted'] == [
M.User.encr('+1-555-0102'),
M.User.encr('+1-555-0103'),
]
- def test_user_login_details_ip_encrypted_is_synced(self):
+ def test_user_login_details_ip_is_stored_encrypted(self):
detail = M.UserLoginDetails(
user_id=c.user._id,
ip='192.0.2.3',
@@ -573,14 +585,19 @@ def test_user_login_details_ip_encrypted_is_synced(self):
assert detail.ip == '192.0.2.3'
assert detail.ip_encrypted == M.UserLoginDetails.encr('192.0.2.3')
- assert state(detail).document['ip'] == '192.0.2.3'
+ assert 'ip' not in state(detail).document
detail.ip = '203.0.113.3'
session(detail).flush(detail)
assert detail.ip == '203.0.113.3'
assert detail.ip_encrypted == M.UserLoginDetails.encr('203.0.113.3')
- assert state(detail).document['ip'] == '203.0.113.3'
+ assert 'ip' not in state(detail).document
+
+ def test_user_login_details_unique_index_uses_encrypted_ip(self):
+ assert M.UserLoginDetails.__mongometa__.unique_indexes == [
+ ('user_id', 'ip_encrypted', 'ua'),
+ ]
def test_set_display_name_pref_updates_encrypted_field_and_cache(self):
user = M.User(username='display-name-cache-test',
diff --git a/Allura/allura/tests/scripts/test_convert_encrypted_field.py b/Allura/allura/tests/scripts/test_convert_encrypted_field.py
index c218db8de..e5415cc38 100644
--- a/Allura/allura/tests/scripts/test_convert_encrypted_field.py
+++ b/Allura/allura/tests/scripts/test_convert_encrypted_field.py
@@ -16,19 +16,19 @@
# under the License.
from ming import schema as S
+from ming.encryption import NestedEncryptedProperty
from ming.odm import FieldProperty
from scripts.convert_encrypted_field import (
+ _encryption_schema_info,
_encrypt_nested_array_value,
_remove_nested_array_value,
- _schema_info_for_field_path,
)
class TestModel:
- socialnetworks = FieldProperty([dict(
+ socialnetworks = NestedEncryptedProperty([dict(
socialnetwork=str,
- accounturl=str,
accounturl_encrypted=S.Binary,
)])
@@ -37,14 +37,34 @@ def encr(cls, value):
return None if value is None else f'encrypted:{value}'.encode()
-def test_schema_info_supports_nested_array_path():
- field_schema, traverses_array = _schema_info_for_field_path(
- TestModel, 'socialnetworks.accounturl_encrypted')
+def test_encryption_schema_info_uses_encrypted_nested_array_path():
+ field_schema, traverses_array = _encryption_schema_info(
+ TestModel,
+ 'socialnetworks.accounturl',
+ 'socialnetworks.accounturl_encrypted',
+ )
assert isinstance(field_schema, S.Binary)
assert traverses_array is True
+def test_encryption_schema_info_supports_model_without_encrypted_field():
+ class PreMigrationTestModel:
+ socialnetworks = FieldProperty([dict(
+ socialnetwork=str,
+ accounturl=str,
+ )])
+
+ field_schema, traverses_array = _encryption_schema_info(
+ PreMigrationTestModel,
+ 'socialnetworks.accounturl',
+ 'socialnetworks.accounturl_encrypted',
+ )
+
+ assert field_schema is None
+ assert traverses_array is True
+
+
def test_encrypt_nested_array_value_updates_every_item():
socialnetworks = [
{'socialnetwork': 'Mastodon', 'accounturl': 'https://example.com/@user'},
diff --git a/scripts/convert_encrypted_field.py b/scripts/convert_encrypted_field.py
index 379b10dbd..aba21ea42 100644
--- a/scripts/convert_encrypted_field.py
+++ b/scripts/convert_encrypted_field.py
@@ -74,9 +74,15 @@ def _schema_info_for_field_path(Model: type[MappedClass], field_name: str):
return current_schema, traverses_array
-def _schema_for_field_path(Model: type[MappedClass], field_name: str):
- field_schema, _ = _schema_info_for_field_path(Model, field_name)
- return field_schema
+def _encryption_schema_info(Model: type[MappedClass], plain_field_name: str,
+ encrypted_field_name: str):
+ try:
+ return _schema_info_for_field_path(Model, encrypted_field_name)
+ except MissingFieldPathError:
+ # Pre-migration support: infer array traversal from the plaintext
+ # schema when the encrypted field has not been added to the model yet.
+ _, traverses_array = _schema_info_for_field_path(Model, plain_field_name)
+ return None, traverses_array
def _get_nested_value(rec: dict, field_name: str):
@@ -212,12 +218,8 @@ def main(class_name: str, plain_field_name: str,
Model: type[MappedClass] = getattr(module, class_basename)
# sanity checks that the fields are correct and ready
- try:
- encr_schema = _schema_for_field_path(Model, encrypted_field_name)
- except MissingFieldPathError:
- # pre-migration support: encrypted field might not yet be present in
- # model schema, but we can still create it with raw MongoDB updates
- encr_schema = None
+ encr_schema, traverses_array = _encryption_schema_info(
+ Model, plain_field_name, encrypted_field_name)
if encr_schema is not None:
assert isinstance(encr_schema, schema.Binary) or _is_encrypted_list_schema(encr_schema)
elif remove_unencrypted:
@@ -239,7 +241,6 @@ def main(class_name: str, plain_field_name: str,
m = mapper(Model)
raw_collection = sess.impl.db[m.collection.m.collection_name]
- _, traverses_array = _schema_info_for_field_path(Model, plain_field_name)
if traverses_array:
plain_field_path = _split_field_path(plain_field_name)
encrypted_field_path = _split_field_path(encrypted_field_name)