You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by cc...@apache.org on 2026/07/21 13:49:19 UTC
(allura) branch cc/8614-part2 created (now ec4802b29)
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a change to branch cc/8614-part2
in repository https://gitbox.apache.org/repos/asf/allura.git
at ec4802b29 fixup! [#8614] Implement field level encryption for User and UserLoginDetails fields
This branch includes the following new commits:
new ae9a0a6ee [#8614] Implement field level encryption for User and UserLoginDetails fields
new ec4802b29 fixup! [#8614] Implement field level encryption for User and UserLoginDetails fields
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(allura) 02/02: fixup! [#8614] Implement field level encryption for User and UserLoginDetails fields
Posted by cc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch cc/8614-part2
in repository https://gitbox.apache.org/repos/asf/allura.git
commit ec4802b299ed124e991194f2d7b9398ca3018fe2
Author: Carlos Cruz <ca...@slashdotmedia.com>
AuthorDate: Mon Jul 20 15:27:21 2026 -0600
fixup! [#8614] Implement field level encryption for User and UserLoginDetails fields
---
scripts/migrations/037-field-encryption.sh | 7 ++++++-
scripts/migrations/038-field-encryption-cleanup.sh | 7 ++++++-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/scripts/migrations/037-field-encryption.sh b/scripts/migrations/037-field-encryption.sh
index f81d6899c..1b13ec439 100755
--- a/scripts/migrations/037-field-encryption.sh
+++ b/scripts/migrations/037-field-encryption.sh
@@ -34,4 +34,9 @@ export PYTHONWARNINGS="${PYTHONWARNINGS:-ignore:pkg_resources is deprecated as a
CMD="paster script $INI ../scripts/convert_encrypted_field.py"
-$CMD allura.model.auth.User email_addresses
+$CMD allura.model.auth.User email_addresses && \
+$CMD allura.model.auth.User last_access.login_ip && \
+$CMD allura.model.auth.User last_access.session_ip && \
+$CMD allura.model.auth.User socialnetworks.accounturl && \
+$CMD allura.model.auth.User telnumbers && \
+$CMD allura.model.auth.UserLoginDetails ip
diff --git a/scripts/migrations/038-field-encryption-cleanup.sh b/scripts/migrations/038-field-encryption-cleanup.sh
index a3a21dbcc..82e44ac6f 100755
--- a/scripts/migrations/038-field-encryption-cleanup.sh
+++ b/scripts/migrations/038-field-encryption-cleanup.sh
@@ -34,4 +34,9 @@ export PYTHONWARNINGS="${PYTHONWARNINGS:-ignore:pkg_resources is deprecated as a
CMD="paster script $INI ../scripts/convert_encrypted_field.py -- --remove-unencrypted"
-$CMD allura.model.auth.User email_addresses
+$CMD allura.model.auth.User email_addresses && \
+$CMD allura.model.auth.User last_access.login_ip && \
+$CMD allura.model.auth.User last_access.session_ip && \
+$CMD allura.model.auth.User socialnetworks.accounturl && \
+$CMD allura.model.auth.User telnumbers && \
+$CMD allura.model.auth.UserLoginDetails ip
(allura) 01/02: [#8614] Implement field level encryption for User and UserLoginDetails fields
Posted by cc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch cc/8614-part2
in repository https://gitbox.apache.org/repos/asf/allura.git
commit ae9a0a6ee87070c5c8490a87096fd0194f985e04
Author: Carlos Cruz <ca...@slashdotmedia.com>
AuthorDate: Mon Jul 20 15:17:14 2026 -0600
[#8614] Implement field level encryption for User and UserLoginDetails fields
---
Allura/allura/ext/user_profile/user_main.py | 7 ++-
Allura/allura/model/auth.py | 53 +++-------------------
.../allura/tests/functional/test_user_profile.py | 19 ++++++++
Allura/allura/tests/model/test_auth.py | 47 +++++++++++++------
.../tests/scripts/test_convert_encrypted_field.py | 32 ++++++++++---
scripts/convert_encrypted_field.py | 21 +++++----
6 files changed, 100 insertions(+), 79 deletions(-)
diff --git a/Allura/allura/ext/user_profile/user_main.py b/Allura/allura/ext/user_profile/user_main.py
index 4e6e253dc..39d5745dc 100644
--- a/Allura/allura/ext/user_profile/user_main.py
+++ b/Allura/allura/ext/user_profile/user_main.py
@@ -306,7 +306,7 @@ def __json__(self):
joined=auth_provider.user_registration_date(self.user),
localization=self.user.get_pref('localization')._deinstrument(),
sex=self.user.get_pref('sex'),
- telnumbers=self.user.get_pref('telnumbers')._deinstrument(),
+ telnumbers=list(self.user.get_pref('telnumbers')),
webpages=self.user.get_pref('webpages')._deinstrument(),
availability=self.user.get_pref('availability')._deinstrument())
@@ -334,7 +334,10 @@ class SocialSection(ProfileSectionBase):
def __json__(self):
return dict(
- socialnetworks=self.user.get_pref('socialnetworks')._deinstrument())
+ socialnetworks=[
+ dict(socialnetwork.items())
+ for socialnetwork in self.user.get_pref('socialnetworks')
+ ])
def check_display(self):
return bool(self.user.get_pref('socialnetworks'))
diff --git a/Allura/allura/model/auth.py b/Allura/allura/model/auth.py
index ac59dc086..233f51962 100644
--- a/Allura/allura/model/auth.py
+++ b/Allura/allura/model/auth.py
@@ -38,7 +38,8 @@
from tg import tmpl_context as c, app_globals as g
from tg import request
from ming import schema as S
-from ming.odm import session, state, MapperExtension
+from ming.encryption import NestedEncryptedProperty
+from ming.odm import session, state
from ming.odm import FieldProperty, RelationProperty, ForeignIdProperty, DecryptedProperty, DecryptedListProperty
from ming.odm.declarative import MappedClass
from ming.odm.odmsession import ThreadLocalODMSession
@@ -266,15 +267,6 @@ def get_decrypted(self, instance):
return super().__get__(instance, type(instance))
-class UserFieldEncryptionMapperExtension(MapperExtension):
-
- def before_insert(self, obj, state, sess):
- obj.sync_personal_data_encrypted()
-
- def before_update(self, obj, state, sess):
- obj.sync_personal_data_encrypted()
-
-
class User(MappedClass, ActivityNode, ActivityObject, SearchIndexable):
SALT_LEN = 8
@@ -283,7 +275,6 @@ class __mongometa__:
session = main_orm_session
indexes = ['tool_data.AuthPasswordReset.hash']
unique_indexes = ['username']
- extensions = [UserFieldEncryptionMapperExtension]
custom_indexes = [
dict(fields=('tool_data.phone_verification.number_hash',), sparse=True),
]
@@ -341,12 +332,11 @@ class __mongometa__:
end_date=S.DateTime)])
# Additional contacts
- socialnetworks = FieldProperty([dict(
+ socialnetworks = NestedEncryptedProperty([dict(
socialnetwork=str,
- accounturl=str,
accounturl_encrypted=S.Binary,
)])
- telnumbers = FieldProperty([str])
+ telnumbers = DecryptedListProperty('telnumbers_encrypted')
telnumbers_encrypted = FieldProperty([S.Binary], if_missing=[])
webpages = FieldProperty([str])
@@ -358,30 +348,14 @@ class __mongometa__:
# Statistics
stats_id = FieldProperty(S.ObjectId, if_missing=None)
- last_access = FieldProperty(dict(
+ last_access = NestedEncryptedProperty(dict(
login_date=S.DateTime,
- login_ip=str,
login_ip_encrypted=S.Binary,
login_ua=str,
session_date=S.DateTime,
- session_ip=str,
session_ip_encrypted=S.Binary,
session_ua=str))
- def sync_personal_data_encrypted(self):
- last_access = dict(self.last_access or {})
- for field_name in ('login_ip', 'session_ip'):
- value = last_access.get(field_name)
- last_access[f'{field_name}_encrypted'] = self.encr(value)
- self.last_access = last_access
-
- socialnetworks = [dict(socialnetwork) for socialnetwork in self.socialnetworks or []]
- for socialnetwork in socialnetworks:
- socialnetwork['accounturl_encrypted'] = self.encr(socialnetwork.get('accounturl'))
- self.socialnetworks = socialnetworks
-
- self.telnumbers_encrypted = [self.encr(telnumber) for telnumber in self.telnumbers or []]
-
def __repr__(self):
try:
display_name = self.display_name
@@ -1219,15 +1193,6 @@ def comment_user(cls, by, message, *args, **kwargs):
return cls.log_user(message, *args, **kwargs)
-class UserLoginDetailsEncryptionMapperExtension(MapperExtension):
-
- def before_insert(self, obj, state, sess):
- obj.sync_ip_encrypted()
-
- def before_update(self, obj, state, sess):
- obj.sync_ip_encrypted()
-
-
class UserLoginDetails(MappedClass):
"""
Store unique entries for users' previous login details. And other trusted actions like successful password reset
@@ -1239,15 +1204,14 @@ class __mongometa__:
name = 'user_login_details'
session = main_explicitflush_orm_session
indexes = ['user_id']
- unique_indexes = [('user_id', 'ip', 'ua'), # DuplicateKeyError checked in add_login_detail
+ unique_indexes = [('user_id', 'ip_encrypted', 'ua'), # DuplicateKeyError checked in add_login_detail
]
- extensions = [UserLoginDetailsEncryptionMapperExtension]
query: Query[UserLoginDetails]
_id = FieldProperty(S.ObjectId)
user_id: ObjectId = AlluraUserProperty(required=True)
- ip = FieldProperty(str)
+ ip = DecryptedProperty(str, 'ip_encrypted')
ip_encrypted = FieldProperty(S.Binary, if_missing=None)
ua = FieldProperty(str)
extra = FieldProperty({
@@ -1255,6 +1219,3 @@ class __mongometa__:
})
user = RelationProperty('User')
-
- def sync_ip_encrypted(self):
- self.ip_encrypted = self.encr(self.ip)
diff --git a/Allura/allura/tests/functional/test_user_profile.py b/Allura/allura/tests/functional/test_user_profile.py
index 2e9cc7af3..d4573b8f1 100644
--- a/Allura/allura/tests/functional/test_user_profile.py
+++ b/Allura/allura/tests/functional/test_user_profile.py
@@ -17,6 +17,7 @@
import mock
import tg
+from ming.odm import ThreadLocalODMSession
from alluratest.controller import TestRestApiBase
from allura.model import Project, User
@@ -100,6 +101,24 @@ def test_profile_user_card(self):
assert user.get_pref('localization')['country'] in r.html.find('span', attrs={'class': 'subitem-loc'}).getText()
assert user.get_pref('webpages')[0] in str(r.html.find('span', attrs={'class': 'subitem-web'}))
+ @td.with_user_project('test-admin')
+ def test_profile_rest_returns_decrypted_contacts(self):
+ user = User.by_username('test-admin')
+ user.set_pref('telnumbers', ['+1-555-0100', '+1-555-0101'])
+ user.set_pref('socialnetworks', [{
+ 'socialnetwork': 'Mastodon',
+ 'accounturl': 'https://example.com/@test-admin',
+ }])
+ ThreadLocalODMSession.flush_all()
+
+ response = self.app.get('/rest/u/test-admin/profile/')
+
+ assert response.json['telnumbers'] == ['+1-555-0100', '+1-555-0101']
+ assert response.json['socialnetworks'] == [{
+ 'socialnetwork': 'Mastodon',
+ 'accounturl': 'https://example.com/@test-admin',
+ }]
+
def test_wrong_profile(self):
self.app.get('/u/no-such-user/profile/', status=404)
diff --git a/Allura/allura/tests/model/test_auth.py b/Allura/allura/tests/model/test_auth.py
index 0af6b9c0f..b400d6510 100644
--- a/Allura/allura/tests/model/test_auth.py
+++ b/Allura/allura/tests/model/test_auth.py
@@ -467,7 +467,7 @@ def test_email_addresses_encrypted_is_synced_on_update(self):
assert 'email_addresses' not in state(user).document
assert state(user).document['email_addresses_encrypted'] == expected_encrypted
- def test_personal_data_encrypted_fields_are_populated_on_creation(self):
+ def test_personal_data_fields_are_stored_encrypted_on_creation(self):
user = M.User(
username='personal-data-encrypted-create-test',
last_access={
@@ -482,19 +482,26 @@ def test_personal_data_encrypted_fields_are_populated_on_creation(self):
)
ThreadLocalODMSession.flush_all()
+ assert user.last_access['login_ip'] == '192.0.2.1'
+ assert user.last_access['session_ip'] == '198.51.100.1'
+ assert user.socialnetworks[0]['accounturl'] == 'https://example.com/@user'
+ assert user.socialnetworks[1]['accounturl'] is None
+ assert user.telnumbers == ['+1-555-0100', None]
+
document = state(user).document
- assert document['last_access']['login_ip'] == '192.0.2.1'
+ assert 'login_ip' not in document['last_access']
assert document['last_access']['login_ip_encrypted'] == M.User.encr('192.0.2.1')
- assert document['last_access']['session_ip'] == '198.51.100.1'
+ assert 'session_ip' not in document['last_access']
assert document['last_access']['session_ip_encrypted'] == M.User.encr('198.51.100.1')
- assert document['socialnetworks'][0]['accounturl'] == 'https://example.com/@user'
+ assert 'accounturl' not in document['socialnetworks'][0]
assert document['socialnetworks'][0]['accounturl_encrypted'] == M.User.encr(
'https://example.com/@user')
+ assert 'accounturl' not in document['socialnetworks'][1]
assert document['socialnetworks'][1]['accounturl_encrypted'] is None
- assert document['telnumbers'] == ['+1-555-0100', None]
- assert user.telnumbers_encrypted == [M.User.encr('+1-555-0100'), None]
+ assert 'telnumbers' not in document
+ assert document['telnumbers_encrypted'] == [M.User.encr('+1-555-0100'), None]
- def test_personal_data_encrypted_fields_are_synced_on_update(self):
+ def test_personal_data_fields_stay_encrypted_on_update(self):
user = M.User(
username='personal-data-encrypted-update-test',
last_access={
@@ -521,22 +528,27 @@ def test_personal_data_encrypted_fields_are_synced_on_update(self):
user.telnumbers.append('+1-555-0103')
ThreadLocalODMSession.flush_all()
+ assert user.last_access['login_ip'] == '203.0.113.2'
+ assert user.last_access['session_ip'] is None
+ assert user.socialnetworks[0]['accounturl'] == 'https://new.example/@user'
+ assert user.telnumbers == ['+1-555-0102', '+1-555-0103']
+
document = state(user).document
- assert document['last_access']['login_ip'] == '203.0.113.2'
+ assert 'login_ip' not in document['last_access']
assert document['last_access']['login_ip_encrypted'] == M.User.encr('203.0.113.2')
- assert document['last_access']['session_ip'] is None
+ assert 'session_ip' not in document['last_access']
assert document['last_access']['session_ip_encrypted'] is None
assert len(document['socialnetworks']) == 1
- assert document['socialnetworks'][0]['accounturl'] == 'https://new.example/@user'
+ assert 'accounturl' not in document['socialnetworks'][0]
assert document['socialnetworks'][0]['accounturl_encrypted'] == M.User.encr(
'https://new.example/@user')
- assert document['telnumbers'] == ['+1-555-0102', '+1-555-0103']
- assert user.telnumbers_encrypted == [
+ assert 'telnumbers' not in document
+ assert document['telnumbers_encrypted'] == [
M.User.encr('+1-555-0102'),
M.User.encr('+1-555-0103'),
]
- def test_user_login_details_ip_encrypted_is_synced(self):
+ def test_user_login_details_ip_is_stored_encrypted(self):
detail = M.UserLoginDetails(
user_id=c.user._id,
ip='192.0.2.3',
@@ -546,14 +558,19 @@ def test_user_login_details_ip_encrypted_is_synced(self):
assert detail.ip == '192.0.2.3'
assert detail.ip_encrypted == M.UserLoginDetails.encr('192.0.2.3')
- assert state(detail).document['ip'] == '192.0.2.3'
+ assert 'ip' not in state(detail).document
detail.ip = '203.0.113.3'
session(detail).flush(detail)
assert detail.ip == '203.0.113.3'
assert detail.ip_encrypted == M.UserLoginDetails.encr('203.0.113.3')
- assert state(detail).document['ip'] == '203.0.113.3'
+ assert 'ip' not in state(detail).document
+
+ def test_user_login_details_unique_index_uses_encrypted_ip(self):
+ assert M.UserLoginDetails.__mongometa__.unique_indexes == [
+ ('user_id', 'ip_encrypted', 'ua'),
+ ]
def test_set_display_name_pref_updates_encrypted_field_and_cache(self):
user = M.User(username='display-name-cache-test',
diff --git a/Allura/allura/tests/scripts/test_convert_encrypted_field.py b/Allura/allura/tests/scripts/test_convert_encrypted_field.py
index c218db8de..e5415cc38 100644
--- a/Allura/allura/tests/scripts/test_convert_encrypted_field.py
+++ b/Allura/allura/tests/scripts/test_convert_encrypted_field.py
@@ -16,19 +16,19 @@
# under the License.
from ming import schema as S
+from ming.encryption import NestedEncryptedProperty
from ming.odm import FieldProperty
from scripts.convert_encrypted_field import (
+ _encryption_schema_info,
_encrypt_nested_array_value,
_remove_nested_array_value,
- _schema_info_for_field_path,
)
class TestModel:
- socialnetworks = FieldProperty([dict(
+ socialnetworks = NestedEncryptedProperty([dict(
socialnetwork=str,
- accounturl=str,
accounturl_encrypted=S.Binary,
)])
@@ -37,14 +37,34 @@ def encr(cls, value):
return None if value is None else f'encrypted:{value}'.encode()
-def test_schema_info_supports_nested_array_path():
- field_schema, traverses_array = _schema_info_for_field_path(
- TestModel, 'socialnetworks.accounturl_encrypted')
+def test_encryption_schema_info_uses_encrypted_nested_array_path():
+ field_schema, traverses_array = _encryption_schema_info(
+ TestModel,
+ 'socialnetworks.accounturl',
+ 'socialnetworks.accounturl_encrypted',
+ )
assert isinstance(field_schema, S.Binary)
assert traverses_array is True
+def test_encryption_schema_info_supports_model_without_encrypted_field():
+ class PreMigrationTestModel:
+ socialnetworks = FieldProperty([dict(
+ socialnetwork=str,
+ accounturl=str,
+ )])
+
+ field_schema, traverses_array = _encryption_schema_info(
+ PreMigrationTestModel,
+ 'socialnetworks.accounturl',
+ 'socialnetworks.accounturl_encrypted',
+ )
+
+ assert field_schema is None
+ assert traverses_array is True
+
+
def test_encrypt_nested_array_value_updates_every_item():
socialnetworks = [
{'socialnetwork': 'Mastodon', 'accounturl': 'https://example.com/@user'},
diff --git a/scripts/convert_encrypted_field.py b/scripts/convert_encrypted_field.py
index 379b10dbd..aba21ea42 100644
--- a/scripts/convert_encrypted_field.py
+++ b/scripts/convert_encrypted_field.py
@@ -74,9 +74,15 @@ def _schema_info_for_field_path(Model: type[MappedClass], field_name: str):
return current_schema, traverses_array
-def _schema_for_field_path(Model: type[MappedClass], field_name: str):
- field_schema, _ = _schema_info_for_field_path(Model, field_name)
- return field_schema
+def _encryption_schema_info(Model: type[MappedClass], plain_field_name: str,
+ encrypted_field_name: str):
+ try:
+ return _schema_info_for_field_path(Model, encrypted_field_name)
+ except MissingFieldPathError:
+ # Pre-migration support: infer array traversal from the plaintext
+ # schema when the encrypted field has not been added to the model yet.
+ _, traverses_array = _schema_info_for_field_path(Model, plain_field_name)
+ return None, traverses_array
def _get_nested_value(rec: dict, field_name: str):
@@ -212,12 +218,8 @@ def main(class_name: str, plain_field_name: str,
Model: type[MappedClass] = getattr(module, class_basename)
# sanity checks that the fields are correct and ready
- try:
- encr_schema = _schema_for_field_path(Model, encrypted_field_name)
- except MissingFieldPathError:
- # pre-migration support: encrypted field might not yet be present in
- # model schema, but we can still create it with raw MongoDB updates
- encr_schema = None
+ encr_schema, traverses_array = _encryption_schema_info(
+ Model, plain_field_name, encrypted_field_name)
if encr_schema is not None:
assert isinstance(encr_schema, schema.Binary) or _is_encrypted_list_schema(encr_schema)
elif remove_unencrypted:
@@ -239,7 +241,6 @@ def main(class_name: str, plain_field_name: str,
m = mapper(Model)
raw_collection = sess.impl.db[m.collection.m.collection_name]
- _, traverses_array = _schema_info_for_field_path(Model, plain_field_name)
if traverses_array:
plain_field_path = _split_field_path(plain_field_name)
encrypted_field_path = _split_field_path(encrypted_field_name)