You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by cc...@apache.org on 2026/07/21 13:49:02 UTC
(allura) branch cc/8614-part1 created (now ffd17d722)
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a change to branch cc/8614-part1
in repository https://gitbox.apache.org/repos/asf/allura.git
at ffd17d722 [#8614] Add support for nested lists and tests to the field conversion script
This branch includes the following new commits:
new 3b8b98c05 [#8614] Add backing encrypted fields for User and UserLoginDetails fields
new ffd17d722 [#8614] Add support for nested lists and tests to the field conversion script
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(allura) 01/02: [#8614] Add backing encrypted fields for User and UserLoginDetails fields
Posted by cc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch cc/8614-part1
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 3b8b98c057dde73083f8219b6d536d845c74c0f5
Author: Carlos Cruz <ca...@slashdotmedia.com>
AuthorDate: Mon Jul 20 12:19:07 2026 -0600
[#8614] Add backing encrypted fields for User and UserLoginDetails fields
---
Allura/allura/model/auth.py | 49 ++++++++++++++++++-
Allura/allura/tests/model/test_auth.py | 88 ++++++++++++++++++++++++++++++++++
2 files changed, 135 insertions(+), 2 deletions(-)
diff --git a/Allura/allura/model/auth.py b/Allura/allura/model/auth.py
index 4f526a5ae..ac59dc086 100644
--- a/Allura/allura/model/auth.py
+++ b/Allura/allura/model/auth.py
@@ -38,7 +38,7 @@
from tg import tmpl_context as c, app_globals as g
from tg import request
from ming import schema as S
-from ming.odm import session, state
+from ming.odm import session, state, MapperExtension
from ming.odm import FieldProperty, RelationProperty, ForeignIdProperty, DecryptedProperty, DecryptedListProperty
from ming.odm.declarative import MappedClass
from ming.odm.odmsession import ThreadLocalODMSession
@@ -266,6 +266,15 @@ def get_decrypted(self, instance):
return super().__get__(instance, type(instance))
+class UserFieldEncryptionMapperExtension(MapperExtension):
+
+ def before_insert(self, obj, state, sess):
+ obj.sync_personal_data_encrypted()
+
+ def before_update(self, obj, state, sess):
+ obj.sync_personal_data_encrypted()
+
+
class User(MappedClass, ActivityNode, ActivityObject, SearchIndexable):
SALT_LEN = 8
@@ -274,6 +283,7 @@ class __mongometa__:
session = main_orm_session
indexes = ['tool_data.AuthPasswordReset.hash']
unique_indexes = ['username']
+ extensions = [UserFieldEncryptionMapperExtension]
custom_indexes = [
dict(fields=('tool_data.phone_verification.number_hash',), sparse=True),
]
@@ -331,8 +341,13 @@ class __mongometa__:
end_date=S.DateTime)])
# Additional contacts
- socialnetworks = FieldProperty([dict(socialnetwork=str, accounturl=str)])
+ socialnetworks = FieldProperty([dict(
+ socialnetwork=str,
+ accounturl=str,
+ accounturl_encrypted=S.Binary,
+ )])
telnumbers = FieldProperty([str])
+ telnumbers_encrypted = FieldProperty([S.Binary], if_missing=[])
webpages = FieldProperty([str])
# Skills list
@@ -346,11 +361,27 @@ class __mongometa__:
last_access = FieldProperty(dict(
login_date=S.DateTime,
login_ip=str,
+ login_ip_encrypted=S.Binary,
login_ua=str,
session_date=S.DateTime,
session_ip=str,
+ session_ip_encrypted=S.Binary,
session_ua=str))
+ def sync_personal_data_encrypted(self):
+ last_access = dict(self.last_access or {})
+ for field_name in ('login_ip', 'session_ip'):
+ value = last_access.get(field_name)
+ last_access[f'{field_name}_encrypted'] = self.encr(value)
+ self.last_access = last_access
+
+ socialnetworks = [dict(socialnetwork) for socialnetwork in self.socialnetworks or []]
+ for socialnetwork in socialnetworks:
+ socialnetwork['accounturl_encrypted'] = self.encr(socialnetwork.get('accounturl'))
+ self.socialnetworks = socialnetworks
+
+ self.telnumbers_encrypted = [self.encr(telnumber) for telnumber in self.telnumbers or []]
+
def __repr__(self):
try:
display_name = self.display_name
@@ -1188,6 +1219,15 @@ def comment_user(cls, by, message, *args, **kwargs):
return cls.log_user(message, *args, **kwargs)
+class UserLoginDetailsEncryptionMapperExtension(MapperExtension):
+
+ def before_insert(self, obj, state, sess):
+ obj.sync_ip_encrypted()
+
+ def before_update(self, obj, state, sess):
+ obj.sync_ip_encrypted()
+
+
class UserLoginDetails(MappedClass):
"""
Store unique entries for users' previous login details. And other trusted actions like successful password reset
@@ -1201,15 +1241,20 @@ class __mongometa__:
indexes = ['user_id']
unique_indexes = [('user_id', 'ip', 'ua'), # DuplicateKeyError checked in add_login_detail
]
+ extensions = [UserLoginDetailsEncryptionMapperExtension]
query: Query[UserLoginDetails]
_id = FieldProperty(S.ObjectId)
user_id: ObjectId = AlluraUserProperty(required=True)
ip = FieldProperty(str)
+ ip_encrypted = FieldProperty(S.Binary, if_missing=None)
ua = FieldProperty(str)
extra = FieldProperty({
str: S.Anything
})
user = RelationProperty('User')
+
+ def sync_ip_encrypted(self):
+ self.ip_encrypted = self.encr(self.ip)
diff --git a/Allura/allura/tests/model/test_auth.py b/Allura/allura/tests/model/test_auth.py
index 15a1b20aa..0af6b9c0f 100644
--- a/Allura/allura/tests/model/test_auth.py
+++ b/Allura/allura/tests/model/test_auth.py
@@ -467,6 +467,94 @@ def test_email_addresses_encrypted_is_synced_on_update(self):
assert 'email_addresses' not in state(user).document
assert state(user).document['email_addresses_encrypted'] == expected_encrypted
+ def test_personal_data_encrypted_fields_are_populated_on_creation(self):
+ user = M.User(
+ username='personal-data-encrypted-create-test',
+ last_access={
+ 'login_ip': '192.0.2.1',
+ 'session_ip': '198.51.100.1',
+ },
+ socialnetworks=[
+ {'socialnetwork': 'Mastodon', 'accounturl': 'https://example.com/@user'},
+ {'socialnetwork': 'Other', 'accounturl': None},
+ ],
+ telnumbers=['+1-555-0100', None],
+ )
+ ThreadLocalODMSession.flush_all()
+
+ document = state(user).document
+ assert document['last_access']['login_ip'] == '192.0.2.1'
+ assert document['last_access']['login_ip_encrypted'] == M.User.encr('192.0.2.1')
+ assert document['last_access']['session_ip'] == '198.51.100.1'
+ assert document['last_access']['session_ip_encrypted'] == M.User.encr('198.51.100.1')
+ assert document['socialnetworks'][0]['accounturl'] == 'https://example.com/@user'
+ assert document['socialnetworks'][0]['accounturl_encrypted'] == M.User.encr(
+ 'https://example.com/@user')
+ assert document['socialnetworks'][1]['accounturl_encrypted'] is None
+ assert document['telnumbers'] == ['+1-555-0100', None]
+ assert user.telnumbers_encrypted == [M.User.encr('+1-555-0100'), None]
+
+ def test_personal_data_encrypted_fields_are_synced_on_update(self):
+ user = M.User(
+ username='personal-data-encrypted-update-test',
+ last_access={
+ 'login_ip': '192.0.2.2',
+ 'session_ip': '198.51.100.2',
+ },
+ socialnetworks=[
+ {'socialnetwork': 'Mastodon', 'accounturl': 'https://old.example/@user'},
+ {'socialnetwork': 'Other', 'accounturl': 'https://remove.example/user'},
+ ],
+ telnumbers=['+1-555-0101', '+1-555-0102'],
+ )
+ ThreadLocalODMSession.flush_all()
+
+ assert user.telnumbers_encrypted == [
+ M.User.encr('+1-555-0101'),
+ M.User.encr('+1-555-0102'),
+ ]
+ user.last_access['login_ip'] = '203.0.113.2'
+ user.last_access['session_ip'] = None
+ user.socialnetworks[0]['accounturl'] = 'https://new.example/@user'
+ user.socialnetworks.pop()
+ user.telnumbers.remove('+1-555-0101')
+ user.telnumbers.append('+1-555-0103')
+ ThreadLocalODMSession.flush_all()
+
+ document = state(user).document
+ assert document['last_access']['login_ip'] == '203.0.113.2'
+ assert document['last_access']['login_ip_encrypted'] == M.User.encr('203.0.113.2')
+ assert document['last_access']['session_ip'] is None
+ assert document['last_access']['session_ip_encrypted'] is None
+ assert len(document['socialnetworks']) == 1
+ assert document['socialnetworks'][0]['accounturl'] == 'https://new.example/@user'
+ assert document['socialnetworks'][0]['accounturl_encrypted'] == M.User.encr(
+ 'https://new.example/@user')
+ assert document['telnumbers'] == ['+1-555-0102', '+1-555-0103']
+ assert user.telnumbers_encrypted == [
+ M.User.encr('+1-555-0102'),
+ M.User.encr('+1-555-0103'),
+ ]
+
+ def test_user_login_details_ip_encrypted_is_synced(self):
+ detail = M.UserLoginDetails(
+ user_id=c.user._id,
+ ip='192.0.2.3',
+ ua='PersonalDataEncryptionTest/1.0',
+ )
+ session(detail).flush(detail)
+
+ assert detail.ip == '192.0.2.3'
+ assert detail.ip_encrypted == M.UserLoginDetails.encr('192.0.2.3')
+ assert state(detail).document['ip'] == '192.0.2.3'
+
+ detail.ip = '203.0.113.3'
+ session(detail).flush(detail)
+
+ assert detail.ip == '203.0.113.3'
+ assert detail.ip_encrypted == M.UserLoginDetails.encr('203.0.113.3')
+ assert state(detail).document['ip'] == '203.0.113.3'
+
def test_set_display_name_pref_updates_encrypted_field_and_cache(self):
user = M.User(username='display-name-cache-test',
display_name='Original Display Name')
(allura) 02/02: [#8614] Add support for nested lists and tests to the field conversion script
Posted by cc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch cc/8614-part1
in repository https://gitbox.apache.org/repos/asf/allura.git
commit ffd17d7228d120d1cc52651a48c36f5fd2c68466
Author: Carlos Cruz <ca...@slashdotmedia.com>
AuthorDate: Mon Jul 20 14:28:13 2026 -0600
[#8614] Add support for nested lists and tests to the field conversion script
---
.../tests/scripts/test_convert_encrypted_field.py | 111 +++++++++++++++
scripts/convert_encrypted_field.py | 153 +++++++++++++++++++--
2 files changed, 256 insertions(+), 8 deletions(-)
diff --git a/Allura/allura/tests/scripts/test_convert_encrypted_field.py b/Allura/allura/tests/scripts/test_convert_encrypted_field.py
new file mode 100644
index 000000000..c218db8de
--- /dev/null
+++ b/Allura/allura/tests/scripts/test_convert_encrypted_field.py
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from ming import schema as S
+from ming.odm import FieldProperty
+
+from scripts.convert_encrypted_field import (
+ _encrypt_nested_array_value,
+ _remove_nested_array_value,
+ _schema_info_for_field_path,
+)
+
+
+class TestModel:
+ socialnetworks = FieldProperty([dict(
+ socialnetwork=str,
+ accounturl=str,
+ accounturl_encrypted=S.Binary,
+ )])
+
+ @classmethod
+ def encr(cls, value):
+ return None if value is None else f'encrypted:{value}'.encode()
+
+
+def test_schema_info_supports_nested_array_path():
+ field_schema, traverses_array = _schema_info_for_field_path(
+ TestModel, 'socialnetworks.accounturl_encrypted')
+
+ assert isinstance(field_schema, S.Binary)
+ assert traverses_array is True
+
+
+def test_encrypt_nested_array_value_updates_every_item():
+ socialnetworks = [
+ {'socialnetwork': 'Mastodon', 'accounturl': 'https://example.com/@user'},
+ {'socialnetwork': 'LinkedIn', 'accounturl': 'https://example.com/in/user'},
+ {'socialnetwork': 'Other', 'accounturl': None},
+ ]
+
+ encrypted, changed = _encrypt_nested_array_value(
+ TestModel,
+ socialnetworks,
+ ['accounturl'],
+ 'accounturl_encrypted',
+ S.Binary(),
+ redo_all=False,
+ )
+
+ assert changed is True
+ assert encrypted == [
+ {
+ 'socialnetwork': 'Mastodon',
+ 'accounturl': 'https://example.com/@user',
+ 'accounturl_encrypted': b'encrypted:https://example.com/@user',
+ },
+ {
+ 'socialnetwork': 'LinkedIn',
+ 'accounturl': 'https://example.com/in/user',
+ 'accounturl_encrypted': b'encrypted:https://example.com/in/user',
+ },
+ {
+ 'socialnetwork': 'Other',
+ 'accounturl': None,
+ 'accounturl_encrypted': None,
+ },
+ ]
+
+
+def test_remove_nested_array_value_preserves_encrypted_and_other_fields():
+ socialnetworks = [
+ {
+ 'socialnetwork': 'Mastodon',
+ 'accounturl': 'https://example.com/@user',
+ 'accounturl_encrypted': b'encrypted:https://example.com/@user',
+ },
+ {
+ 'socialnetwork': 'Other',
+ 'accounturl': None,
+ 'accounturl_encrypted': None,
+ },
+ ]
+
+ encrypted_only, changed = _remove_nested_array_value(
+ socialnetworks, ['accounturl'])
+
+ assert changed is True
+ assert encrypted_only == [
+ {
+ 'socialnetwork': 'Mastodon',
+ 'accounturl_encrypted': b'encrypted:https://example.com/@user',
+ },
+ {
+ 'socialnetwork': 'Other',
+ 'accounturl_encrypted': None,
+ },
+ ]
diff --git a/scripts/convert_encrypted_field.py b/scripts/convert_encrypted_field.py
index e99c25a77..379b10dbd 100644
--- a/scripts/convert_encrypted_field.py
+++ b/scripts/convert_encrypted_field.py
@@ -45,7 +45,7 @@ def _split_field_path(field_name: str) -> list[str]:
return field_path
-def _schema_for_field_path(Model: type[MappedClass], field_name: str):
+def _schema_info_for_field_path(Model: type[MappedClass], field_name: str):
field_path = _split_field_path(field_name)
top_level_name = field_path[0]
try:
@@ -56,11 +56,12 @@ def _schema_for_field_path(Model: type[MappedClass], field_name: str):
assert isinstance(top_level_prop, FieldProperty)
current_schema = top_level_prop.field.schema
+ traverses_array = False
for i, path_part in enumerate(field_path[1:], start=1):
traversed = '.'.join(field_path[:i])
- if isinstance(current_schema, schema.Array):
- raise AssertionError(
- f'Nested array paths are not supported: {field_name!r} (array at {traversed!r})')
+ while isinstance(current_schema, schema.Array):
+ traverses_array = True
+ current_schema = current_schema.field_type
if not isinstance(current_schema, schema.Object):
raise AssertionError(
f'Invalid nested field path {field_name!r}; {traversed!r} is not an object field')
@@ -70,7 +71,12 @@ def _schema_for_field_path(Model: type[MappedClass], field_name: str):
current_schema = current_schema.fields[path_part]
- return current_schema
+ return current_schema, traverses_array
+
+
+def _schema_for_field_path(Model: type[MappedClass], field_name: str):
+ field_schema, _ = _schema_info_for_field_path(Model, field_name)
+ return field_schema
def _get_nested_value(rec: dict, field_name: str):
@@ -80,6 +86,67 @@ def _get_nested_value(rec: dict, field_name: str):
return value
+def _transform_nested_array_value(value, field_path, transform_leaf):
+ if isinstance(value, list):
+ transformed = []
+ changed = False
+ for item in value:
+ transformed_item, item_changed = _transform_nested_array_value(
+ item, field_path, transform_leaf)
+ transformed.append(transformed_item)
+ changed |= item_changed
+ return (transformed, True) if changed else (value, False)
+
+ if not isinstance(value, dict) or not field_path:
+ return value, False
+
+ field_name = field_path[0]
+ if len(field_path) == 1:
+ return transform_leaf(value, field_name)
+ if field_name not in value:
+ return value, False
+
+ transformed_child, changed = _transform_nested_array_value(
+ value[field_name], field_path[1:], transform_leaf)
+ if not changed:
+ return value, False
+
+ transformed = dict(value)
+ transformed[field_name] = transformed_child
+ return transformed, True
+
+
+def _encrypt_nested_array_value(Model, value, plain_field_path, encrypted_field_name,
+ field_schema, redo_all):
+ def encrypt_leaf(item, plain_field_name):
+ if plain_field_name not in item:
+ return item, False
+
+ plain_value = item[plain_field_name]
+ if not redo_all and encrypted_field_name in item:
+ encrypted_value = item[encrypted_field_name]
+ if encrypted_value is not None or plain_value is None:
+ return item, False
+
+ transformed = dict(item)
+ transformed[encrypted_field_name] = _encrypt_field_value(
+ Model, plain_value, field_schema)
+ return transformed, True
+
+ return _transform_nested_array_value(value, plain_field_path, encrypt_leaf)
+
+
+def _remove_nested_array_value(value, plain_field_path):
+ def remove_leaf(item, plain_field_name):
+ if plain_field_name not in item:
+ return item, False
+ transformed = dict(item)
+ del transformed[plain_field_name]
+ return transformed, True
+
+ return _transform_nested_array_value(value, plain_field_path, remove_leaf)
+
+
def _is_encrypted_list_schema(field_schema) -> bool:
return (
isinstance(field_schema, schema.Array)
@@ -93,6 +160,42 @@ def _encrypt_field_value(Model: type[MappedClass], value, field_schema):
return Model.encr(value)
+def _update_nested_array_records(raw_collection, plain_field_name, transform, limit):
+ field_path = _split_field_path(plain_field_name)
+ top_level_field = field_path[0]
+ nested_field_path = field_path[1:]
+ assert nested_field_path
+
+ query = {plain_field_name: {'$exists': True}}
+ projection = {'_id': 1, top_level_field: 1}
+ count = 0
+ last_id = None
+ while count < limit if limit else True:
+ chunk_query = (
+ {'$and': [query, {'_id': {'$gt': last_id}}]}
+ if last_id is not None else query
+ )
+ docs = list(raw_collection.find(chunk_query, projection).sort('_id', 1).limit(CHUNK_SIZE))
+ if not docs:
+ break
+
+ for doc in docs:
+ transformed, changed = transform(doc[top_level_field], nested_field_path)
+ if changed:
+ raw_collection.update_one(
+ {'_id': doc['_id']},
+ {'$set': {top_level_field: transformed}},
+ )
+ count += 1
+ if limit and count >= limit:
+ break
+
+ last_id = docs[-1]['_id']
+ print(f'Updated {count} nested-array records so far...')
+
+ return count
+
+
def main(class_name: str, plain_field_name: str,
*, remove_unencrypted: bool = False, redo_all: bool = False, limit: int | None = None):
"""
@@ -133,6 +236,43 @@ def main(class_name: str, plain_field_name: str,
# TODO: figure out how it works with inheritance
sess = session(Model)
+ m = mapper(Model)
+ raw_collection = sess.impl.db[m.collection.m.collection_name]
+
+ _, traverses_array = _schema_info_for_field_path(Model, plain_field_name)
+ if traverses_array:
+ plain_field_path = _split_field_path(plain_field_name)
+ encrypted_field_path = _split_field_path(encrypted_field_name)
+ assert plain_field_path[:-1] == encrypted_field_path[:-1]
+
+ encrypted_count = _update_nested_array_records(
+ raw_collection,
+ plain_field_name,
+ lambda value, nested_path: _encrypt_nested_array_value(
+ Model,
+ value,
+ nested_path,
+ encrypted_field_path[-1],
+ encr_schema,
+ redo_all,
+ ),
+ limit,
+ )
+ print(
+ f'Encrypted {encrypted_count} {class_name} records with '
+ f'nested-array {plain_field_name} values')
+
+ if remove_unencrypted:
+ removed_count = _update_nested_array_records(
+ raw_collection,
+ plain_field_name,
+ _remove_nested_array_value,
+ limit,
+ )
+ print(
+ f'Removed {removed_count} {class_name} unencrypted '
+ f'nested-array {plain_field_name} values')
+ return
# encrypt all records that need it. Even for --remove-unencrypted, to make sure everything's converted first
bulk_update_values = ["", None]
@@ -161,9 +301,6 @@ def main(class_name: str, plain_field_name: str,
if not redo_all:
q[encrypted_field_name] = None
- m = mapper(Model)
- raw_collection = sess.impl.db[m.collection.m.collection_name]
-
count = 0
projection = {'_id': 1, plain_field_name: 1}
last_id = None