You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2026/08/13 17:29:49 UTC
(allura) 02/02: [#8614] Add support for nested lists and tests to the field conversion script
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
commit ee99615a158a4e91340d9f351f77832b1fe93d92
Author: Carlos Cruz <ca...@slashdotmedia.com>
AuthorDate: Mon Jul 20 14:28:13 2026 -0600
[#8614] Add support for nested lists and tests to the field conversion script
---
Allura/allura/controllers/auth.py | 10 +-
Allura/allura/tests/functional/test_auth.py | 14 +-
.../tests/scripts/test_convert_encrypted_field.py | 111 +++++++++++++++
scripts/convert_encrypted_field.py | 153 +++++++++++++++++++--
4 files changed, 274 insertions(+), 14 deletions(-)
diff --git a/Allura/allura/controllers/auth.py b/Allura/allura/controllers/auth.py
index e62012338..fd851fe99 100644
--- a/Allura/allura/controllers/auth.py
+++ b/Allura/allura/controllers/auth.py
@@ -1187,8 +1187,14 @@ def add_social_network(self, **kw):
@validate(F.remove_socialnetwork_form, error_handler=index)
def remove_social_network(self, **kw):
require_authenticated()
- c.user.remove_multivalue_pref('socialnetworks',
- {'socialnetwork': kw['socialnetwork'], 'accounturl': kw['account']})
+ target = {'socialnetwork': kw['socialnetwork'], 'accounturl': kw['account']}
+ stored = next(
+ (account for account in c.user.get_pref('socialnetworks')
+ if account.socialnetwork == kw['socialnetwork']
+ and account.accounturl == kw['account']),
+ target,
+ )
+ c.user.remove_multivalue_pref('socialnetworks', stored)
flash('Your personal contacts were successfully updated!')
redirect('.')
diff --git a/Allura/allura/tests/functional/test_auth.py b/Allura/allura/tests/functional/test_auth.py
index 85825af98..a389a18d7 100644
--- a/Allura/allura/tests/functional/test_auth.py
+++ b/Allura/allura/tests/functional/test_auth.py
@@ -1459,6 +1459,12 @@ def test_contacts_not_allowed(self):
@td.with_user_project('test-admin')
def test_contacts(self):
+ def socialnetwork_data(user):
+ return [
+ {'socialnetwork': social.socialnetwork, 'accounturl': social.accounturl}
+ for social in user.socialnetworks
+ ]
+
# Get page
self.app.get('/auth/user_info/contacts/')
# Add social network account
@@ -1486,7 +1492,7 @@ def test_contacts(self):
assert len(user.socialnetworks) == 2
expected = [{'socialnetwork': socialnetwork, 'accounturl': accounturl},
{'socialnetwork': socialnetwork2, 'accounturl': accounturl2}]
- assert all([social in expected for social in user.socialnetworks])
+ assert all(social in expected for social in socialnetwork_data(user))
socialnetwork3 = 'Mastodon'
accounturl3 = '@username@server.social'
@@ -1508,7 +1514,7 @@ def test_contacts(self):
assert len(user.socialnetworks) == 2
expected = [{'socialnetwork': socialnetwork2, 'accounturl': accounturl2},
{'socialnetwork': socialnetwork3, 'accounturl': accounturl3}]
- assert all([social in expected for social in user.socialnetworks])
+ assert all(social in expected for social in socialnetwork_data(user))
# Add empty social network account
self.app.post('/auth/user_info/contacts/add_social_network',
@@ -1519,7 +1525,7 @@ def test_contacts(self):
assert len(user.socialnetworks) == 2
expected = [{'socialnetwork': socialnetwork2, 'accounturl': accounturl2},
{'socialnetwork': socialnetwork3, 'accounturl': accounturl3}]
- assert all([social in expected for social in user.socialnetworks])
+ assert all(social in expected for social in socialnetwork_data(user))
# Add invalid social network account
self.app.post('/auth/user_info/contacts/add_social_network',
@@ -1530,7 +1536,7 @@ def test_contacts(self):
assert len(user.socialnetworks) == 2
expected = [{'socialnetwork': socialnetwork2, 'accounturl': accounturl2},
{'socialnetwork': socialnetwork3, 'accounturl': accounturl3}]
- assert all([social in expected for social in user.socialnetworks])
+ assert all(social in expected for social in socialnetwork_data(user))
# Add telephone number
telnumber = '+3902123456'
diff --git a/Allura/allura/tests/scripts/test_convert_encrypted_field.py b/Allura/allura/tests/scripts/test_convert_encrypted_field.py
new file mode 100644
index 000000000..c218db8de
--- /dev/null
+++ b/Allura/allura/tests/scripts/test_convert_encrypted_field.py
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from ming import schema as S
+from ming.odm import FieldProperty
+
+from scripts.convert_encrypted_field import (
+ _encrypt_nested_array_value,
+ _remove_nested_array_value,
+ _schema_info_for_field_path,
+)
+
+
+class TestModel:
+ socialnetworks = FieldProperty([dict(
+ socialnetwork=str,
+ accounturl=str,
+ accounturl_encrypted=S.Binary,
+ )])
+
+ @classmethod
+ def encr(cls, value):
+ return None if value is None else f'encrypted:{value}'.encode()
+
+
+def test_schema_info_supports_nested_array_path():
+ field_schema, traverses_array = _schema_info_for_field_path(
+ TestModel, 'socialnetworks.accounturl_encrypted')
+
+ assert isinstance(field_schema, S.Binary)
+ assert traverses_array is True
+
+
+def test_encrypt_nested_array_value_updates_every_item():
+ socialnetworks = [
+ {'socialnetwork': 'Mastodon', 'accounturl': 'https://example.com/@user'},
+ {'socialnetwork': 'LinkedIn', 'accounturl': 'https://example.com/in/user'},
+ {'socialnetwork': 'Other', 'accounturl': None},
+ ]
+
+ encrypted, changed = _encrypt_nested_array_value(
+ TestModel,
+ socialnetworks,
+ ['accounturl'],
+ 'accounturl_encrypted',
+ S.Binary(),
+ redo_all=False,
+ )
+
+ assert changed is True
+ assert encrypted == [
+ {
+ 'socialnetwork': 'Mastodon',
+ 'accounturl': 'https://example.com/@user',
+ 'accounturl_encrypted': b'encrypted:https://example.com/@user',
+ },
+ {
+ 'socialnetwork': 'LinkedIn',
+ 'accounturl': 'https://example.com/in/user',
+ 'accounturl_encrypted': b'encrypted:https://example.com/in/user',
+ },
+ {
+ 'socialnetwork': 'Other',
+ 'accounturl': None,
+ 'accounturl_encrypted': None,
+ },
+ ]
+
+
+def test_remove_nested_array_value_preserves_encrypted_and_other_fields():
+ socialnetworks = [
+ {
+ 'socialnetwork': 'Mastodon',
+ 'accounturl': 'https://example.com/@user',
+ 'accounturl_encrypted': b'encrypted:https://example.com/@user',
+ },
+ {
+ 'socialnetwork': 'Other',
+ 'accounturl': None,
+ 'accounturl_encrypted': None,
+ },
+ ]
+
+ encrypted_only, changed = _remove_nested_array_value(
+ socialnetworks, ['accounturl'])
+
+ assert changed is True
+ assert encrypted_only == [
+ {
+ 'socialnetwork': 'Mastodon',
+ 'accounturl_encrypted': b'encrypted:https://example.com/@user',
+ },
+ {
+ 'socialnetwork': 'Other',
+ 'accounturl_encrypted': None,
+ },
+ ]
diff --git a/scripts/convert_encrypted_field.py b/scripts/convert_encrypted_field.py
index e99c25a77..379b10dbd 100644
--- a/scripts/convert_encrypted_field.py
+++ b/scripts/convert_encrypted_field.py
@@ -45,7 +45,7 @@ def _split_field_path(field_name: str) -> list[str]:
return field_path
-def _schema_for_field_path(Model: type[MappedClass], field_name: str):
+def _schema_info_for_field_path(Model: type[MappedClass], field_name: str):
field_path = _split_field_path(field_name)
top_level_name = field_path[0]
try:
@@ -56,11 +56,12 @@ def _schema_for_field_path(Model: type[MappedClass], field_name: str):
assert isinstance(top_level_prop, FieldProperty)
current_schema = top_level_prop.field.schema
+ traverses_array = False
for i, path_part in enumerate(field_path[1:], start=1):
traversed = '.'.join(field_path[:i])
- if isinstance(current_schema, schema.Array):
- raise AssertionError(
- f'Nested array paths are not supported: {field_name!r} (array at {traversed!r})')
+ while isinstance(current_schema, schema.Array):
+ traverses_array = True
+ current_schema = current_schema.field_type
if not isinstance(current_schema, schema.Object):
raise AssertionError(
f'Invalid nested field path {field_name!r}; {traversed!r} is not an object field')
@@ -70,7 +71,12 @@ def _schema_for_field_path(Model: type[MappedClass], field_name: str):
current_schema = current_schema.fields[path_part]
- return current_schema
+ return current_schema, traverses_array
+
+
+def _schema_for_field_path(Model: type[MappedClass], field_name: str):
+ field_schema, _ = _schema_info_for_field_path(Model, field_name)
+ return field_schema
def _get_nested_value(rec: dict, field_name: str):
@@ -80,6 +86,67 @@ def _get_nested_value(rec: dict, field_name: str):
return value
+def _transform_nested_array_value(value, field_path, transform_leaf):
+ if isinstance(value, list):
+ transformed = []
+ changed = False
+ for item in value:
+ transformed_item, item_changed = _transform_nested_array_value(
+ item, field_path, transform_leaf)
+ transformed.append(transformed_item)
+ changed |= item_changed
+ return (transformed, True) if changed else (value, False)
+
+ if not isinstance(value, dict) or not field_path:
+ return value, False
+
+ field_name = field_path[0]
+ if len(field_path) == 1:
+ return transform_leaf(value, field_name)
+ if field_name not in value:
+ return value, False
+
+ transformed_child, changed = _transform_nested_array_value(
+ value[field_name], field_path[1:], transform_leaf)
+ if not changed:
+ return value, False
+
+ transformed = dict(value)
+ transformed[field_name] = transformed_child
+ return transformed, True
+
+
+def _encrypt_nested_array_value(Model, value, plain_field_path, encrypted_field_name,
+ field_schema, redo_all):
+ def encrypt_leaf(item, plain_field_name):
+ if plain_field_name not in item:
+ return item, False
+
+ plain_value = item[plain_field_name]
+ if not redo_all and encrypted_field_name in item:
+ encrypted_value = item[encrypted_field_name]
+ if encrypted_value is not None or plain_value is None:
+ return item, False
+
+ transformed = dict(item)
+ transformed[encrypted_field_name] = _encrypt_field_value(
+ Model, plain_value, field_schema)
+ return transformed, True
+
+ return _transform_nested_array_value(value, plain_field_path, encrypt_leaf)
+
+
+def _remove_nested_array_value(value, plain_field_path):
+ def remove_leaf(item, plain_field_name):
+ if plain_field_name not in item:
+ return item, False
+ transformed = dict(item)
+ del transformed[plain_field_name]
+ return transformed, True
+
+ return _transform_nested_array_value(value, plain_field_path, remove_leaf)
+
+
def _is_encrypted_list_schema(field_schema) -> bool:
return (
isinstance(field_schema, schema.Array)
@@ -93,6 +160,42 @@ def _encrypt_field_value(Model: type[MappedClass], value, field_schema):
return Model.encr(value)
+def _update_nested_array_records(raw_collection, plain_field_name, transform, limit):
+ field_path = _split_field_path(plain_field_name)
+ top_level_field = field_path[0]
+ nested_field_path = field_path[1:]
+ assert nested_field_path
+
+ query = {plain_field_name: {'$exists': True}}
+ projection = {'_id': 1, top_level_field: 1}
+ count = 0
+ last_id = None
+ while count < limit if limit else True:
+ chunk_query = (
+ {'$and': [query, {'_id': {'$gt': last_id}}]}
+ if last_id is not None else query
+ )
+ docs = list(raw_collection.find(chunk_query, projection).sort('_id', 1).limit(CHUNK_SIZE))
+ if not docs:
+ break
+
+ for doc in docs:
+ transformed, changed = transform(doc[top_level_field], nested_field_path)
+ if changed:
+ raw_collection.update_one(
+ {'_id': doc['_id']},
+ {'$set': {top_level_field: transformed}},
+ )
+ count += 1
+ if limit and count >= limit:
+ break
+
+ last_id = docs[-1]['_id']
+ print(f'Updated {count} nested-array records so far...')
+
+ return count
+
+
def main(class_name: str, plain_field_name: str,
*, remove_unencrypted: bool = False, redo_all: bool = False, limit: int | None = None):
"""
@@ -133,6 +236,43 @@ def main(class_name: str, plain_field_name: str,
# TODO: figure out how it works with inheritance
sess = session(Model)
+ m = mapper(Model)
+ raw_collection = sess.impl.db[m.collection.m.collection_name]
+
+ _, traverses_array = _schema_info_for_field_path(Model, plain_field_name)
+ if traverses_array:
+ plain_field_path = _split_field_path(plain_field_name)
+ encrypted_field_path = _split_field_path(encrypted_field_name)
+ assert plain_field_path[:-1] == encrypted_field_path[:-1]
+
+ encrypted_count = _update_nested_array_records(
+ raw_collection,
+ plain_field_name,
+ lambda value, nested_path: _encrypt_nested_array_value(
+ Model,
+ value,
+ nested_path,
+ encrypted_field_path[-1],
+ encr_schema,
+ redo_all,
+ ),
+ limit,
+ )
+ print(
+ f'Encrypted {encrypted_count} {class_name} records with '
+ f'nested-array {plain_field_name} values')
+
+ if remove_unencrypted:
+ removed_count = _update_nested_array_records(
+ raw_collection,
+ plain_field_name,
+ _remove_nested_array_value,
+ limit,
+ )
+ print(
+ f'Removed {removed_count} {class_name} unencrypted '
+ f'nested-array {plain_field_name} values')
+ return
# encrypt all records that need it. Even for --remove-unencrypted, to make sure everything's converted first
bulk_update_values = ["", None]
@@ -161,9 +301,6 @@ def main(class_name: str, plain_field_name: str,
if not redo_all:
q[encrypted_field_name] = None
- m = mapper(Model)
- raw_collection = sess.impl.db[m.collection.m.collection_name]
-
count = 0
projection = {'_id': 1, plain_field_name: 1}
last_id = None