You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by je...@apache.org on 2014/08/04 17:23:47 UTC
[1/5] git commit: [#7589] ticket:622 Make User indexable
Repository: allura
Updated Branches:
refs/heads/je/42cc_7589 [created] cfb0845c3
[#7589] ticket:622 Make User indexable
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/b6a6273e
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/b6a6273e
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/b6a6273e
Branch: refs/heads/je/42cc_7589
Commit: b6a6273e09b28870da10697a71a92e3745c6c228
Parents: 84b1a1a
Author: Igor Bondarenko <je...@gmail.com>
Authored: Mon Aug 4 14:16:46 2014 +0300
Committer: Igor Bondarenko <je...@gmail.com>
Committed: Mon Aug 4 14:16:46 2014 +0300
----------------------------------------------------------------------
Allura/allura/lib/plugin.py | 8 ++++++++
Allura/allura/model/auth.py | 38 ++++++++++++++++++++++++++++++++++++--
2 files changed, 44 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/b6a6273e/Allura/allura/lib/plugin.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/plugin.py b/Allura/allura/lib/plugin.py
index 6e75edd..f23b833 100644
--- a/Allura/allura/lib/plugin.py
+++ b/Allura/allura/lib/plugin.py
@@ -279,6 +279,10 @@ class AuthenticationProvider(object):
return True
return False
+ def index_user(self, user):
+ """Put here additional fields for user index in SOLR."""
+ return {}
+
class LocalAuthenticationProvider(AuthenticationProvider):
@@ -370,6 +374,10 @@ class LocalAuthenticationProvider(AuthenticationProvider):
d = datetime.utcfromtimestamp(calendar.timegm(d.utctimetuple()))
return d
+ def index_user(self, user):
+ fields = super(LocalAuthenticationProvider, self).index_user(user)
+ return dict(user_registration_date_dt=self.user_registration_date(user), **fields)
+
def ldap_conn(who=None, cred=None):
'''
http://git-wip-us.apache.org/repos/asf/allura/blob/b6a6273e/Allura/allura/model/auth.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/auth.py b/Allura/allura/model/auth.py
index c9ba690..fb70d87 100644
--- a/Allura/allura/model/auth.py
+++ b/Allura/allura/model/auth.py
@@ -46,6 +46,7 @@ import allura.tasks.mail_tasks
from allura.lib import helpers as h
from allura.lib import plugin
from allura.lib.decorators import memoize
+from allura.lib.search import SearchIndexable
from .session import main_orm_session, main_doc_session
from .session import project_orm_session
@@ -212,7 +213,7 @@ class FieldPropertyDisplayName(FieldProperty):
return display_name
-class User(MappedClass, ActivityNode, ActivityObject):
+class User(MappedClass, ActivityNode, ActivityObject, SearchIndexable):
SALT_LEN = 8
class __mongometa__:
@@ -221,6 +222,8 @@ class User(MappedClass, ActivityNode, ActivityObject):
indexes = ['tool_data.sfx.userid', 'tool_data.AuthPasswordReset.hash']
unique_indexes = ['username']
+ type_s = 'User'
+
_id = FieldProperty(S.ObjectId)
sfx_userid = FieldProperty(S.Deprecated)
username = FieldProperty(str)
@@ -229,7 +232,7 @@ class User(MappedClass, ActivityNode, ActivityObject):
last_password_updated = FieldProperty(datetime)
projects = FieldProperty(S.Deprecated)
# full mount point: prefs dict
- tool_preferences = FieldProperty({str: {str: None}})
+ tool_preferences = FieldProperty(S.Deprecated)
tool_data = FieldProperty({str: {str: None}}) # entry point: prefs dict
disabled = FieldProperty(bool, if_missing=False)
@@ -282,6 +285,37 @@ class User(MappedClass, ActivityNode, ActivityObject):
session_ip=str,
session_ua=str))
+ def index(self):
+ provider = plugin.AuthenticationProvider.get(None) # no need in request here
+ localization = '%s/%s' % (
+ self.get_pref('localization')['country'],
+ self.get_pref('localization')['city'])
+ fields = dict(
+ id=self.index_id(),
+ title='User %s' % self.username,
+ type_s=self.type_s,
+ username_s=self.username,
+ email_addresses=self.email_addresses,
+ last_password_updated_dt=self.last_password_updated,
+ disabled_b=self.disabled,
+ results_per_page_i=self.get_pref('results_per_page'),
+ email_address_s=self.get_pref('email_address'),
+ email_format_s=self.get_pref('email_format'),
+ disable_user_messages_b=self.get_pref('disable_user_messages'),
+ display_name_s=self.get_pref('display_name'),
+ sex_s=self.get_pref('sex'),
+ birthday_dt=self.get_pref('birthday'),
+ localization_s=localization,
+ timezone_s=self.get_pref('timezone'),
+ socialnetworks=self.get_pref('socialnetworks'),
+ telnumbers=self.get_pref('telnumbers'),
+ skypeaccount_s=self.get_pref('skypeaccount'),
+ webpages=self.get_pref('webpages'),
+ skills=self.get_skills(),
+ last_access=self.last_access,
+ )
+ return dict(provider.index_user(self), **fields)
+
def track_login(self, req):
user_ip = req.headers.get('X_FORWARDED_FOR', req.remote_addr)
user_agent = req.headers.get('User-Agent')
[3/5] git commit: [#7589] ticket:622 Fix field types/appearance in
the index
Posted by je...@apache.org.
[#7589] ticket:622 Fix field types/appearance in the index
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/a71f2762
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/a71f2762
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/a71f2762
Branch: refs/heads/je/42cc_7589
Commit: a71f2762b5663790191f204a070ccd8f081f0492
Parents: b600813
Author: Igor Bondarenko <je...@gmail.com>
Authored: Mon Aug 4 15:19:22 2014 +0300
Committer: Igor Bondarenko <je...@gmail.com>
Committed: Mon Aug 4 15:19:22 2014 +0300
----------------------------------------------------------------------
Allura/allura/model/auth.py | 21 ++++++++++++++-------
1 file changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/a71f2762/Allura/allura/model/auth.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/auth.py b/Allura/allura/model/auth.py
index fb70d87..78ac52c 100644
--- a/Allura/allura/model/auth.py
+++ b/Allura/allura/model/auth.py
@@ -290,12 +290,14 @@ class User(MappedClass, ActivityNode, ActivityObject, SearchIndexable):
localization = '%s/%s' % (
self.get_pref('localization')['country'],
self.get_pref('localization')['city'])
+ socialnetworks = ' '.join(['%s: %s' % (n['socialnetwork'], n['accounturl'])
+ for n in self.get_pref('socialnetworks')])
fields = dict(
id=self.index_id(),
title='User %s' % self.username,
type_s=self.type_s,
username_s=self.username,
- email_addresses=self.email_addresses,
+ email_addresses_t=' '.join(self.email_addresses),
last_password_updated_dt=self.last_password_updated,
disabled_b=self.disabled,
results_per_page_i=self.get_pref('results_per_page'),
@@ -304,15 +306,20 @@ class User(MappedClass, ActivityNode, ActivityObject, SearchIndexable):
disable_user_messages_b=self.get_pref('disable_user_messages'),
display_name_s=self.get_pref('display_name'),
sex_s=self.get_pref('sex'),
- birthday_dt=self.get_pref('birthday'),
+ birthdate_dt=self.get_pref('birthdate'),
localization_s=localization,
timezone_s=self.get_pref('timezone'),
- socialnetworks=self.get_pref('socialnetworks'),
- telnumbers=self.get_pref('telnumbers'),
+ socialnetworks_t=socialnetworks,
+ telnumbers_t=' '.join(self.get_pref('telnumbers')),
skypeaccount_s=self.get_pref('skypeaccount'),
- webpages=self.get_pref('webpages'),
- skills=self.get_skills(),
- last_access=self.last_access,
+ webpages_t=' '.join(self.get_pref('webpages')),
+ skills_t=' '.join([s['skill'].fullpath for s in self.get_skills()]),
+ last_access_login_date_dt=self.last_access['login_date'],
+ last_access_login_ip_s=self.last_access['login_ip'],
+ last_access_login_ua_t=self.last_access['login_ua'],
+ last_access_session_date_dt=self.last_access['session_date'],
+ last_access_session_ip_s=self.last_access['session_ip'],
+ last_access_session_ua_t=self.last_access['session_ua'],
)
return dict(provider.index_user(self), **fields)
[5/5] git commit: [#7589] ticket:622 Add command to reindex all users
Posted by je...@apache.org.
[#7589] ticket:622 Add command to reindex all users
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/cfb0845c
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/cfb0845c
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/cfb0845c
Branch: refs/heads/je/42cc_7589
Commit: cfb0845c3010901df81b1ac2903b277ff3923585
Parents: 5585e02
Author: Igor Bondarenko <je...@gmail.com>
Authored: Mon Aug 4 17:16:51 2014 +0300
Committer: Igor Bondarenko <je...@gmail.com>
Committed: Mon Aug 4 17:16:51 2014 +0300
----------------------------------------------------------------------
Allura/allura/scripts/reindex_users.py | 89 +++++++++++++++++++++++++++++
1 file changed, 89 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/cfb0845c/Allura/allura/scripts/reindex_users.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/reindex_users.py b/Allura/allura/scripts/reindex_users.py
new file mode 100644
index 0000000..8a0762b
--- /dev/null
+++ b/Allura/allura/scripts/reindex_users.py
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import logging
+
+from pymongo.errors import InvalidDocument
+
+from allura.scripts import ScriptTask
+from allura import model as M
+from allura.tasks.index_tasks import add_users
+from allura.lib.utils import chunked_find, chunked_list
+from allura.lib.exceptions import CompoundError
+
+
+log = logging.getLogger(__name__)
+
+
+class ReindexUsers(ScriptTask):
+
+ @classmethod
+ def execute(cls, options):
+ for chunk in chunked_find(M.User, {}):
+ user_ids = []
+ for u in chunk:
+ log.info('Reindex user %s', u.username)
+ if options.dry_run:
+ continue
+ user_ids.append(u._id)
+ try:
+ for chunk in chunked_list(user_ids, options.max_chunk):
+ if options.tasks:
+ cls._post_add_users(chunk)
+ else:
+ add_users(chunk)
+ except CompoundError, err:
+ log.exception('Error indexing users:\n%r', err)
+ log.error('%s', err.format_error())
+ M.main_orm_session.flush()
+ M.main_orm_session.clear()
+ log.info('Reindex %s', 'queued' if options.tasks else 'done')
+
+ @classmethod
+ def _post_add_users(cls, chunk):
+ """
+ Post task, recursively splitting and re-posting if the resulting
+ mongo document is too large.
+ """
+ try:
+ add_users.post(chunk)
+ except InvalidDocument as e:
+ # there are many types of InvalidDocument, only recurse if its
+ # expected to help
+ if e.args[0].startswith('BSON document too large'):
+ cls._post_add_users(chunk[:len(chunk) // 2])
+ cls._post_add_users(chunk[len(chunk) // 2:])
+ else:
+ raise
+
+ @classmethod
+ def parser(cls):
+ parser = argparse.ArgumentParser(description='Reindex all users')
+ parser.add_argument('--dry-run', action='store_true', dest='dry_run',
+ default=False, help='Log names of projects that would be reindexed, '
+ 'but do not perform the actual reindex.')
+ parser.add_argument('--tasks', action='store_true', dest='tasks',
+ help='Run each individual index operation as a background task.')
+ parser.add_argument(
+ '--max-chunk', dest='max_chunk', type=int, default=100 * 1000,
+ help='Max number of artifacts to index in one Solr update command')
+ return parser
+
+
+if __name__ == '__main__':
+ ReindexUsers.main()
[2/5] git commit: [#7589] ticket:622 Tasks for indexing users
Posted by je...@apache.org.
[#7589] ticket:622 Tasks for indexing users
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/b6008138
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/b6008138
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/b6008138
Branch: refs/heads/je/42cc_7589
Commit: b600813858fa61ad8b4d5b6ccdab0c6507dbb4bb
Parents: b6a6273
Author: Igor Bondarenko <je...@gmail.com>
Authored: Mon Aug 4 14:50:18 2014 +0300
Committer: Igor Bondarenko <je...@gmail.com>
Committed: Mon Aug 4 14:50:18 2014 +0300
----------------------------------------------------------------------
Allura/allura/model/session.py | 4 +++-
Allura/allura/tasks/index_tasks.py | 12 ++++++++++++
2 files changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/b6008138/Allura/allura/model/session.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/session.py b/Allura/allura/model/session.py
index f364aeb..e2eca8f 100644
--- a/Allura/allura/model/session.py
+++ b/Allura/allura/model/session.py
@@ -62,7 +62,9 @@ class IndexerSessionExtension(ManagedSessionExtension):
TASKS = {
'allura.model.project.Project': {'add': index_tasks.add_projects,
- 'del': index_tasks.del_projects}
+ 'del': index_tasks.del_projects},
+ 'allura.model.auth.User': {'add': index_tasks.add_users,
+ 'del': index_tasks.del_users},
}
def _objects_by_types(self, obj_list):
http://git-wip-us.apache.org/repos/asf/allura/blob/b6008138/Allura/allura/tasks/index_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/index_tasks.py b/Allura/allura/tasks/index_tasks.py
index 3f02cef..c9e853d 100644
--- a/Allura/allura/tasks/index_tasks.py
+++ b/Allura/allura/tasks/index_tasks.py
@@ -57,6 +57,18 @@ def del_projects(project_solr_ids):
@task
+def add_users(user_ids):
+ from allura.model import User
+ users = User.query.find(dict(_id={'$in': user_ids})).all()
+ __add_objects(users)
+
+
+@task
+def del_users(user_solr_ids):
+ __del_objects(user_solr_ids)
+
+
+@task
def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
'''
Add the referenced artifacts to SOLR and shortlinks.
[4/5] git commit: [#7589] ticket:622 Test for User.index()
Posted by je...@apache.org.
[#7589] ticket:622 Test for User.index()
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/5585e026
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/5585e026
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/5585e026
Branch: refs/heads/je/42cc_7589
Commit: 5585e02692ecc87bb7350cc042f6680bfb714d2a
Parents: a71f276
Author: Igor Bondarenko <je...@gmail.com>
Authored: Mon Aug 4 15:44:08 2014 +0300
Committer: Igor Bondarenko <je...@gmail.com>
Committed: Mon Aug 4 15:44:08 2014 +0300
----------------------------------------------------------------------
Allura/allura/tests/model/test_auth.py | 36 +++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/5585e026/Allura/allura/tests/model/test_auth.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/model/test_auth.py b/Allura/allura/tests/model/test_auth.py
index f3869ee..65321e9 100644
--- a/Allura/allura/tests/model/test_auth.py
+++ b/Allura/allura/tests/model/test_auth.py
@@ -275,3 +275,39 @@ def test_user_track_active():
c.user = M.User.by_username(c.user.username)
assert_equal(c.user.last_access['session_ip'], 'new addr')
assert_equal(c.user.last_access['session_ua'], 'new browser')
+
+
+@with_setup(setUp)
+def test_user_index():
+ c.user.email_addresses = ['email1', 'email2']
+ c.user.set_pref('email_address', 'email2')
+ idx = c.user.index()
+ assert_equal(idx['id'], c.user.index_id())
+ assert_equal(idx['title'], 'User test-admin')
+ assert_equal(idx['type_s'], 'User')
+ assert_equal(idx['username_s'], 'test-admin')
+ assert_equal(idx['email_addresses_t'], 'email1 email2')
+ assert_equal(idx['email_address_s'], 'email2')
+ assert_in('last_password_updated_dt', idx)
+ assert_equal(idx['disabled_b'], False)
+ assert_in('results_per_page_i', idx)
+ assert_in('email_format_s', idx)
+ assert_in('disable_user_messages_b', idx)
+ assert_equal(idx['display_name_s'], 'Test Admin')
+ assert_equal(idx['sex_s'], 'Unknown')
+ assert_in('birthdate_dt', idx)
+ assert_in('localization_s', idx)
+ assert_in('timezone_s', idx)
+ assert_in('socialnetworks_t', idx)
+ assert_in('telnumbers_t', idx)
+ assert_in('skypeaccount_s', idx)
+ assert_in('webpages_t', idx)
+ assert_in('skills_t', idx)
+ assert_in('last_access_login_date_dt', idx)
+ assert_in('last_access_login_ip_s', idx)
+ assert_in('last_access_login_ua_t', idx)
+ assert_in('last_access_session_date_dt', idx)
+ assert_in('last_access_session_ip_s', idx)
+ assert_in('last_access_session_ua_t', idx)
+ # provided bby auth provider
+ assert_in('user_registration_date_dt', idx)