You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by je...@apache.org on 2014/09/22 11:23:01 UTC
[1/2] git commit: [#7644] ticket:658 Pre-populate roles cache to
reduce # of mongo queries
Repository: allura
Updated Branches:
refs/heads/je/42cc_7644 [created] fc8c22ffb
[#7644] ticket:658 Pre-populate roles cache to reduce # of mongo queries
Also fix users_with_named_role so that it would actually use cache.
Template for `/nf/new_projects/` displays admins for each project (and
there can be a lot of projects) and it generates a lot of mongo queries to
lookup user roles. Pre-populating the cache once helps to reduce query
count more than twice, and improves overall page rendering time.
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/fc8c22ff
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/fc8c22ff
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/fc8c22ff
Branch: refs/heads/je/42cc_7644
Commit: fc8c22ffbdeb3a584e9b1e301cd1a16497b4cae9
Parents: 5b61991
Author: Igor Bondarenko <je...@gmail.com>
Authored: Fri Sep 19 12:26:33 2014 +0300
Committer: Igor Bondarenko <je...@gmail.com>
Committed: Mon Sep 22 10:37:02 2014 +0300
----------------------------------------------------------------------
Allura/allura/controllers/site_admin.py | 7 +++++--
Allura/allura/lib/security.py | 2 +-
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/fc8c22ff/Allura/allura/controllers/site_admin.py
----------------------------------------------------------------------
diff --git a/Allura/allura/controllers/site_admin.py b/Allura/allura/controllers/site_admin.py
index 755c405..8c56f9f 100644
--- a/Allura/allura/controllers/site_admin.py
+++ b/Allura/allura/controllers/site_admin.py
@@ -36,7 +36,7 @@ from allura.lib import validators as v
from allura.lib.decorators import require_post
from allura.lib.plugin import SiteAdminExtension, ProjectRegistrationProvider, AuthenticationProvider
from allura.lib import search
-from allura.lib.security import require_access
+from allura.lib.security import require_access, Credentials
from allura.lib.widgets import form_fields as ffw
from allura.ext.admin.widgets import AuditLog
from allura.lib.widgets import forms
@@ -184,7 +184,10 @@ class SiteAdminController(object):
'neighborhood_id': {'$ne': nb._id},
'deleted': False,
'_id': {'$lt': start, '$gt': end},
- }).sort('_id', -1))
+ }).sort('_id', -1)).all()
+ # pre-populate roles cache, so we won't query mongo for roles for every project
+ # when getting admins with p.admins() in a template
+ Credentials.get().load_project_roles(*[p._id for p in projects])
step = start_dt - end_dt
params = request.params.copy()
params['start-dt'] = (start_dt + step).strftime('%Y/%m/%d %H:%M:%S')
http://git-wip-us.apache.org/repos/asf/allura/blob/fc8c22ff/Allura/allura/lib/security.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/security.py b/Allura/allura/lib/security.py
index c966e34..ad7fd2a 100644
--- a/Allura/allura/lib/security.py
+++ b/Allura/allura/lib/security.py
@@ -147,7 +147,7 @@ class Credentials(object):
def users_with_named_role(self, project_id, name):
""" returns in sorted order """
- role = RoleCache(self, self.project_role.find({'project_id': project_id, 'name': name}))
+ role = RoleCache(self, [r for r in self.project_roles(project_id) if r['name'] == name])
return sorted(role.users_that_reach, key=lambda u: u.username)
def userids_with_named_role(self, project_id, name):
[2/2] git commit: [#7644] ticket:658 Script to generate a lot of new
projects
Posted by je...@apache.org.
[#7644] ticket:658 Script to generate a lot of new projects
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/5b619914
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/5b619914
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/5b619914
Branch: refs/heads/je/42cc_7644
Commit: 5b619914878dbfb79c33262795a07d6bb308fa35
Parents: c972fc0
Author: Igor Bondarenko <je...@gmail.com>
Authored: Fri Sep 19 12:20:49 2014 +0300
Committer: Igor Bondarenko <je...@gmail.com>
Committed: Mon Sep 22 10:37:02 2014 +0300
----------------------------------------------------------------------
scripts/perf/generate-projects.py | 40 ++++++++++++++++++++++++++++++++++
1 file changed, 40 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/5b619914/scripts/perf/generate-projects.py
----------------------------------------------------------------------
diff --git a/scripts/perf/generate-projects.py b/scripts/perf/generate-projects.py
new file mode 100644
index 0000000..0374969
--- /dev/null
+++ b/scripts/perf/generate-projects.py
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import re
+from ming.odm import ThreadLocalORMSession
+from allura import model as M
+
+def main(start, cnt):
+ n = M.Neighborhood.query.get(url_prefix='/p/')
+ admin = M.User.by_username('admin1')
+ #M.Project.query.remove({'shortname': re.compile('gen-proj-.*')})
+ #ThreadLocalORMSession.flush_all()
+ for i in range(start, cnt):
+ name = 'gen-proj-{}'.format(i)
+ project = n.register_project(name, admin)
+ if (i-start) > 0 and (i-start) % 100 == 0:
+ print 'Created {} projects'.format(i-start)
+ print 'Flushing...'
+ ThreadLocalORMSession.flush_all()
+ print 'Done'
+
+if __name__ == '__main__':
+ import sys
+ start = int(sys.argv[1])
+ cnt = int(sys.argv[2])
+ main(start, cnt)