You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by je...@apache.org on 2014/09/18 09:30:03 UTC
[28/43] git commit: [#7589] ticket:622 Add command to reindex all
users
[#7589] ticket:622 Add command to reindex all users
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/95904794
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/95904794
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/95904794
Branch: refs/heads/je/42cc_7656
Commit: 9590479434670c2844a14469c714be4c747a623e
Parents: 596ec0f
Author: Igor Bondarenko <je...@gmail.com>
Authored: Mon Aug 4 17:16:51 2014 +0300
Committer: Igor Bondarenko <je...@gmail.com>
Committed: Thu Sep 18 09:33:57 2014 +0300
----------------------------------------------------------------------
Allura/allura/scripts/reindex_users.py | 89 +++++++++++++++++++++++++++++
1 file changed, 89 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/95904794/Allura/allura/scripts/reindex_users.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/reindex_users.py b/Allura/allura/scripts/reindex_users.py
new file mode 100644
index 0000000..8a0762b
--- /dev/null
+++ b/Allura/allura/scripts/reindex_users.py
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import logging
+
+from pymongo.errors import InvalidDocument
+
+from allura.scripts import ScriptTask
+from allura import model as M
+from allura.tasks.index_tasks import add_users
+from allura.lib.utils import chunked_find, chunked_list
+from allura.lib.exceptions import CompoundError
+
+
+log = logging.getLogger(__name__)
+
+
+class ReindexUsers(ScriptTask):
+
+ @classmethod
+ def execute(cls, options):
+ for chunk in chunked_find(M.User, {}):
+ user_ids = []
+ for u in chunk:
+ log.info('Reindex user %s', u.username)
+ if options.dry_run:
+ continue
+ user_ids.append(u._id)
+ try:
+ for chunk in chunked_list(user_ids, options.max_chunk):
+ if options.tasks:
+ cls._post_add_users(chunk)
+ else:
+ add_users(chunk)
+ except CompoundError, err:
+ log.exception('Error indexing users:\n%r', err)
+ log.error('%s', err.format_error())
+ M.main_orm_session.flush()
+ M.main_orm_session.clear()
+ log.info('Reindex %s', 'queued' if options.tasks else 'done')
+
+ @classmethod
+ def _post_add_users(cls, chunk):
+ """
+ Post task, recursively splitting and re-posting if the resulting
+ mongo document is too large.
+ """
+ try:
+ add_users.post(chunk)
+ except InvalidDocument as e:
+ # there are many types of InvalidDocument, only recurse if its
+ # expected to help
+ if e.args[0].startswith('BSON document too large'):
+ cls._post_add_users(chunk[:len(chunk) // 2])
+ cls._post_add_users(chunk[len(chunk) // 2:])
+ else:
+ raise
+
+ @classmethod
+ def parser(cls):
+ parser = argparse.ArgumentParser(description='Reindex all users')
+ parser.add_argument('--dry-run', action='store_true', dest='dry_run',
+ default=False, help='Log names of projects that would be reindexed, '
+ 'but do not perform the actual reindex.')
+ parser.add_argument('--tasks', action='store_true', dest='tasks',
+ help='Run each individual index operation as a background task.')
+ parser.add_argument(
+ '--max-chunk', dest='max_chunk', type=int, default=100 * 1000,
+ help='Max number of artifacts to index in one Solr update command')
+ return parser
+
+
+if __name__ == '__main__':
+ ReindexUsers.main()