You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by he...@apache.org on 2015/05/29 22:40:38 UTC
[16/45] allura git commit: [#7878] Used 2to3 to see what issues would
come up
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/project-import.py
----------------------------------------------------------------------
diff --git a/scripts/project-import.py b/scripts/project-import.py
index b42a6fd..ecde35c 100644
--- a/scripts/project-import.py
+++ b/scripts/project-import.py
@@ -1,3 +1,7 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -229,7 +233,7 @@ def create_project(p, nbhd, user, options):
p.admin,
project_name=p.name.name,
private_project=p.private)
- except Exception, e:
+ except Exception as e:
log.exception('[%s] %s' % (worker_name, str(e)))
return 0
else:
@@ -325,7 +329,7 @@ def main(options):
for j in jobs:
j.join()
- if j.exitcode <> 0:
+ if j.exitcode != 0:
return j.exitcode
return 0
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/publicize-neighborhood.py
----------------------------------------------------------------------
diff --git a/scripts/publicize-neighborhood.py b/scripts/publicize-neighborhood.py
index c32d384..1a62aa1 100644
--- a/scripts/publicize-neighborhood.py
+++ b/scripts/publicize-neighborhood.py
@@ -1,3 +1,7 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/recover-user-databases.py
----------------------------------------------------------------------
diff --git a/scripts/recover-user-databases.py b/scripts/recover-user-databases.py
index 5abf6f3..f3d61c6 100644
--- a/scripts/recover-user-databases.py
+++ b/scripts/recover-user-databases.py
@@ -1,3 +1,7 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/refresh_last_commits.py
----------------------------------------------------------------------
diff --git a/scripts/refresh_last_commits.py b/scripts/refresh_last_commits.py
new file mode 100644
index 0000000..d1b5632
--- /dev/null
+++ b/scripts/refresh_last_commits.py
@@ -0,0 +1,193 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import logging
+from datetime import datetime
+from contextlib import contextmanager
+
+import faulthandler
+from pylons import tmpl_context as c
+from ming.orm import ThreadLocalORMSession, session
+
+from allura import model as M
+from allura.lib.utils import chunked_find
+from allura.tasks.repo_tasks import refresh
+from allura.scripts import ScriptTask
+
+log = logging.getLogger(__name__)
+
+
+class RefreshLastCommits(ScriptTask):
+
+ @classmethod
+ def parser(cls):
+ def _repo_type_list(s):
+ repo_types = []
+ for repo_type in s.split(','):
+ repo_type = repo_type.strip()
+ if repo_type not in ['git', 'hg']:
+ raise argparse.ArgumentTypeError(
+ '{0} is not a valid repo type.'.format(repo_type))
+ repo_types.append(repo_type)
+ return repo_types
+ parser = argparse.ArgumentParser(description='Using existing commit data, '
+ 'refresh the last commit metadata in MongoDB. Run for all repos (no args), '
+ 'or restrict by neighborhood, project, or code tool mount point.')
+ parser.add_argument('--nbhd', action='store', default='', dest='nbhd',
+ help='Restrict update to a particular neighborhood, e.g. /p/.')
+ parser.add_argument(
+ '--project', action='store', default='', dest='project',
+ help='Restrict update to a particular project. To specify a '
+ 'subproject, use a slash: project/subproject.')
+ parser.add_argument('--project-regex', action='store', default='',
+ dest='project_regex',
+ help='Restrict update to projects for which the shortname matches '
+ 'the provided regex.')
+ parser.add_argument(
+ '--repo-types', action='store', type=_repo_type_list,
+ default=['git', 'hg'], dest='repo_types',
+ help='Only refresh last commits for repos of the given type(s). Defaults to: '
+ 'git,hg. Example: --repo-types=git')
+ parser.add_argument('--mount-point', default='', dest='mount_point',
+ help='Restrict update to repos at the given tool mount point. ')
+ parser.add_argument('--clean', action='store_true', dest='clean',
+ default=False, help='Remove last commit mongo docs for '
+ 'project(s) being refreshed before doing the refresh.')
+ parser.add_argument('--dry-run', action='store_true', dest='dry_run',
+ default=False, help='Log names of projects that would have their ')
+ parser.add_argument('--limit', action='store', type=int, dest='limit',
+ default=False, help='Limit of how many commits to process')
+ return parser
+
+ @classmethod
+ def execute(cls, options):
+ q_project = {}
+ if options.nbhd:
+ nbhd = M.Neighborhood.query.get(url_prefix=options.nbhd)
+ if not nbhd:
+ return "Invalid neighborhood url prefix."
+ q_project['neighborhood_id'] = nbhd._id
+ if options.project:
+ q_project['shortname'] = options.project
+ elif options.project_regex:
+ q_project['shortname'] = {'$regex': options.project_regex}
+
+ log.info('Refreshing last commit data')
+
+ for chunk in chunked_find(M.Project, q_project):
+ for p in chunk:
+ log.info("Refreshing last commit data for project '%s'." %
+ p.shortname)
+ if options.dry_run:
+ continue
+ c.project = p
+ if options.mount_point:
+ mount_points = [options.mount_point]
+ else:
+ mount_points = [ac.options.mount_point for ac in
+ M.AppConfig.query.find(dict(project_id=p._id))]
+ for app in (p.app_instance(mp) for mp in mount_points):
+ c.app = app
+ if not hasattr(app, 'repo'):
+ continue
+ if c.app.repo.tool.lower() not in options.repo_types:
+ log.info("Skipping %r: wrong type (%s)", c.app.repo,
+ c.app.repo.tool.lower())
+ continue
+
+ c.app.repo.status = 'analyzing'
+ session(c.app.repo).flush(c.app.repo)
+ try:
+ ci_ids = list(
+ reversed(list(c.app.repo.all_commit_ids())))
+ if options.clean:
+ cls._clean(ci_ids)
+
+ log.info('Refreshing all last commits in %r',
+ c.app.repo)
+ cls.refresh_repo_lcds(ci_ids, options)
+ new_commit_ids = app.repo.unknown_commit_ids()
+ if len(new_commit_ids) > 0:
+ refresh.post()
+ except:
+ log.exception('Error refreshing %r', c.app.repo)
+ raise
+ finally:
+ c.app.repo.status = 'ready'
+ session(c.app.repo).flush(c.app.repo)
+ ThreadLocalORMSession.flush_all()
+
+ @classmethod
+ def refresh_repo_lcds(cls, commit_ids, options):
+ tree_cache = {}
+ timings = []
+ model_cache = M.repository.ModelCache(
+ max_instances={M.repository.LastCommit: 4000},
+ max_queries={M.repository.LastCommit: 4000},
+ )
+ lcid_cache = {}
+ timings = []
+ print('Processing last commits')
+ for i, commit_id in enumerate(commit_ids):
+ commit = M.repository.Commit.query.get(_id=commit_id)
+ if commit is None:
+ print("Commit missing, skipping: %s" % commit_id)
+ continue
+ commit.set_context(c.app.repo)
+ with time(timings):
+ M.repo_refresh.compute_lcds(commit, model_cache, lcid_cache)
+ ThreadLocalORMSession.flush_all()
+ if i % 100 == 0:
+ cls._print_stats(i, timings, 100)
+ if options.limit and i >= options.limit:
+ break
+ ThreadLocalORMSession.flush_all()
+
+ @classmethod
+ def _clean(cls, commit_ids):
+ # delete LastCommitDocs
+ i = M.repository.LastCommitDoc.m.find(
+ dict(commit_id={'$in': commit_ids})).count()
+ log.info("Deleting %i LastCommitDoc docs for %i commits...",
+ i, len(commit_ids))
+ M.repository.LastCommitDoc.m.remove(dict(commit_id={'$in': commit_ids}))
+
+ @classmethod
+ def _print_stats(cls, processed, timings, debug_step):
+ mt = max(timings)
+ tt = sum(timings)
+ at = tt / len(timings)
+ mat = sum(timings[-debug_step:]) / debug_step
+ print(' Processed %d commits (max: %f, avg: %f, mavg: %f, tot: %f)' % (
+ processed, mt, at, mat, tt))
+
+
+@contextmanager
+def time(timings):
+ s = datetime.utcnow()
+ yield
+ timings.append((datetime.utcnow() - s).total_seconds())
+
+
+if __name__ == '__main__':
+ faulthandler.enable()
+ RefreshLastCommits.main()
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/refreshrepo.py
----------------------------------------------------------------------
diff --git a/scripts/refreshrepo.py b/scripts/refreshrepo.py
new file mode 100644
index 0000000..15520a8
--- /dev/null
+++ b/scripts/refreshrepo.py
@@ -0,0 +1,203 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import logging
+
+import faulthandler
+from pylons import tmpl_context as c
+from ming.orm import ThreadLocalORMSession
+
+from allura import model as M
+from allura.lib.utils import chunked_find, chunked_list
+from allura.scripts import ScriptTask
+
+log = logging.getLogger(__name__)
+
+
+class RefreshRepo(ScriptTask):
+
+ @classmethod
+ def execute(cls, options):
+ q_project = {}
+ if options.nbhd:
+ nbhd = M.Neighborhood.query.get(url_prefix=options.nbhd)
+ if not nbhd:
+ return "Invalid neighborhood url prefix."
+ q_project['neighborhood_id'] = nbhd._id
+ if options.project:
+ q_project['shortname'] = options.project
+ elif options.project_regex:
+ q_project['shortname'] = {'$regex': options.project_regex}
+
+ log.info('Refreshing repositories')
+ for chunk in chunked_find(M.Project, q_project):
+ for p in chunk:
+ log.info("Refreshing repos for project '%s'." % p.shortname)
+ if options.dry_run:
+ continue
+ c.project = p
+ if options.mount_point:
+ mount_points = [options.mount_point]
+ else:
+ mount_points = [ac.options.mount_point for ac in
+ M.AppConfig.query.find(dict(project_id=p._id))]
+ for app in (p.app_instance(mp) for mp in mount_points):
+ c.app = app
+ if not hasattr(app, 'repo'):
+ continue
+ if c.app.repo.tool.lower() not in options.repo_types:
+ log.info("Skipping %r: wrong type (%s)", c.app.repo,
+ c.app.repo.tool.lower())
+ continue
+
+ if options.clean:
+ ci_ids = list(c.app.repo.all_commit_ids())
+ log.info("Deleting mongo data for %i commits...",
+ len(ci_ids))
+ # like the tree_ids themselves below, we need to process these in
+ # chunks to avoid hitting the BSON max size limit
+ tree_ids = []
+ for ci_ids_chunk in chunked_list(ci_ids, 3000):
+ tree_ids.extend([
+ tree_id for doc in
+ M.repository.TreesDoc.m.find(
+ {"_id": {"$in": ci_ids_chunk}},
+ {"tree_ids": 1})
+ for tree_id in doc.get("tree_ids", [])])
+
+ i = M.repository.CommitDoc.m.find(
+ {"_id": {"$in": ci_ids_chunk}}).count()
+ if i:
+ log.info("Deleting %i CommitDoc docs...", i)
+ M.repository.CommitDoc.m.remove(
+ {"_id": {"$in": ci_ids_chunk}})
+
+ # delete these in chunks, otherwise the query doc can
+ # exceed the max BSON size limit (16MB at the moment)
+ for tree_ids_chunk in chunked_list(tree_ids, 300000):
+ i = M.repository.TreeDoc.m.find(
+ {"_id": {"$in": tree_ids_chunk}}).count()
+ if i:
+ log.info("Deleting %i TreeDoc docs...", i)
+ M.repository.TreeDoc.m.remove(
+ {"_id": {"$in": tree_ids_chunk}})
+ del tree_ids
+
+ # delete these after TreeDoc and LastCommitDoc so that if
+ # we crash, we don't lose the ability to delete those
+ for ci_ids_chunk in chunked_list(ci_ids, 3000):
+ # delete TreesDocs
+ i = M.repository.TreesDoc.m.find(
+ {"_id": {"$in": ci_ids_chunk}}).count()
+ if i:
+ log.info("Deleting %i TreesDoc docs...", i)
+ M.repository.TreesDoc.m.remove(
+ {"_id": {"$in": ci_ids_chunk}})
+
+ # delete LastCommitDocs
+ i = M.repository.LastCommitDoc.m.find(
+ dict(commit_ids={'$in': ci_ids_chunk})).count()
+ if i:
+ log.info(
+ "Deleting %i remaining LastCommitDoc docs, by repo id...", i)
+ M.repository.LastCommitDoc.m.remove(
+ dict(commit_ids={'$in': ci_ids_chunk}))
+
+ i = M.repository.CommitRunDoc.m.find(
+ {"commit_ids": {"$in": ci_ids_chunk}}).count()
+ if i:
+ log.info("Deleting %i CommitRunDoc docs...", i)
+ M.repository.CommitRunDoc.m.remove(
+ {"commit_ids": {"$in": ci_ids_chunk}})
+ del ci_ids
+
+ try:
+ if options.all:
+ log.info('Refreshing ALL commits in %r',
+ c.app.repo)
+ else:
+ log.info('Refreshing NEW commits in %r',
+ c.app.repo)
+ if options.profile:
+ import cProfile
+ cProfile.runctx(
+ 'c.app.repo.refresh(options.all, notify=options.notify)',
+ globals(), locals(), 'refresh.profile')
+ else:
+ c.app.repo.refresh(
+ options.all, notify=options.notify)
+ except:
+ log.exception('Error refreshing %r', c.app.repo)
+ ThreadLocalORMSession.flush_all()
+
+ @classmethod
+ def parser(cls):
+ def repo_type_list(s):
+ repo_types = []
+ for repo_type in s.split(','):
+ repo_type = repo_type.strip()
+ if repo_type not in ['svn', 'git', 'hg']:
+ raise argparse.ArgumentTypeError(
+ '{} is not a valid repo type.'.format(repo_type))
+ repo_types.append(repo_type)
+ return repo_types
+
+ parser = argparse.ArgumentParser(description='Scan repos on filesytem and '
+ 'update repo metadata in MongoDB. Run for all repos (no args), '
+ 'or restrict by neighborhood, project, or code tool mount point.')
+ parser.add_argument('--nbhd', action='store', default='', dest='nbhd',
+ help='Restrict update to a particular neighborhood, e.g. /p/.')
+ parser.add_argument(
+ '--project', action='store', default='', dest='project',
+ help='Restrict update to a particular project. To specify a '
+ 'subproject, use a slash: project/subproject.')
+ parser.add_argument('--project-regex', action='store', default='',
+ dest='project_regex',
+ help='Restrict update to projects for which the shortname matches '
+ 'the provided regex.')
+ parser.add_argument(
+ '--repo-types', action='store', type=repo_type_list,
+ default=['svn', 'git', 'hg'], dest='repo_types',
+ help='Only refresh repos of the given type(s). Defaults to: '
+ 'svn,git,hg. Example: --repo-types=git,hg')
+ parser.add_argument('--mount-point', default='', dest='mount_point',
+ help='Restrict update to repos at the given tool mount point. ')
+ parser.add_argument('--clean', action='store_true', dest='clean',
+ default=False, help='Remove repo-related mongo docs (for '
+ 'project(s) being refreshed only) before doing the refresh.')
+ parser.add_argument(
+ '--all', action='store_true', dest='all', default=False,
+ help='Refresh all commits (not just the ones that are new).')
+ parser.add_argument('--notify', action='store_true', dest='notify',
+ default=False, help='Send email notifications of new commits.')
+ parser.add_argument('--dry-run', action='store_true', dest='dry_run',
+ default=False, help='Log names of projects that would have their '
+ 'repos refreshed, but do not perform the actual refresh.')
+ parser.add_argument('--profile', action='store_true', dest='profile',
+ default=False, help='Enable the profiler (slow). Will log '
+ 'profiling output to ./refresh.profile')
+ return parser
+
+
+if __name__ == '__main__':
+ faulthandler.enable()
+ RefreshRepo.main()
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/reindex_projects.py
----------------------------------------------------------------------
diff --git a/scripts/reindex_projects.py b/scripts/reindex_projects.py
new file mode 100644
index 0000000..2c51d43
--- /dev/null
+++ b/scripts/reindex_projects.py
@@ -0,0 +1,117 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import logging
+
+from pymongo.errors import InvalidDocument
+from pylons import tmpl_context as c, app_globals as g
+
+from allura.scripts import ScriptTask
+from allura import model as M
+from allura.tasks.index_tasks import add_projects
+from allura.lib.utils import chunked_find, chunked_list
+from allura.lib.exceptions import CompoundError
+
+
+log = logging.getLogger(__name__)
+
+
+class ReindexProjects(ScriptTask):
+
+ @classmethod
+ def execute(cls, options):
+ q_project = {}
+ if options.nbhd:
+ nbhd = M.Neighborhood.query.get(url_prefix=options.nbhd)
+ if not nbhd:
+ return "Invalid neighborhood url prefix."
+ q_project['neighborhood_id'] = nbhd._id
+ if options.project:
+ q_project['shortname'] = options.project
+ elif options.project_regex:
+ q_project['shortname'] = {'$regex': options.project_regex}
+
+ for chunk in chunked_find(M.Project, q_project):
+ project_ids = []
+ for p in chunk:
+ log.info('Reindex project %s', p.shortname)
+ if options.dry_run:
+ continue
+ c.project = p
+ project_ids.append(p._id)
+
+ try:
+ for chunk in chunked_list(project_ids, options.max_chunk):
+ if options.tasks:
+ cls._post_add_projects(chunk)
+ else:
+ add_projects(chunk)
+ except CompoundError as err:
+ log.exception('Error indexing projects:\n%r', err)
+ log.error('%s', err.format_error())
+ M.main_orm_session.flush()
+ M.main_orm_session.clear()
+ log.info('Reindex %s', 'queued' if options.tasks else 'done')
+
+ @classmethod
+ def _post_add_projects(cls, chunk):
+ """
+ Post task, recursively splitting and re-posting if the resulting
+ mongo document is too large.
+ """
+ try:
+ add_projects.post(chunk)
+ except InvalidDocument as e:
+ # there are many types of InvalidDocument, only recurse if its
+ # expected to help
+ if e.args[0].startswith('BSON document too large'):
+ cls._post_add_projects(chunk[:len(chunk) // 2])
+ cls._post_add_projects(chunk[len(chunk) // 2:])
+ else:
+ raise
+
+ @classmethod
+ def parser(cls):
+ parser = argparse.ArgumentParser(description='Reindex all projects')
+ parser.add_argument('-n', '--nbhd', action='store', default='', dest='nbhd',
+ help='Restrict reindex to a particular neighborhood, e.g. /p/.')
+ parser.add_argument(
+ '-p', '--project', action='store', default='', dest='project',
+ help='Restrict update to a particular project. To specify a '
+ 'subproject, use a slash: project/subproject.')
+ parser.add_argument('--project-regex', action='store', default='',
+ dest='project_regex',
+ help='Restrict update to projects for which the shortname matches '
+ 'the provided regex.')
+ parser.add_argument('--dry-run', action='store_true', dest='dry_run',
+ default=False, help='Log names of projects that would be reindexed, '
+ 'but do not perform the actual reindex.')
+ parser.add_argument('--tasks', action='store_true', dest='tasks',
+ help='Run each individual index operation as a background task.')
+ parser.add_argument(
+ '--max-chunk', dest='max_chunk', type=int, default=100 * 1000,
+ help='Max number of artifacts to index in one Solr update command')
+ return parser
+
+
+if __name__ == '__main__':
+ ReindexProjects.main()
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/reindex_users.py
----------------------------------------------------------------------
diff --git a/scripts/reindex_users.py b/scripts/reindex_users.py
new file mode 100644
index 0000000..4dc03aa
--- /dev/null
+++ b/scripts/reindex_users.py
@@ -0,0 +1,93 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import logging
+
+from pymongo.errors import InvalidDocument
+
+from allura.scripts import ScriptTask
+from allura import model as M
+from allura.tasks.index_tasks import add_users
+from allura.lib.utils import chunked_find, chunked_list
+from allura.lib.exceptions import CompoundError
+
+
+log = logging.getLogger(__name__)
+
+
+class ReindexUsers(ScriptTask):
+
+ @classmethod
+ def execute(cls, options):
+ for chunk in chunked_find(M.User, {}):
+ user_ids = []
+ for u in chunk:
+ log.info('Reindex user %s', u.username)
+ if options.dry_run:
+ continue
+ user_ids.append(u._id)
+ try:
+ for chunk in chunked_list(user_ids, options.max_chunk):
+ if options.tasks:
+ cls._post_add_users(chunk)
+ else:
+ add_users(chunk)
+ except CompoundError as err:
+ log.exception('Error indexing users:\n%r', err)
+ log.error('%s', err.format_error())
+ M.main_orm_session.flush()
+ M.main_orm_session.clear()
+ log.info('Reindex %s', 'queued' if options.tasks else 'done')
+
+ @classmethod
+ def _post_add_users(cls, chunk):
+ """
+ Post task, recursively splitting and re-posting if the resulting
+ mongo document is too large.
+ """
+ try:
+ add_users.post(chunk)
+ except InvalidDocument as e:
+ # there are many types of InvalidDocument, only recurse if its
+ # expected to help
+ if e.args[0].startswith('BSON document too large'):
+ cls._post_add_users(chunk[:len(chunk) // 2])
+ cls._post_add_users(chunk[len(chunk) // 2:])
+ else:
+ raise
+
+ @classmethod
+ def parser(cls):
+ parser = argparse.ArgumentParser(description='Reindex all users')
+ parser.add_argument('--dry-run', action='store_true', dest='dry_run',
+ default=False, help='Log names of projects that would be reindexed, '
+ 'but do not perform the actual reindex.')
+ parser.add_argument('--tasks', action='store_true', dest='tasks',
+ help='Run each individual index operation as a background task.')
+ parser.add_argument(
+ '--max-chunk', dest='max_chunk', type=int, default=100 * 1000,
+ help='Max number of artifacts to index in one Solr update command')
+ return parser
+
+
+if __name__ == '__main__':
+ ReindexUsers.main()
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/remove_duplicate_troves.py
----------------------------------------------------------------------
diff --git a/scripts/remove_duplicate_troves.py b/scripts/remove_duplicate_troves.py
new file mode 100644
index 0000000..f61fc71
--- /dev/null
+++ b/scripts/remove_duplicate_troves.py
@@ -0,0 +1,134 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import logging
+from itertools import groupby
+from collections import defaultdict
+from operator import itemgetter
+
+from ming.odm import ThreadLocalORMSession
+
+from allura.scripts import ScriptTask
+from allura import model as M
+
+
+log = logging.getLogger(__name__)
+
+
+class RemoveDuplicateTroves(ScriptTask):
+
+ trove_types = [
+ 'trove_root_database',
+ 'trove_developmentstatus',
+ 'trove_audience',
+ 'trove_license',
+ 'trove_os',
+ 'trove_language',
+ 'trove_topic',
+ 'trove_natlanguage',
+ 'trove_environment',
+ ]
+
+ @classmethod
+ def execute(cls, options):
+ duplicates = cls._find_duplicates()
+ log.info('Found %s duplicate categories: %s', len(duplicates), list(duplicates.keys()))
+ for name, dups in duplicates.items():
+ projects_with_category = {}
+ for dup in dups:
+ projects = cls._projects_with_category(dup._id)
+ projects_with_category[dup._id] = projects
+ log.info('Following projects are using category %s:', name)
+ for _id, ps in projects_with_category.items():
+ log.info(' with id %s: %s', _id, [p.shortname for p in ps])
+ priority = [(i, len(ps)) for i, ps in list(projects_with_category.items())]
+ priority = sorted(priority, key=itemgetter(1), reverse=True)
+ priority = [p[0] for p in priority]
+ live, kill = priority[0], priority[1:]
+ log.info('%s will live %s will die', live, kill)
+ if sum([len(projects_with_category[_id]) for _id in kill]) > 0:
+ # Duplicates are used somewhere, need to reasign for all projects that use them
+ projects = []
+ ids_to_kill = set(kill)
+ for p in [projects_with_category[_id] for _id in kill]:
+ projects.extend(p)
+ for p in projects:
+ for tt in cls.trove_types:
+ _ids = ids_to_kill.intersection(getattr(p, tt))
+ for _id in _ids:
+ log.info('Removing %s from %s.%s and adding %s instead', _id, p.shortname, tt, live)
+ if not options.dry_run:
+ getattr(p, tt).remove(_id)
+ getattr(p, tt).append(live)
+ log.info('Removing categories %s', kill)
+ if not options.dry_run:
+ M.TroveCategory.query.remove({'_id': {'$in': kill}})
+ ThreadLocalORMSession.flush_all()
+
+ @classmethod
+ def _find_duplicates(cls):
+ dups = []
+ agpl = M.TroveCategory.query.find({'shortname': 'agpl'}).all()
+ if len(agpl) > 1:
+ # agpl is present twice with different cat_id
+ # (update in creation command updated only one of duplicates),
+ # so code below will not catch it
+ dups.extend(agpl)
+ for cat in M.TroveCategory.query.find():
+ if M.TroveCategory.query.find({
+ 'shortname': cat.shortname,
+ 'trove_cat_id': cat.trove_cat_id,
+ 'trove_parent_id': cat.trove_parent_id,
+ 'fullname': cat.fullname,
+ 'fullpath': cat.fullpath,
+ }).count() > 1:
+ dups.append(cat)
+ result = defaultdict(list)
+ for k, v in groupby(dups, lambda x: x.shortname):
+ result[k].extend(list(v))
+ return result
+
+ @classmethod
+ def _projects_with_category(cls, _id):
+ p = M.Project.query.find({'$or': [
+ {'trove_root_database': _id},
+ {'trove_developmentstatus': _id},
+ {'trove_audience': _id},
+ {'trove_license': _id},
+ {'trove_os': _id},
+ {'trove_language': _id},
+ {'trove_topic': _id},
+ {'trove_natlanguage': _id},
+ {'trove_environment':_id},
+ ]})
+ return p.all()
+
+ @classmethod
+ def parser(cls):
+ parser = argparse.ArgumentParser(description='Remove duplicate troves')
+ parser.add_argument('--dry-run', action='store_true', dest='dry_run',
+ default=False, help='Print what will be changed but do not change anything')
+ return parser
+
+
+if __name__ == '__main__':
+ RemoveDuplicateTroves.main()
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/rethumb.py
----------------------------------------------------------------------
diff --git a/scripts/rethumb.py b/scripts/rethumb.py
index fa27f32..59ce311 100644
--- a/scripts/rethumb.py
+++ b/scripts/rethumb.py
@@ -1,3 +1,7 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -117,7 +121,7 @@ class RethumbCommand(base.Command):
# Otherwise, we'll take attachment classes belonging to app's
# package
- ep = iter_entry_points('allura', app.tool_name).next()
+ ep = next(iter_entry_points('allura', app.tool_name))
app_package = ep.module_name.split('.', 1)[0]
if app_package == 'allura':
# Apps in allura known to not define own attachment types
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/scripttask.py
----------------------------------------------------------------------
diff --git a/scripts/scripttask.py b/scripts/scripttask.py
new file mode 100644
index 0000000..f3d6e6e
--- /dev/null
+++ b/scripts/scripttask.py
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Provides ScriptTask, a base class for implementing a command-line script that
+can be run as a task.
+
+To use, subclass ScriptTask and implement two methods::
+
+ class MyScript(ScriptTask):
+ @classmethod
+ def parser(cls):
+ '''Define and return an argparse.ArgumentParser instance'''
+ pass
+
+ @classmethod
+ def execute(cls, options):
+ '''Your main code goes here.'''
+ pass
+
+To call as a script::
+
+ if __name__ == '__main__':
+ MyScript.main()
+
+To call as a task::
+
+ # post the task with cmd-line-style args
+ MyScript.post('-p myproject --dry-run')
+
+"""
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import argparse
+import logging
+import shlex
+import sys
+
+from allura.lib.decorators import task
+
+
+log = logging.getLogger(__name__)
+
+
+class ScriptTask(object):
+
+ """Base class for a command-line script that is also executable as a task."""
+
+ class __metaclass__(type):
+
+ @property
+ def __doc__(cls):
+ return cls.parser().format_help()
+
+ def __new__(meta, classname, bases, classDict):
+ return task(type.__new__(meta, classname, bases, classDict))
+
+ def __new__(cls, arg_string=''):
+ return cls._execute_task(arg_string)
+
+ @classmethod
+ def _execute_task(cls, arg_string):
+ try:
+ options = cls.parser().parse_args(shlex.split(arg_string or ''))
+ except SystemExit:
+ raise Exception("Error parsing args: '%s'" % arg_string)
+ return cls.execute(options)
+
+ @classmethod
+ def execute(cls, options):
+ """User code goes here."""
+ pass
+
+ @classmethod
+ def parser(cls):
+ """Return an argument parser appropriate for your script."""
+ return argparse.ArgumentParser(description="Default no-op parser")
+
+ @classmethod
+ def main(cls):
+ options = cls.parser().parse_args()
+ cls.execute(options)
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/scrub-allura-data.py
----------------------------------------------------------------------
diff --git a/scripts/scrub-allura-data.py b/scripts/scrub-allura-data.py
index c0e0828..e569710 100644
--- a/scripts/scrub-allura-data.py
+++ b/scripts/scrub-allura-data.py
@@ -1,3 +1,7 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/setup-scm-server.py
----------------------------------------------------------------------
diff --git a/scripts/setup-scm-server.py b/scripts/setup-scm-server.py
index 050122c..77c3bce 100644
--- a/scripts/setup-scm-server.py
+++ b/scripts/setup-scm-server.py
@@ -1,3 +1,7 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/teamforge-import.py
----------------------------------------------------------------------
diff --git a/scripts/teamforge-import.py b/scripts/teamforge-import.py
index 9f601a0..e06b90d 100644
--- a/scripts/teamforge-import.py
+++ b/scripts/teamforge-import.py
@@ -1,3 +1,7 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -132,7 +136,7 @@ def main():
project_ids = [p.id for p in projects.dataRows]
if options.list_project_ids:
- print ' '.join(project_ids)
+ print(' '.join(project_ids))
return
if not os.path.exists(options.output_dir):
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/test-branches-against-tickets.py
----------------------------------------------------------------------
diff --git a/scripts/test-branches-against-tickets.py b/scripts/test-branches-against-tickets.py
index a742f03..1391251 100755
--- a/scripts/test-branches-against-tickets.py
+++ b/scripts/test-branches-against-tickets.py
@@ -1,3 +1,7 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/trac_export.py
----------------------------------------------------------------------
diff --git a/scripts/trac_export.py b/scripts/trac_export.py
old mode 100755
new mode 100644
index ac90b17..488d65f
--- a/scripts/trac_export.py
+++ b/scripts/trac_export.py
@@ -17,6 +17,316 @@
# specific language governing permissions and limitations
# under the License.
+import logging
+import sys
+import csv
+import urllib.parse
+import urllib.request, urllib.error, urllib.parse
+import json
+import time
+import re
+from optparse import OptionParser
+from itertools import islice
+
+from BeautifulSoup import BeautifulSoup, NavigableString
+import dateutil.parser
+import pytz
+
+try:
+ from forgeimporters.base import ProjectExtractor
+ urlopen = ProjectExtractor.urlopen
+except ImportError:
+ try:
+ from allura.lib.helpers import urlopen
+ except ImportError:
+ from urllib.request import urlopen
+
+log = logging.getLogger(__name__)
+
+
+def parse_options():
+ optparser = OptionParser(usage=''' %prog <Trac URL>
+
+Export ticket data from a Trac instance''')
+ optparser.add_option('-o', '--out-file', dest='out_filename',
+ help='Write to file (default stdout)')
+ optparser.add_option('--no-attachments', dest='do_attachments',
+ action='store_false', default=True, help='Export attachment info')
+ optparser.add_option('--only-tickets', dest='only_tickets',
+ action='store_true', help='Export only ticket list')
+ optparser.add_option('--start', dest='start_id', type='int', default=1,
+ help='Start with given ticket numer (or next accessible)')
+ optparser.add_option('--limit', dest='limit', type='int',
+ default=None, help='Limit number of tickets')
+ optparser.add_option('-v', '--verbose', dest='verbose',
+ action='store_true', help='Verbose operation')
+ options, args = optparser.parse_args()
+ if len(args) != 1:
+ optparser.error("Wrong number of arguments.")
+ return options, args
+
+
+class TracExport(object):
+
+ PAGE_SIZE = 100
+ TICKET_URL = 'ticket/%d'
+ QUERY_MAX_ID_URL = 'query?col=id&order=id&desc=1&max=2'
+ QUERY_BY_PAGE_URL = 'query?col=id&col=time&col=changetime&order=id&max=' + \
+ str(PAGE_SIZE) + '&page=%d'
+ ATTACHMENT_LIST_URL = 'attachment/ticket/%d/'
+ ATTACHMENT_URL = 'raw-attachment/ticket/%d/%s'
+
+ FIELD_MAP = {
+ 'reporter': 'submitter',
+ 'owner': 'assigned_to',
+ }
+
+ def __init__(self, base_url, start_id=1, verbose=False, do_attachments=True):
+ """start_id - start with at least that ticket number (actual returned
+ ticket may have higher id if we don't have access to exact
+ one).
+ """
+ self.base_url = base_url.rstrip('/') + '/'
+ # Contains additional info for a ticket which cannot
+ # be get with single-ticket export (create/mod times is
+ # and example).
+ self.ticket_map = {}
+ self.start_id = start_id
+ self.page = (start_id - 1) / self.PAGE_SIZE + 1
+ self.verbose = verbose
+ self.do_attachments = do_attachments
+ self.exhausted = False
+ self.ticket_queue = self.next_ticket_ids()
+
+ def remap_fields(self, dict):
+ "Remap fields to adhere to standard taxonomy."
+ out = {}
+ for k, v in dict.items():
+ key = self.match_pattern(r'\W*(\w+)\W*', k)
+ out[self.FIELD_MAP.get(key, key)] = v
+
+ out['id'] = int(out['id'])
+ if 'private' in out:
+ out['private'] = bool(int(out['private']))
+ return out
+
+ def full_url(self, suburl, type=None):
+ url = urllib.parse.urljoin(self.base_url, suburl)
+ if type is None:
+ return url
+ glue = '&' if '?' in suburl else '?'
+ return url + glue + 'format=' + type
+
+ def log_url(self, url):
+ log.info(url)
+ if self.verbose:
+ print(url, file=sys.stderr)
+
+ @classmethod
+ def trac2z_date(cls, s):
+ d = dateutil.parser.parse(s)
+ if d.tzinfo is not None:
+ d = d.astimezone(pytz.UTC)
+ return d.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+ @staticmethod
+ def match_pattern(regexp, string):
+ m = re.match(regexp, string)
+ assert m
+ return m.group(1)
+
+ def csvopen(self, url):
+ self.log_url(url)
+ f = urlopen(url)
+ # Trac doesn't throw 403 error, just shows normal 200 HTML page
+ # telling that access denied. So, we'll emulate 403 ourselves.
+ # TODO: currently, any non-csv result treated as 403.
+ if not f.info()['Content-Type'].startswith('text/csv'):
+ raise urllib.error.HTTPError(
+ url, 403, 'Forbidden - emulated', f.info(), f)
+ return f
+
+ def parse_ticket(self, id):
+ # Use CSV export to get ticket fields
+ url = self.full_url(self.TICKET_URL % id, 'csv')
+ f = self.csvopen(url)
+ reader = csv.DictReader(f)
+ ticket_fields = next(reader)
+ ticket_fields['class'] = 'ARTIFACT'
+ ticket = self.remap_fields(ticket_fields)
+
+ # Use HTML export to get ticket description and comments
+ import html2text
+ html2text.BODY_WIDTH = 0
+ url = self.full_url(self.TICKET_URL % id)
+ self.log_url(url)
+ d = BeautifulSoup(urlopen(url))
+ self.clean_missing_wiki_links(d)
+ desc = d.find('div', 'description').find('div', 'searchable')
+ ticket['description'] = html2text.html2text(
+ desc.renderContents('utf8').decode('utf8')) if desc else ''
+ comments = []
+ for comment in d.findAll('form', action='#comment'):
+ c = {}
+ c['submitter'] = re.sub(
+ r'.* by ', '', comment.find('h3', 'change').text).strip()
+ c['date'] = self.trac2z_date(
+ comment.find('a', 'timeline')['title'].replace(' in Timeline', ''))
+ changes = str(comment.find('ul', 'changes') or '')
+ body = comment.find('div', 'comment')
+ body = body.renderContents('utf8').decode('utf8') if body else ''
+ c['comment'] = html2text.html2text(changes + body)
+ c['class'] = 'COMMENT'
+ comments.append(c)
+ ticket['comments'] = comments
+ return ticket
+
+ def parse_ticket_attachments(self, id):
+ SIZE_PATTERN = r'(\d+) bytes'
+ TIMESTAMP_PATTERN = r'(.+) in Timeline'
+ # Scrape HTML to get ticket attachments
+ url = self.full_url(self.ATTACHMENT_LIST_URL % id)
+ self.log_url(url)
+ f = urlopen(url)
+ soup = BeautifulSoup(f)
+ attach = soup.find('div', id='attachments')
+ list = []
+ while attach:
+ attach = attach.findNext('dt')
+ if not attach:
+ break
+ d = {}
+ d['filename'] = attach.a['href'].rsplit('/', 1)[1]
+ d['url'] = self.full_url(self.ATTACHMENT_URL % (id, d['filename']))
+ size_s = attach.span['title']
+ d['size'] = int(self.match_pattern(SIZE_PATTERN, size_s))
+ timestamp_s = attach.find('a', {'class': 'timeline'})['title']
+ d['date'] = self.trac2z_date(
+ self.match_pattern(TIMESTAMP_PATTERN, timestamp_s))
+ d['by'] = attach.find(
+ text=re.compile('added by')).nextSibling.renderContents()
+ d['description'] = ''
+ # Skip whitespace
+ while attach.nextSibling and type(attach.nextSibling) is NavigableString:
+ attach = attach.nextSibling
+ # if there's a description, there will be a <dd> element, other
+ # immediately next <dt>
+ if attach.nextSibling and attach.nextSibling.name == 'dd':
+ desc_el = attach.nextSibling
+ if desc_el:
+ # TODO: Convert to Allura link syntax as needed
+ d['description'] = ''.join(
+ desc_el.findAll(text=True)).strip()
+ list.append(d)
+ return list
+
+ def get_max_ticket_id(self):
+ url = self.full_url(self.QUERY_MAX_ID_URL, 'csv')
+ f = self.csvopen(url)
+ reader = csv.DictReader(f)
+ fields = next(reader)
+ print(fields)
+ return int(fields['id'])
+
+ def get_ticket(self, id, extra={}):
+ '''Get ticket with given id
+ extra: extra fields to add to ticket (parsed elsewhere)
+ '''
+ t = self.parse_ticket(id)
+ if self.do_attachments:
+ atts = self.parse_ticket_attachments(id)
+ if atts:
+ t['attachments'] = atts
+ t.update(extra)
+ return t
+
+ def next_ticket_ids(self):
+ 'Go thru ticket list and collect available ticket ids.'
+ # We could just do CSV export, which by default dumps entire list
+ # Alas, for many busy servers with long ticket list, it will just
+ # time out. So, let's paginate it instead.
+ res = []
+
+ url = self.full_url(self.QUERY_BY_PAGE_URL % self.page, 'csv')
+ try:
+ f = self.csvopen(url)
+ except urllib.error.HTTPError as e:
+ if 'emulated' in e.msg:
+ body = e.fp.read()
+ if 'beyond the number of pages in the query' in body or 'Log in with a SourceForge account' in body:
+ raise StopIteration
+ raise
+ reader = csv.reader(f)
+ cols = next(reader)
+ for r in reader:
+ if r and r[0].isdigit():
+ id = int(r[0])
+ extra = {'date': self.trac2z_date(
+ r[1]), 'date_updated': self.trac2z_date(r[2])}
+ res.append((id, extra))
+ self.page += 1
+
+ if len(res) < self.PAGE_SIZE:
+ self.exhausted = True
+
+ return res
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ while True:
+ # queue empty, try to fetch more
+ if len(self.ticket_queue) == 0 and not self.exhausted:
+ self.ticket_queue = self.next_ticket_ids()
+ # there aren't any more, we're really done
+ if len(self.ticket_queue) == 0:
+ raise StopIteration
+ id, extra = self.ticket_queue.pop(0)
+ if id >= self.start_id:
+ break
+ return self.get_ticket(id, extra)
+
+ def clean_missing_wiki_links(self, doc):
+ for link in doc.findAll('a', 'missing wiki'):
+ link.string = link.string.rstrip('?')
+
+
+class DateJSONEncoder(json.JSONEncoder):
+
+ def default(self, obj):
+ if isinstance(obj, time.struct_time):
+ return time.strftime('%Y-%m-%dT%H:%M:%SZ', obj)
+ return json.JSONEncoder.default(self, obj)
+
+
+def export(url, start_id=1, verbose=False, do_attachments=True,
+ only_tickets=False, limit=None, **kw):
+ ex = TracExport(url, start_id=start_id,
+ verbose=verbose, do_attachments=do_attachments)
+
+ doc = [t for t in islice(ex, limit)]
+
+ if not only_tickets:
+ doc = {
+ 'class': 'PROJECT',
+ 'trackers': {'default': {'artifacts': doc}}
+ }
+ return doc
+
+
+def main():
+ options, args = parse_options()
+ doc = export(args[0], **vars(options))
+
+ out_file = sys.stdout
+ if options.out_filename:
+ out_file = open(options.out_filename, 'w')
+ out_file.write(
+ json.dumps(doc, cls=DateJSONEncoder, indent=2, sort_keys=True))
+ # It's bad habit not to terminate lines
+ out_file.write('\n')
+
+
if __name__ == '__main__':
- from allura.scripts.trac_export import main
main()
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/trim_emails.py
----------------------------------------------------------------------
diff --git a/scripts/trim_emails.py b/scripts/trim_emails.py
new file mode 100644
index 0000000..6e7f542
--- /dev/null
+++ b/scripts/trim_emails.py
@@ -0,0 +1,58 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+from ming.orm import session
+
+from allura.scripts import ScriptTask
+from allura import model as M
+from allura.lib.utils import chunked_find
+
+
+log = logging.getLogger(__name__)
+
+
+class TrimEmails(ScriptTask):
+
+ @classmethod
+ def execute(cls, options):
+ for chunk in chunked_find(M.User, {}):
+ for u in chunk:
+ log.info('Trimming emails for user %s', u.username)
+ new_addresses = [M.EmailAddress.canonical(addr) for addr in u.email_addresses]
+ u.email_addresses = new_addresses
+ if u.preferences.email_address is not None:
+ u.preferences.email_address = M.EmailAddress.canonical(
+ u.preferences.email_address)
+ session(u).flush(u)
+ for chunk in chunked_find(M.EmailAddress, {}):
+ for a in chunk:
+ log.info('Trimming email address entry %s', a.email)
+ a.email = M.EmailAddress.canonical(a.email)
+ session(a).flush(a)
+ M.main_orm_session.flush()
+ M.main_orm_session.clear()
+ log.info('Finished trimming emails')
+
+
+if __name__ == '__main__':
+ TrimEmails.main()
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/update_checkout_url.py
----------------------------------------------------------------------
diff --git a/scripts/update_checkout_url.py b/scripts/update_checkout_url.py
new file mode 100644
index 0000000..8bf115f
--- /dev/null
+++ b/scripts/update_checkout_url.py
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Find repos with blank checkout url and top-level "trunk" dir,
+and point checkout_url at trunk.
+"""
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import logging
+
+from ming.orm import ThreadLocalORMSession
+
+from allura import model as M
+from allura.lib import utils
+from allura.scripts import ScriptTask
+
+from forgesvn.model.svn import Repository, svn_path_exists
+
+log = logging.getLogger(__name__)
+
+
+class UpdateCheckoutUrl(ScriptTask):
+
+ @classmethod
+ def execute(cls, options):
+ query = {'tool_name': {'$regex': '^svn$', '$options': 'i'},
+ 'options.checkout_url': ''}
+ for chunk in utils.chunked_find(M.AppConfig, query):
+ for config in chunk:
+ repo = Repository.query.get(app_config_id=config._id)
+ trunk_path = "file://{0}{1}/trunk".format(repo.fs_path,
+ repo.name)
+ if svn_path_exists(trunk_path):
+ config.options['checkout_url'] = "trunk"
+ log.info("Update checkout_url for: %s", trunk_path)
+ ThreadLocalORMSession.flush_all()
+
+if __name__ == '__main__':
+ UpdateCheckoutUrl.main()
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/wiki-copy.py
----------------------------------------------------------------------
diff --git a/scripts/wiki-copy.py b/scripts/wiki-copy.py
index 0692852..52e274f 100644
--- a/scripts/wiki-copy.py
+++ b/scripts/wiki-copy.py
@@ -17,6 +17,10 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
import os
import sys
import urllib
@@ -51,17 +55,17 @@ def main():
page_data = urllib.urlopen(from_url).read()
page_json = json.loads(page_data)
if options.debug:
- print page_json['text']
+ print(page_json['text'])
break
resp = oauth_client.request(
to_url, 'POST', body=urllib.urlencode(dict(text=page_json['text'].encode('utf-8'))))
if resp[0]['status'] == '200':
- print "Posted {0} to {1}".format(page_json['title'], to_url)
+ print("Posted {0} to {1}".format(page_json['title'], to_url))
else:
- print "Error posting {0} to {1}: {2} (project may not exist)".format(page_json['title'], to_url, resp[0]['status'])
+ print("Error posting {0} to {1}: {2} (project may not exist)".format(page_json['title'], to_url, resp[0]['status']))
break
except:
- print "Error processing " + p
+ print("Error processing " + p)
raise
@@ -114,8 +118,8 @@ def make_oauth_client(base_url):
# save oauth token for later use
cp.write(open(config_file, 'w'))
- print 'Saving oauth tokens in {} for later re-use'.format(config_file)
- print
+ print('Saving oauth tokens in {} for later re-use'.format(config_file))
+ print()
access_token = oauth.Token(oauth_token, oauth_token_secret)
oauth_client = oauth.Client(consumer, access_token)
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/wiki-export.py
----------------------------------------------------------------------
diff --git a/scripts/wiki-export.py b/scripts/wiki-export.py
index 025d3ca..0f8e776 100755
--- a/scripts/wiki-export.py
+++ b/scripts/wiki-export.py
@@ -17,6 +17,10 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
import sys
from optparse import OptionParser
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/scripts/wiki-post.py
----------------------------------------------------------------------
diff --git a/scripts/wiki-post.py b/scripts/wiki-post.py
index 7e16bb1..1151f81 100755
--- a/scripts/wiki-post.py
+++ b/scripts/wiki-post.py
@@ -17,6 +17,10 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
import types
from sys import stdout
import os
@@ -35,7 +39,7 @@ def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'):
This function was borrowed from Django
"""
- if strings_only and isinstance(s, (types.NoneType, int)):
+ if strings_only and isinstance(s, (type(None), int)):
return s
elif not isinstance(s, basestring):
try:
@@ -117,7 +121,7 @@ def main():
url = options.url or config.get('wiki', 'url')
if pagename_given:
url = urljoin(url, urllib.quote(pagename))
- print url
+ print(url)
sign = Signer(token)
params = [('text', markdown)] if method == 'PUT' else []
@@ -128,7 +132,7 @@ def main():
else:
result = urlopen(url + '?' + urlencode(params))
stdout.write(result.read())
- except HTTPError, e:
+ except HTTPError as e:
stdout.write(e.read())
if __name__ == '__main__':
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/tasks/__init__.py
----------------------------------------------------------------------
diff --git a/tasks/__init__.py b/tasks/__init__.py
new file mode 100644
index 0000000..144e298
--- /dev/null
+++ b/tasks/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/tasks/activity_tasks.py
----------------------------------------------------------------------
diff --git a/tasks/activity_tasks.py b/tasks/activity_tasks.py
new file mode 100644
index 0000000..f25f69e
--- /dev/null
+++ b/tasks/activity_tasks.py
@@ -0,0 +1,29 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pylons import app_globals as g
+
+from allura.lib.decorators import task
+
+
+@task
+def create_timelines(node_id):
+ g.director.create_timelines(node_id)
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/tasks/admin_tasks.py
----------------------------------------------------------------------
diff --git a/tasks/admin_tasks.py b/tasks/admin_tasks.py
new file mode 100644
index 0000000..7056c83
--- /dev/null
+++ b/tasks/admin_tasks.py
@@ -0,0 +1,43 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import inspect
+
+from pylons import tmpl_context as c
+
+from allura import model as M
+from allura.lib.decorators import task
+
+
+@task
+def install_app(*args, **kwargs):
+ """Install an application directly onto c.project, bypassing the UI and
+ any app constraints like ``installable=False``.
+
+ """
+ c.project.install_app(*args, **kwargs)
+
+install_app.__doc__ += '''
+ Arguments::
+
+ ''' + inspect.formatargspec(*inspect.getargspec(
+ M.Project.install_app
+)).replace('self, ', '')
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/tasks/event_tasks.py
----------------------------------------------------------------------
diff --git a/tasks/event_tasks.py b/tasks/event_tasks.py
new file mode 100644
index 0000000..43df95d
--- /dev/null
+++ b/tasks/event_tasks.py
@@ -0,0 +1,40 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+
+from allura.lib.decorators import task, event_handler
+from allura.lib.exceptions import CompoundError
+
+
+@task
+def event(event_type, *args, **kwargs):
+ exceptions = []
+ for t in event_handler.listeners[event_type]:
+ try:
+ t(event_type, *args, **kwargs)
+ except:
+ exceptions.append(sys.exc_info())
+ if exceptions:
+ if len(exceptions) == 1:
+ raise exceptions[0][0](exceptions[0][1]).with_traceback(exceptions[0][2])
+ else:
+ raise CompoundError(*exceptions)
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/tasks/export_tasks.py
----------------------------------------------------------------------
diff --git a/tasks/export_tasks.py b/tasks/export_tasks.py
new file mode 100644
index 0000000..0564685
--- /dev/null
+++ b/tasks/export_tasks.py
@@ -0,0 +1,132 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import os.path
+import logging
+import shutil
+
+import tg
+from pylons import app_globals as g, tmpl_context as c
+
+from allura.tasks import mail_tasks
+from allura.lib.decorators import task
+from allura.lib import helpers as h
+from allura.model.repository import zipdir
+
+
+log = logging.getLogger(__name__)
+
+
+@task
+def bulk_export(tools, filename=None, send_email=True):
+ '''
+ Export the current project data. Send notification to current user.
+
+ :param list tools: list of mount_points to export
+ :param str filename: optional filename to use
+ '''
+ # it's very handy to use c.* within a @task,
+ # but let's be explicit and keep it separate from the main code
+ return BulkExport().process(c.project, tools, c.user, filename, send_email)
+
+
+class BulkExport(object):
+
+ def process(self, project, tools, user, filename=None, send_email=True):
+ export_filename = filename or project.bulk_export_filename()
+ export_path = self.get_export_path(
+ project.bulk_export_path(), export_filename)
+ if not os.path.exists(export_path):
+ os.makedirs(export_path)
+ apps = [project.app_instance(tool) for tool in tools]
+ exportable = self.filter_exportable(apps)
+ results = [self.export(export_path, app) for app in exportable]
+ exported = self.filter_successful(results)
+ if exported:
+ zipdir(export_path,
+ os.path.join(os.path.dirname(export_path), export_filename))
+ shutil.rmtree(export_path)
+
+ if not user:
+ log.info('No user. Skipping notification.')
+ return
+ if not send_email:
+ return
+
+ tmpl = g.jinja2_env.get_template(
+ 'allura:templates/mail/bulk_export.html')
+ instructions = tg.config.get('bulk_export_download_instructions', '')
+ instructions = instructions.format(
+ project=project.shortname,
+ filename=export_filename,
+ c=c,
+ )
+ exported_names = [a.config.options.mount_point for a in exported]
+ tmpl_context = {
+ 'instructions': instructions,
+ 'project': project,
+ 'tools': exported_names,
+ 'not_exported_tools': list(set(tools) - set(exported_names)),
+ }
+
+ email = {
+ 'toaddr': str(user._id),
+ 'fromaddr': str(tg.config['forgemail.return_path']),
+ 'sender': str(tg.config['forgemail.return_path']),
+ 'reply_to': str(tg.config['forgemail.return_path']),
+ 'message_id': h.gen_message_id(),
+ 'subject': 'Bulk export for project %s completed' % project.shortname,
+ 'text': tmpl.render(tmpl_context)
+ }
+
+ mail_tasks.sendsimplemail.post(**email)
+
+ def get_export_path(self, export_base_path, export_filename):
+ """Create temporary directory for export files"""
+ # Name temporary directory after project shortname,
+ # thus zipdir() will use proper prefix inside the archive.
+ tmp_dir_suffix = os.path.splitext(export_filename)[0]
+ path = os.path.join(export_base_path, tmp_dir_suffix)
+ return path
+
+ def filter_exportable(self, apps):
+ return [app for app in apps if app and app.exportable]
+
+ def export(self, export_path, app):
+ tool = app.config.options.mount_point
+ json_file = os.path.join(export_path, '%s.json' % tool)
+ try:
+ with open(json_file, 'w') as f:
+ app.bulk_export(f)
+ except Exception:
+ log.error('Error exporting: %s on %s', tool,
+ app.project.shortname, exc_info=True)
+ try:
+ os.remove(json_file)
+ except:
+ pass
+ return None
+ else:
+ return app
+
+ def filter_successful(self, results):
+ return [result for result in results if result is not None]
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/tasks/index_tasks.py
----------------------------------------------------------------------
diff --git a/tasks/index_tasks.py b/tasks/index_tasks.py
new file mode 100644
index 0000000..d239ac9
--- /dev/null
+++ b/tasks/index_tasks.py
@@ -0,0 +1,138 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+import logging
+from contextlib import contextmanager
+
+from pylons import app_globals as g
+
+from allura.lib.decorators import task
+from allura.lib.exceptions import CompoundError
+from allura.lib.solr import make_solr_from_config
+
+
+log = logging.getLogger(__name__)
+
+
+def __get_solr(solr_hosts=None):
+ return make_solr_from_config(solr_hosts) if solr_hosts else g.solr
+
+
+def __add_objects(objects, solr_hosts=None):
+ solr_instance = __get_solr(solr_hosts)
+ solr_instance.add([obj.solarize() for obj in objects])
+
+
+def __del_objects(object_solr_ids):
+ solr_instance = __get_solr()
+ solr_query = 'id:({0})'.format(' || '.join(object_solr_ids))
+ solr_instance.delete(q=solr_query)
+
+
+@task
+def add_projects(project_ids):
+ from allura.model.project import Project
+ projects = Project.query.find(dict(_id={'$in': project_ids})).all()
+ __add_objects(projects)
+
+
+@task
+def del_projects(project_solr_ids):
+ __del_objects(project_solr_ids)
+
+
+@task
+def add_users(user_ids):
+ from allura.model import User
+ users = User.query.find(dict(_id={'$in': user_ids})).all()
+ __add_objects(users)
+
+
+@task
+def del_users(user_solr_ids):
+ __del_objects(user_solr_ids)
+
+
+@task
+def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
+ '''
+ Add the referenced artifacts to SOLR and shortlinks.
+
+ :param solr_hosts: a list of solr hosts to use instead of the defaults
+ :type solr_hosts: [str]
+ '''
+ from allura import model as M
+ from allura.lib.search import find_shortlinks
+
+ exceptions = []
+ solr_updates = []
+ with _indexing_disabled(M.session.artifact_orm_session._get()):
+ for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
+ try:
+ artifact = ref.artifact
+ if artifact is None:
+ continue
+ s = artifact.solarize()
+ if s is None:
+ continue
+ if update_solr:
+ solr_updates.append(s)
+ if update_refs:
+ if isinstance(artifact, M.Snapshot):
+ continue
+ # Find shortlinks in the raw text, not the escaped html
+ # created by the `solarize()`.
+ link_text = artifact.index().get('text') or ''
+ shortlinks = find_shortlinks(link_text)
+ ref.references = [link.ref_id for link in shortlinks]
+ except Exception:
+ log.error('Error indexing artifact %s', ref._id)
+ exceptions.append(sys.exc_info())
+ __get_solr(solr_hosts).add(solr_updates)
+
+ if len(exceptions) == 1:
+ raise exceptions[0][0](exceptions[0][1]).with_traceback(exceptions[0][2])
+ if exceptions:
+ raise CompoundError(*exceptions)
+
+
+@task
+def del_artifacts(ref_ids):
+ from allura import model as M
+ if ref_ids:
+ __del_objects(ref_ids)
+ M.ArtifactReference.query.remove(dict(_id={'$in': ref_ids}))
+ M.Shortlink.query.remove(dict(ref_id={'$in': ref_ids}))
+
+
+@task
+def commit():
+ g.solr.commit()
+
+
+@contextmanager
+def _indexing_disabled(session):
+ session.disable_index = session.skip_mod_date = True
+ try:
+ yield session
+ finally:
+ session.disable_index = session.skip_mod_date = False