You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2015/12/11 20:32:58 UTC
[1/5] allura git commit: [#8033] backwards compatibility script
Repository: allura
Updated Branches:
refs/heads/db/8033 [created] e1aeec7f0
[#8033] backwards compatibility script
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/2878e9cf
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/2878e9cf
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/2878e9cf
Branch: refs/heads/db/8033
Commit: 2878e9cf8044d9c720778c020fd62db3ace4d87e
Parents: 2f81405
Author: Dave Brondsema <da...@brondsema.net>
Authored: Fri Dec 11 12:15:40 2015 -0500
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Fri Dec 11 12:16:49 2015 -0500
----------------------------------------------------------------------
scripts/create-allura-sitemap.py | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/2878e9cf/scripts/create-allura-sitemap.py
----------------------------------------------------------------------
diff --git a/scripts/create-allura-sitemap.py b/scripts/create-allura-sitemap.py
new file mode 100644
index 0000000..7f76f6b
--- /dev/null
+++ b/scripts/create-allura-sitemap.py
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from allura.scripts.create_sitemap_files import CreateSitemapFiles
+
+# this file exists for backwards compatibility
+
+if __name__ == '__main__':
+ CreateSitemapFiles.main()
[2/5] allura git commit: [#8033] move
scripts/create-allura-sitemap.py to ScriptTask
Posted by br...@apache.org.
[#8033] move scripts/create-allura-sitemap.py to ScriptTask
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/2f81405d
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/2f81405d
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/2f81405d
Branch: refs/heads/db/8033
Commit: 2f81405dd2d722263b27a883f009ee5c747621cc
Parents: 47b6504
Author: Dave Brondsema <da...@brondsema.net>
Authored: Fri Dec 11 12:14:12 2015 -0500
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Fri Dec 11 12:16:49 2015 -0500
----------------------------------------------------------------------
Allura/allura/scripts/create_sitemap_files.py | 174 +++++++++++++++++++
Allura/docs/getting_started/administration.rst | 16 +-
scripts/create-allura-sitemap.py | 175 --------------------
3 files changed, 182 insertions(+), 183 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/2f81405d/Allura/allura/scripts/create_sitemap_files.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/create_sitemap_files.py b/Allura/allura/scripts/create_sitemap_files.py
new file mode 100644
index 0000000..e4d98a9
--- /dev/null
+++ b/Allura/allura/scripts/create_sitemap_files.py
@@ -0,0 +1,174 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Generate Allura sitemap xml files. You will need to configure your webserver to serve the files.
+
+This takes a while to run on a prod-sized data set. There are a couple of
+things that would make it faster, if we need/want to.
+
+1. Monkeypatch forgetracker.model.ticket.Globals.bin_count to skip the
+ refresh (Solr search) and just return zero for everything, since we don't
+ need bin counts for the sitemap.
+
+2. Use multiprocessing to distribute the offsets to n subprocesses.
+"""
+
+import os
+from datetime import datetime
+import argparse
+
+from jinja2 import Template
+import pylons
+import webob
+from pylons import tmpl_context as c
+from ming.orm import ThreadLocalORMSession
+
+from allura import model as M
+from allura.lib import security, utils
+from allura.scripts import ScriptTask
+
+
+MAX_SITEMAP_URLS = 50000
+BASE_URL = 'http://sourceforge.net'
+
+INDEX_TEMPLATE = """\
+<?xml version="1.0" encoding="utf-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ {% for sitemap in sitemaps -%}
+ <sitemap>
+ <loc>{{ sitemap }}</loc>
+ <lastmod>{{ now }}</lastmod>
+ </sitemap>
+ {%- endfor %}
+</sitemapindex>
+"""
+
+SITEMAP_TEMPLATE = """\
+<?xml version="1.0" encoding="utf-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ {% for loc in locs -%}
+ <url>
+ <loc>{{ loc.url }}</loc>
+ <lastmod>{{ loc.date }}</lastmod>
+ <changefreq>daily</changefreq>
+ </url>
+ {% endfor %}
+</urlset>
+"""
+
+
+class CreateSitemapFiles(ScriptTask):
+
+ @classmethod
+ def execute(cls, options):
+ # This script will indirectly call app.sidebar_menu() for every app in
+ # every project. Some of the sidebar_menu methods expect the
+ # pylons.request threadlocal object to be present. So, we're faking it.
+ #
+ # The fact that this isn't a 'real' request doesn't matter for the
+ # purposes of the sitemap.
+ pylons.request._push_object(webob.Request.blank('/'))
+
+ output_path = options.output_dir
+ if os.path.exists(output_path):
+ raise Exception('%s directory already exists.' % output_path)
+ os.mkdir(output_path)
+
+ now = datetime.utcnow().date()
+ sitemap_content_template = Template(SITEMAP_TEMPLATE)
+
+ def write_sitemap(urls, file_no):
+ sitemap_content = sitemap_content_template.render(dict(now=now, locs=urls))
+ with open(os.path.join(output_path, 'sitemap-%d.xml' % file_no), 'w') as f:
+ f.write(sitemap_content)
+
+ creds = security.Credentials.get()
+ locs = []
+ file_count = 0
+
+ nbhd_id = []
+ if options.neighborhood:
+ prefix = ['/%s/' % n for n in options.neighborhood]
+ nbhd_id = [nbhd._id for nbhd in M.Neighborhood.query.find({'url_prefix': {'$in': prefix}})]
+
+ # write sitemap files, MAX_SITEMAP_URLS per file
+ for chunk in utils.chunked_find(M.Project, {'deleted': False, 'neighborhood_id': {'$nin': nbhd_id}}):
+ for p in chunk:
+ c.project = p
+ try:
+ for s in p.sitemap(excluded_tools=['git', 'hg', 'svn']):
+ url = BASE_URL + s.url if s.url[0] == '/' else s.url
+ locs.append({'url': url,
+ 'date': p.last_updated.strftime("%Y-%m-%d")})
+
+ except Exception, e:
+ print "Error creating sitemap for project '%s': %s" %\
+ (p.shortname, e)
+ creds.clear()
+ if len(locs) >= options.urls_per_file:
+ write_sitemap(locs[:options.urls_per_file], file_count)
+ del locs[:options.urls_per_file]
+ file_count += 1
+ M.main_orm_session.clear()
+ ThreadLocalORMSession.close_all()
+ while locs:
+ write_sitemap(locs[:options.urls_per_file], file_count)
+ del locs[:options.urls_per_file]
+ file_count += 1
+ # write sitemap index file
+ if file_count:
+ sitemap_index_vars = dict(
+ now=now,
+ sitemaps=[
+ '%s/allura_sitemap/sitemap-%d.xml' % (BASE_URL, n)
+ for n in range(file_count)])
+ sitemap_index_content = Template(
+ INDEX_TEMPLATE).render(sitemap_index_vars)
+ with open(os.path.join(output_path, 'sitemap.xml'), 'w') as f:
+ f.write(sitemap_index_content)
+
+ @classmethod
+ def parser(cls):
+ class Validate(argparse.Action):
+ def __call__(self, parser, namespace, value, option_string=None):
+ value = min(value, MAX_SITEMAP_URLS)
+ setattr(namespace, self.dest, value)
+
+ parser = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument('-o', '--output-dir',
+ dest='output_dir',
+ default='/tmp/allura_sitemap',
+ help='Output directory (absolute path).'
+ '[default: %(default)s]')
+ parser.add_argument('-u', '--urls-per-file', dest='urls_per_file',
+ default=10000, type=int,
+ help='Number of URLs per sitemap file. [default: %(default)s, max: ' +
+ str(MAX_SITEMAP_URLS) + ']',
+ action=Validate)
+ parser.add_argument('-n', '--neighborhood', dest='neighborhood',
+ help="URL prefix of excluded neighborhood(s)",
+ default=None, nargs='*')
+ return parser
+
+
+def get_parser():
+ return CreateSitemapFiles.parser()
+
+if __name__ == '__main__':
+ CreateSitemapFiles.main()
http://git-wip-us.apache.org/repos/asf/allura/blob/2f81405d/Allura/docs/getting_started/administration.rst
----------------------------------------------------------------------
diff --git a/Allura/docs/getting_started/administration.rst b/Allura/docs/getting_started/administration.rst
index e5d5a10..d1b4b93 100644
--- a/Allura/docs/getting_started/administration.rst
+++ b/Allura/docs/getting_started/administration.rst
@@ -72,10 +72,10 @@ Commands can be discovered and run via the `paster` command when you are in the
Scripts are in the `scripts/` directory and run slightly differently, via `paster script`. An extra
:kbd:`--` is required to separate script arguments from paster arguments. Example::
- paster script development.ini ../scripts/create-allura-sitemap.py -- --help
+ paster script development.ini ../scripts/add_user_to_group.py -- --help
... help output ...
- paster script development.ini ../scripts/create-allura-sitemap.py -- -u 100
+ paster script development.ini ../scripts/add_user_to_group.py -- --nbhd /u/ johndoe Admin
To run these when using docker, prefix with :code:`docker-compose run taskd` and use :code:`docker-dev.ini` like::
@@ -224,15 +224,15 @@ reindex_users.py
:prog: paster script development.ini allura/scripts/reindex_users.py --
-create-allura-sitemap.py
-------------------------
+create_sitemap_files.py
+-----------------------
-*Cannot currently be run as a background task.*
+*Can be run as a background task using task name:* :code:`allura.scripts.create_sitemap_files.CreateSitemapFiles`
.. argparse::
- :file: ../../scripts/create-allura-sitemap.py
- :func: parser
- :prog: paster script development.ini ../scripts/create-allura-sitemap.py --
+ :module: allura.scripts.create_sitemap_files
+ :func: get_parser
+ :prog: paster script development.ini allura/scripts/create_sitemap_files.py --
publicize-neighborhood.py
http://git-wip-us.apache.org/repos/asf/allura/blob/2f81405d/scripts/create-allura-sitemap.py
----------------------------------------------------------------------
diff --git a/scripts/create-allura-sitemap.py b/scripts/create-allura-sitemap.py
deleted file mode 100644
index 9ab8f38..0000000
--- a/scripts/create-allura-sitemap.py
+++ /dev/null
@@ -1,175 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""
-Generate Allura sitemap xml files. You will need to configure your webserver to serve the files.
-
-This takes a while to run on a prod-sized data set. There are a couple of
-things that would make it faster, if we need/want to.
-
-1. Monkeypatch forgetracker.model.ticket.Globals.bin_count to skip the
- refresh (Solr search) and just return zero for everything, since we don't
- need bin counts for the sitemap.
-
-2. Use multiprocessing to distribute the offsets to n subprocesses.
-"""
-
-import os
-import sys
-from datetime import datetime
-from jinja2 import Template
-
-import pylons
-import webob
-from pylons import tmpl_context as c
-
-from allura import model as M
-from allura.lib import security, utils
-from ming.orm import ThreadLocalORMSession
-
-MAX_SITEMAP_URLS = 50000
-BASE_URL = 'http://sourceforge.net'
-
-INDEX_TEMPLATE = """\
-<?xml version="1.0" encoding="utf-8"?>
-<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
- {% for sitemap in sitemaps -%}
- <sitemap>
- <loc>{{ sitemap }}</loc>
- <lastmod>{{ now }}</lastmod>
- </sitemap>
- {%- endfor %}
-</sitemapindex>
-"""
-
-SITEMAP_TEMPLATE = """\
-<?xml version="1.0" encoding="utf-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
- {% for loc in locs -%}
- <url>
- <loc>{{ loc.url }}</loc>
- <lastmod>{{ loc.date }}</lastmod>
- <changefreq>daily</changefreq>
- </url>
- {% endfor %}
-</urlset>
-"""
-
-
-def main(options):
- # This script will indirectly call app.sidebar_menu() for every app in
- # every project. Some of the sidebar_menu methods expect the
- # pylons.request threadlocal object to be present. So, we're faking it.
- #
- # The fact that this isn't a 'real' request doesn't matter for the
- # purposes of the sitemap.
- pylons.request._push_object(webob.Request.blank('/'))
-
- output_path = options.output_dir
- if os.path.exists(output_path):
- sys.exit('Error: %s directory already exists.' % output_path)
- try:
- os.mkdir(output_path)
- except OSError, e:
- sys.exit("Error: Couldn't create %s:\n%s" % (output_path, e))
-
- now = datetime.utcnow().date()
- sitemap_content_template = Template(SITEMAP_TEMPLATE)
-
- def write_sitemap(urls, file_no):
- sitemap_content = sitemap_content_template.render(dict(
- now=now, locs=urls))
- with open(os.path.join(output_path, 'sitemap-%d.xml' % file_no), 'w') as f:
- f.write(sitemap_content)
-
- creds = security.Credentials.get()
- locs = []
- file_count = 0
-
- nbhd_id = []
- if options.neighborhood:
- prefix = ['/%s/' % n for n in options.neighborhood]
- nbhd_id = [nbhd._id for nbhd in M.Neighborhood.query.find({'url_prefix': {'$in': prefix}})]
-
- # write sitemap files, MAX_SITEMAP_URLS per file
- for chunk in utils.chunked_find(M.Project, {'deleted': False, 'neighborhood_id': {'$nin': nbhd_id}}):
- for p in chunk:
- c.project = p
- try:
- for s in p.sitemap(excluded_tools=['git', 'hg', 'svn']):
- url = BASE_URL + s.url if s.url[0] == '/' else s.url
- locs.append({'url': url,
- 'date': p.last_updated.strftime("%Y-%m-%d")})
-
- except Exception, e:
- print "Error creating sitemap for project '%s': %s" %\
- (p.shortname, e)
- creds.clear()
- if len(locs) >= options.urls_per_file:
- write_sitemap(locs[:options.urls_per_file], file_count)
- del locs[:options.urls_per_file]
- file_count += 1
- M.main_orm_session.clear()
- ThreadLocalORMSession.close_all()
- while locs:
- write_sitemap(locs[:options.urls_per_file], file_count)
- del locs[:options.urls_per_file]
- file_count += 1
- # write sitemap index file
- if file_count:
- sitemap_index_vars = dict(
- now=now,
- sitemaps=[
- '%s/allura_sitemap/sitemap-%d.xml' % (BASE_URL, n)
- for n in range(file_count)])
- sitemap_index_content = Template(
- INDEX_TEMPLATE).render(sitemap_index_vars)
- with open(os.path.join(output_path, 'sitemap.xml'), 'w') as f:
- f.write(sitemap_index_content)
-
-
-def parser():
- import argparse
- class Validate(argparse.Action):
- def __call__(self, parser, namespace, value, option_string=None):
- value = min(value, MAX_SITEMAP_URLS)
- setattr(namespace, self.dest, value)
-
- parser = argparse.ArgumentParser(description=__doc__,
- formatter_class=argparse.RawDescriptionHelpFormatter)
- parser.add_argument('-o', '--output-dir',
- dest='output_dir',
- default='/tmp/allura_sitemap',
- help='Output directory (absolute path).'
- '[default: %(default)s]')
- parser.add_argument('-u', '--urls-per-file', dest='urls_per_file',
- default=10000, type=int,
- help='Number of URLs per sitemap file. '
- '[default: %(default)s, max: ' +
- str(MAX_SITEMAP_URLS) + ']',
- action=Validate)
- parser.add_argument('-n', '--neighborhood', dest='neighborhood',
- help="URL prefix of excluded neighborhood(s)",
- default=None, nargs='*')
- return parser
-
-def parse_options():
- return parser().parse_args()
-
-
-if __name__ == '__main__':
- sys.exit(main(parse_options()))
[3/5] allura git commit: [#8033] make dir for scripts tests
Posted by br...@apache.org.
[#8033] make dir for scripts tests
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/c9ce7e30
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/c9ce7e30
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/c9ce7e30
Branch: refs/heads/db/8033
Commit: c9ce7e30c768824e580dc39a8ab6d9b8c5dbd62c
Parents: 2878e9c
Author: Dave Brondsema <da...@brondsema.net>
Authored: Fri Dec 11 12:22:49 2015 -0500
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Fri Dec 11 14:32:50 2015 -0500
----------------------------------------------------------------------
Allura/allura/tests/scripts/__init__.py | 0
.../tests/scripts/test_delete_projects.py | 145 +++++++++++++++++++
Allura/allura/tests/test_delete_projects.py | 145 -------------------
3 files changed, 145 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/c9ce7e30/Allura/allura/tests/scripts/__init__.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/scripts/__init__.py b/Allura/allura/tests/scripts/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/allura/blob/c9ce7e30/Allura/allura/tests/scripts/test_delete_projects.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/scripts/test_delete_projects.py b/Allura/allura/tests/scripts/test_delete_projects.py
new file mode 100644
index 0000000..77915cf
--- /dev/null
+++ b/Allura/allura/tests/scripts/test_delete_projects.py
@@ -0,0 +1,145 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from ming.odm import session, Mapper, ThreadLocalODMSession
+from mock import patch
+from pylons import app_globals as g
+from nose.tools import assert_equal
+
+from alluratest.controller import TestController
+from allura.tests.decorators import audits, out_audits
+from allura import model as M
+from allura.scripts import delete_projects
+from allura.lib import plugin
+
+
+class TestDeleteProjects(TestController):
+
+ def setUp(self):
+ super(TestDeleteProjects, self).setUp()
+ n = M.Neighborhood.query.get(name='Projects')
+ admin = M.User.by_username('test-admin')
+ self.p_shortname = 'test-delete'
+ self.proj = n.register_project(self.p_shortname, admin)
+
+ def run_script(self, options):
+ cls = delete_projects.DeleteProjects
+ opts = cls.parser().parse_args(options)
+ cls.execute(opts)
+
+ def things_related_to_project(self, pid):
+ result = []
+ ac_ids = [ac._id for ac in M.AppConfig.query.find(dict(project_id=pid))]
+ for m in Mapper.all_mappers():
+ cls = m.mapped_class
+ things = None
+ if 'project_id' in m.property_index:
+ things = cls.query.find(dict(project_id=pid)).all()
+ elif 'app_config_id' in m.property_index:
+ things = cls.query.find(dict(app_config_id={'$in': ac_ids})).all()
+ if things:
+ result.extend(things)
+ return result
+
+ def test_project_is_deleted(self):
+ p = M.Project.query.get(shortname=self.p_shortname)
+ assert p is not None, 'Can not find project to delete'
+ self.run_script(['p/{}'.format(p.shortname)])
+ session(p).expunge(p)
+ p = M.Project.query.get(shortname=p.shortname)
+ assert p is None, 'Project is not deleted'
+
+ def test_artifacts_are_deleted(self):
+ pid = M.Project.query.get(shortname=self.p_shortname)._id
+ things = self.things_related_to_project(pid)
+ assert len(things) > 0, 'No things related to project to begin with'
+ self.run_script(['p/{}'.format(self.p_shortname)])
+ things = self.things_related_to_project(pid)
+ assert len(things) == 0, 'Not all things are deleted: %s' % things
+
+ def test_subproject_is_deleted(self):
+ p = M.Project.query.get(shortname='test/sub1')
+ assert p is not None, 'Can not find subproject to delete'
+ self.run_script(['p/test/sub1'])
+ session(p).expunge(p)
+ p = M.Project.query.get(shortname='test/sub1')
+ assert p is None, 'Project is not deleted'
+ p = M.Project.query.get(shortname='test')
+ assert p is not None, 'Parent project should not be deleted'
+
+ def test_subproject_artifacts_are_deleted(self):
+ parent_pid = M.Project.query.get(shortname='test')._id
+ pid = M.Project.query.get(shortname='test/sub1')._id
+ things = self.things_related_to_project(pid)
+ assert len(things) > 0, 'No things related to subproject to begin with'
+ parent_things_before = self.things_related_to_project(parent_pid)
+ self.run_script(['p/test/sub1'])
+ things = self.things_related_to_project(pid)
+ assert len(things) == 0, 'Not all things are deleted: %s' % things
+ parent_things_after = self.things_related_to_project(parent_pid)
+ assert_equal(len(parent_things_before), len(parent_things_after))
+
+ @patch('allura.lib.plugin.solr_del_project_artifacts', autospec=True)
+ def test_solr_index_is_deleted(self, del_solr):
+ pid = M.Project.query.get(shortname=self.p_shortname)._id
+ self.run_script(['p/{}'.format(self.p_shortname)])
+ del_solr.post.assert_called_once_with(pid)
+
+ @patch.object(plugin.g, 'post_event', autospec=True)
+ def test_event_is_fired(self, post_event):
+ pid = M.Project.query.get(shortname=self.p_shortname)._id
+ self.run_script(['p/{}'.format(self.p_shortname)])
+ post_event.assert_called_once_with('project_deleted', project_id=pid, reason=None)
+
+ @patch.object(plugin.g, 'post_event', autospec=True)
+ @patch('allura.scripts.delete_projects.log', autospec=True)
+ def test_delete_with_reason(self, log, post_event):
+ p = M.Project.query.get(shortname=self.p_shortname)
+ pid = p._id
+ assert p is not None, 'Can not find project to delete'
+ self.run_script(['-r', 'The Reason', 'p/{}'.format(p.shortname)])
+ session(p).expunge(p)
+ p = M.Project.query.get(shortname=p.shortname)
+ assert p is None, 'Project is not deleted'
+ log.info.assert_called_once_with('Purging %s%s. Reason: %s', '/p/', 'test-delete', 'The Reason')
+ post_event.assert_called_once_with('project_deleted', project_id=pid, reason='The Reason')
+
+ def _disable_users(self, disable):
+ dev = M.User.by_username('test-user')
+ self.proj.add_user(dev, ['Developer'])
+ ThreadLocalODMSession.flush_all()
+ g.credentials.clear()
+ proj = u'p/{}'.format(self.p_shortname)
+ msg = u'Account disabled because project /{} is deleted. Reason: The Reason'.format(proj)
+ opts = ['-r', 'The Reason', proj]
+ if disable:
+ opts.insert(0, '--disable-users')
+ _audit = audits if disable else out_audits
+ with _audit(msg):
+ self.run_script(opts)
+ admin = M.User.by_username('test-admin')
+ dev = M.User.by_username('test-user')
+ assert admin.disabled is disable
+ assert dev.disabled is disable
+
+ @patch('allura.model.auth.request', autospec=True)
+ def test_disable_users(self, req):
+ req.url = None
+ self._disable_users(disable=True)
+
+ def test_not_disable_users(self):
+ self._disable_users(disable=False)
http://git-wip-us.apache.org/repos/asf/allura/blob/c9ce7e30/Allura/allura/tests/test_delete_projects.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_delete_projects.py b/Allura/allura/tests/test_delete_projects.py
deleted file mode 100644
index 77915cf..0000000
--- a/Allura/allura/tests/test_delete_projects.py
+++ /dev/null
@@ -1,145 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from ming.odm import session, Mapper, ThreadLocalODMSession
-from mock import patch
-from pylons import app_globals as g
-from nose.tools import assert_equal
-
-from alluratest.controller import TestController
-from allura.tests.decorators import audits, out_audits
-from allura import model as M
-from allura.scripts import delete_projects
-from allura.lib import plugin
-
-
-class TestDeleteProjects(TestController):
-
- def setUp(self):
- super(TestDeleteProjects, self).setUp()
- n = M.Neighborhood.query.get(name='Projects')
- admin = M.User.by_username('test-admin')
- self.p_shortname = 'test-delete'
- self.proj = n.register_project(self.p_shortname, admin)
-
- def run_script(self, options):
- cls = delete_projects.DeleteProjects
- opts = cls.parser().parse_args(options)
- cls.execute(opts)
-
- def things_related_to_project(self, pid):
- result = []
- ac_ids = [ac._id for ac in M.AppConfig.query.find(dict(project_id=pid))]
- for m in Mapper.all_mappers():
- cls = m.mapped_class
- things = None
- if 'project_id' in m.property_index:
- things = cls.query.find(dict(project_id=pid)).all()
- elif 'app_config_id' in m.property_index:
- things = cls.query.find(dict(app_config_id={'$in': ac_ids})).all()
- if things:
- result.extend(things)
- return result
-
- def test_project_is_deleted(self):
- p = M.Project.query.get(shortname=self.p_shortname)
- assert p is not None, 'Can not find project to delete'
- self.run_script(['p/{}'.format(p.shortname)])
- session(p).expunge(p)
- p = M.Project.query.get(shortname=p.shortname)
- assert p is None, 'Project is not deleted'
-
- def test_artifacts_are_deleted(self):
- pid = M.Project.query.get(shortname=self.p_shortname)._id
- things = self.things_related_to_project(pid)
- assert len(things) > 0, 'No things related to project to begin with'
- self.run_script(['p/{}'.format(self.p_shortname)])
- things = self.things_related_to_project(pid)
- assert len(things) == 0, 'Not all things are deleted: %s' % things
-
- def test_subproject_is_deleted(self):
- p = M.Project.query.get(shortname='test/sub1')
- assert p is not None, 'Can not find subproject to delete'
- self.run_script(['p/test/sub1'])
- session(p).expunge(p)
- p = M.Project.query.get(shortname='test/sub1')
- assert p is None, 'Project is not deleted'
- p = M.Project.query.get(shortname='test')
- assert p is not None, 'Parent project should not be deleted'
-
- def test_subproject_artifacts_are_deleted(self):
- parent_pid = M.Project.query.get(shortname='test')._id
- pid = M.Project.query.get(shortname='test/sub1')._id
- things = self.things_related_to_project(pid)
- assert len(things) > 0, 'No things related to subproject to begin with'
- parent_things_before = self.things_related_to_project(parent_pid)
- self.run_script(['p/test/sub1'])
- things = self.things_related_to_project(pid)
- assert len(things) == 0, 'Not all things are deleted: %s' % things
- parent_things_after = self.things_related_to_project(parent_pid)
- assert_equal(len(parent_things_before), len(parent_things_after))
-
- @patch('allura.lib.plugin.solr_del_project_artifacts', autospec=True)
- def test_solr_index_is_deleted(self, del_solr):
- pid = M.Project.query.get(shortname=self.p_shortname)._id
- self.run_script(['p/{}'.format(self.p_shortname)])
- del_solr.post.assert_called_once_with(pid)
-
- @patch.object(plugin.g, 'post_event', autospec=True)
- def test_event_is_fired(self, post_event):
- pid = M.Project.query.get(shortname=self.p_shortname)._id
- self.run_script(['p/{}'.format(self.p_shortname)])
- post_event.assert_called_once_with('project_deleted', project_id=pid, reason=None)
-
- @patch.object(plugin.g, 'post_event', autospec=True)
- @patch('allura.scripts.delete_projects.log', autospec=True)
- def test_delete_with_reason(self, log, post_event):
- p = M.Project.query.get(shortname=self.p_shortname)
- pid = p._id
- assert p is not None, 'Can not find project to delete'
- self.run_script(['-r', 'The Reason', 'p/{}'.format(p.shortname)])
- session(p).expunge(p)
- p = M.Project.query.get(shortname=p.shortname)
- assert p is None, 'Project is not deleted'
- log.info.assert_called_once_with('Purging %s%s. Reason: %s', '/p/', 'test-delete', 'The Reason')
- post_event.assert_called_once_with('project_deleted', project_id=pid, reason='The Reason')
-
- def _disable_users(self, disable):
- dev = M.User.by_username('test-user')
- self.proj.add_user(dev, ['Developer'])
- ThreadLocalODMSession.flush_all()
- g.credentials.clear()
- proj = u'p/{}'.format(self.p_shortname)
- msg = u'Account disabled because project /{} is deleted. Reason: The Reason'.format(proj)
- opts = ['-r', 'The Reason', proj]
- if disable:
- opts.insert(0, '--disable-users')
- _audit = audits if disable else out_audits
- with _audit(msg):
- self.run_script(opts)
- admin = M.User.by_username('test-admin')
- dev = M.User.by_username('test-user')
- assert admin.disabled is disable
- assert dev.disabled is disable
-
- @patch('allura.model.auth.request', autospec=True)
- def test_disable_users(self, req):
- req.url = None
- self._disable_users(disable=True)
-
- def test_not_disable_users(self):
- self._disable_users(disable=False)
[4/5] allura git commit: [#8033] tests for create_sitemap_files.py
Posted by br...@apache.org.
[#8033] tests for create_sitemap_files.py
* remove unused template files
* better config options for output domain & url dir
* could not actually test for c.app problem since it'd be very compilicated to run
script as a paster command and also have sample data set up and also have `c` set
as the right type of object
* it validates XML and stuff though :)
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/d274bc6f
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/d274bc6f
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/d274bc6f
Branch: refs/heads/db/8033
Commit: d274bc6fef913d1b8447df238fad01f726e5d3e6
Parents: c9ce7e3
Author: Dave Brondsema <da...@brondsema.net>
Authored: Fri Dec 11 14:30:19 2015 -0500
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Fri Dec 11 14:32:50 2015 -0500
----------------------------------------------------------------------
Allura/allura/scripts/create_sitemap_files.py | 9 ++-
Allura/allura/templates/sitemap.xml | 28 ----------
Allura/allura/templates/sitemap_index.xml | 28 ----------
.../tests/scripts/test_create_sitemap_files.py | 58 ++++++++++++++++++++
4 files changed, 64 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/d274bc6f/Allura/allura/scripts/create_sitemap_files.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/create_sitemap_files.py b/Allura/allura/scripts/create_sitemap_files.py
index e4d98a9..bd34320 100644
--- a/Allura/allura/scripts/create_sitemap_files.py
+++ b/Allura/allura/scripts/create_sitemap_files.py
@@ -37,6 +37,7 @@ import pylons
import webob
from pylons import tmpl_context as c
from ming.orm import ThreadLocalORMSession
+from tg import config
from allura import model as M
from allura.lib import security, utils
@@ -44,7 +45,6 @@ from allura.scripts import ScriptTask
MAX_SITEMAP_URLS = 50000
-BASE_URL = 'http://sourceforge.net'
INDEX_TEMPLATE = """\
<?xml version="1.0" encoding="utf-8"?>
@@ -112,7 +112,7 @@ class CreateSitemapFiles(ScriptTask):
c.project = p
try:
for s in p.sitemap(excluded_tools=['git', 'hg', 'svn']):
- url = BASE_URL + s.url if s.url[0] == '/' else s.url
+ url = config['base_url'] + s.url if s.url[0] == '/' else s.url
locs.append({'url': url,
'date': p.last_updated.strftime("%Y-%m-%d")})
@@ -135,7 +135,7 @@ class CreateSitemapFiles(ScriptTask):
sitemap_index_vars = dict(
now=now,
sitemaps=[
- '%s/allura_sitemap/sitemap-%d.xml' % (BASE_URL, n)
+ '%s%s/sitemap-%d.xml' % (config['base_url'], options.url_dir, n)
for n in range(file_count)])
sitemap_index_content = Template(
INDEX_TEMPLATE).render(sitemap_index_vars)
@@ -164,6 +164,9 @@ class CreateSitemapFiles(ScriptTask):
parser.add_argument('-n', '--neighborhood', dest='neighborhood',
help="URL prefix of excluded neighborhood(s)",
default=None, nargs='*')
+ parser.add_argument('--url-dir', dest='url_dir',
+ default='/allura_sitemap',
+ help='URL directory in which the files will be served from')
return parser
http://git-wip-us.apache.org/repos/asf/allura/blob/d274bc6f/Allura/allura/templates/sitemap.xml
----------------------------------------------------------------------
diff --git a/Allura/allura/templates/sitemap.xml b/Allura/allura/templates/sitemap.xml
deleted file mode 100644
index d139349..0000000
--- a/Allura/allura/templates/sitemap.xml
+++ /dev/null
@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
- {% for loc in locs %}
- <url>
- <loc>{{ loc }}</loc>
- <lastmod>{{ now }}</lastmod>
- <changefreq>daily</changefreq>
- </url>
- {% endfor %}
-</urlset>
http://git-wip-us.apache.org/repos/asf/allura/blob/d274bc6f/Allura/allura/templates/sitemap_index.xml
----------------------------------------------------------------------
diff --git a/Allura/allura/templates/sitemap_index.xml b/Allura/allura/templates/sitemap_index.xml
deleted file mode 100644
index 9b1e244..0000000
--- a/Allura/allura/templates/sitemap_index.xml
+++ /dev/null
@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
- {% for sitemap in sitemaps %}
- <sitemap>
- <loc>{{ sitemap }}</loc>
- <lastmod>{{ now }}</lastmod>
- <changefreq>daily</changefreq>
- </sitemap>
- {% endfor %}
-</sitemapindex>
http://git-wip-us.apache.org/repos/asf/allura/blob/d274bc6f/Allura/allura/tests/scripts/test_create_sitemap_files.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/scripts/test_create_sitemap_files.py b/Allura/allura/tests/scripts/test_create_sitemap_files.py
new file mode 100644
index 0000000..240d7ba
--- /dev/null
+++ b/Allura/allura/tests/scripts/test_create_sitemap_files.py
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from shutil import rmtree
+import xml.etree.ElementTree as ET
+
+from pylons import tmpl_context as c
+from nose.tools import assert_in
+from testfixtures import TempDirectory
+
+from alluratest.controller import setup_basic_test
+from allura import model as M
+from allura.lib import helpers as h
+from allura.scripts.create_sitemap_files import CreateSitemapFiles
+
+
+class TestCreateSitemapFiles(object):
+
+ def setUp(self):
+ setup_basic_test()
+
+ def run_script(self, options):
+ cls = CreateSitemapFiles
+ opts = cls.parser().parse_args(options)
+ with h.push_config(c, user=M.User.anonymous()): # tasks & scripts have c.user set
+ cls.execute(opts)
+
+ def test_create(self):
+ with TempDirectory() as tmpdir:
+ rmtree(tmpdir.path) # needs to be non-existent for the script
+ self.run_script(['-o', tmpdir.path])
+
+ tmpdir.check('sitemap-0.xml', 'sitemap.xml')
+
+ xml_index = ET.parse(os.path.join(tmpdir.path, 'sitemap.xml'))
+ ns = {'ns0': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
+ locs = [loc.text for loc in xml_index.findall('ns0:sitemap/ns0:loc', ns)]
+ assert_in('http://localhost:8080/allura_sitemap/sitemap-0.xml', locs)
+
+ xml_0 = ET.parse(os.path.join(tmpdir.path, 'sitemap-0.xml'))
+ urls = [loc.text for loc in xml_0.findall('ns0:url/ns0:loc', ns)]
+ assert_in('http://localhost:8080/p/wiki/', urls)
+ assert_in('http://localhost:8080/p/test/sub1/', urls)
[5/5] allura git commit: [#8033] more careful check for c.app
Posted by br...@apache.org.
[#8033] more careful check for c.app
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/e1aeec7f
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/e1aeec7f
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/e1aeec7f
Branch: refs/heads/db/8033
Commit: e1aeec7f0b33c70cbbdec8027ed799fdd948be02
Parents: d274bc6
Author: Dave Brondsema <da...@brondsema.net>
Authored: Fri Dec 11 12:12:07 2015 -0500
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Fri Dec 11 14:32:50 2015 -0500
----------------------------------------------------------------------
Allura/allura/model/project.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/e1aeec7f/Allura/allura/model/project.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/project.py b/Allura/allura/model/project.py
index 1dfd3a3..28315ad 100644
--- a/Allura/allura/model/project.py
+++ b/Allura/allura/model/project.py
@@ -549,7 +549,7 @@ class Project(SearchIndexable, MappedClass, ActivityNode, ActivityObject):
log.exception('AppConfig %s references invalid tool %s',
ac._id, ac.tool_name)
continue
- if c.app and c.app.config._id == ac._id:
+ if getattr(c, 'app', None) and c.app.config._id == ac._id:
# slight performance gain (depending on the app) by using the current app if we're on it
app = c.app
else: