You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2024/03/20 21:42:50 UTC
(allura) branch db/8539 created (now a7c783e6d)
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a change to branch db/8539
in repository https://gitbox.apache.org/repos/asf/allura.git
at a7c783e6d [#8539] code simplifications
This branch includes the following new commits:
new c5a60409d [#8539] upgrade ruff
new 2dc7b9fbd [#8539] remove old teamforge import script, which had lots of hardcoded specifics
new 99daa1fdf [#8539] add bandit checks
new 5c76a8524 [#8539] add some pylint checks
new a7c783e6d [#8539] code simplifications
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(allura) 04/05: [#8539] add some pylint checks
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8539
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 5c76a8524570698a5bc6b1e91ee0319ef33b5c41
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Wed Mar 20 17:10:45 2024 -0400
[#8539] add some pylint checks
---
Allura/allura/command/base.py | 2 +-
Allura/allura/lib/helpers.py | 2 +-
Allura/allura/lib/patches.py | 2 +-
Allura/allura/tests/functional/test_discuss.py | 3 +--
Allura/allura/websetup/bootstrap.py | 2 +-
ruff.toml | 4 ++++
scripts/ApacheAccessHandler.py | 2 +-
7 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/Allura/allura/command/base.py b/Allura/allura/command/base.py
index 65b7c758e..de09018c9 100644
--- a/Allura/allura/command/base.py
+++ b/Allura/allura/command/base.py
@@ -87,7 +87,7 @@ class Command(command.Command, metaclass=MetaParserDocstring):
return tg.config
def basic_setup(self):
- global log, M
+ global log, M # noqa: PLW0603
if self.args[0]:
# Probably being called from the command line - load the config
# file
diff --git a/Allura/allura/lib/helpers.py b/Allura/allura/lib/helpers.py
index 5f5c41bdf..26d031446 100644
--- a/Allura/allura/lib/helpers.py
+++ b/Allura/allura/lib/helpers.py
@@ -32,7 +32,7 @@ import json
import logging
import string
import random
-import pickle as pickle
+import pickle
from hashlib import sha1
from datetime import datetime, timedelta
from collections import defaultdict, OrderedDict
diff --git a/Allura/allura/lib/patches.py b/Allura/allura/lib/patches.py
index 7c6e416eb..964624d0b 100644
--- a/Allura/allura/lib/patches.py
+++ b/Allura/allura/lib/patches.py
@@ -29,7 +29,7 @@ import six
_patched = False
def apply():
- global _patched
+ global _patched # noqa: PLW0603
if _patched:
return
_patched = True
diff --git a/Allura/allura/tests/functional/test_discuss.py b/Allura/allura/tests/functional/test_discuss.py
index f87f4352d..100f9b276 100644
--- a/Allura/allura/tests/functional/test_discuss.py
+++ b/Allura/allura/tests/functional/test_discuss.py
@@ -426,8 +426,7 @@ class TestAttachment(TestDiscussBase):
if 'attachment' in alink['href']:
alink = str(alink['href'])
return alink
- else:
- assert False, 'attachment link not found'
+ assert False, 'attachment link not found'
def test_attach(self):
r = self.app.post(self.post_link + 'attach',
diff --git a/Allura/allura/websetup/bootstrap.py b/Allura/allura/websetup/bootstrap.py
index eadd141e9..21cfa6781 100644
--- a/Allura/allura/websetup/bootstrap.py
+++ b/Allura/allura/websetup/bootstrap.py
@@ -56,7 +56,7 @@ def bootstrap(command, conf, vars):
REGISTRY.register(ew.widget_context,
ew.core.WidgetContext('http', ew.ResourceManager()))
- create_test_data = asbool(os.getenv('ALLURA_TEST_DATA', True))
+ create_test_data = asbool(os.getenv('ALLURA_TEST_DATA', 'true'))
# if this is a test_run, skip user project creation to save time
make_user_projects = not test_run
diff --git a/ruff.toml b/ruff.toml
index 7da944339..4337590c5 100644
--- a/ruff.toml
+++ b/ruff.toml
@@ -33,6 +33,9 @@ lint.select = [
"G010", # logging.warn
"T10", # debugger breakpoints
"T20", # print()
+ "PLC",
+ "PLE",
+ "PLW",
"FA", # future annotations (to ensure compatibility with `target-version`)
]
@@ -57,6 +60,7 @@ lint.ignore = [
'S324', # md5 & sha1
'S603', # subprocess
'S607', # partial path
+ 'PLW2901', # loop var overwritten
]
[lint.per-file-ignores]
diff --git a/scripts/ApacheAccessHandler.py b/scripts/ApacheAccessHandler.py
index 217e4ecf6..fc10499a1 100644
--- a/scripts/ApacheAccessHandler.py
+++ b/scripts/ApacheAccessHandler.py
@@ -52,7 +52,7 @@ def load_requests_lib(req):
exec(compile(open(activate_this, "rb").read(), activate_this, 'exec'), {'__file__': activate_this}) # noqa: S102
except Exception as e:
log(req, "Couldn't activate venv via {}: {}".format(activate_this, repr(e)))
- global requests
+ global requests # noqa: PLW0603
import requests as requests_lib
requests = requests_lib
(allura) 05/05: [#8539] code simplifications
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8539
in repository https://gitbox.apache.org/repos/asf/allura.git
commit a7c783e6d18be9881311bd124c7842679bad3ae0
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Wed Mar 20 17:16:19 2024 -0400
[#8539] code simplifications
---
Allura/allura/command/taskd_cleanup.py | 4 +--
Allura/allura/controllers/project.py | 2 +-
Allura/allura/controllers/rest.py | 2 +-
Allura/allura/controllers/site_admin.py | 2 +-
Allura/allura/lib/utils.py | 2 +-
Allura/allura/tests/decorators.py | 8 ++----
.../allura/tests/functional/test_neighborhood.py | 4 +--
.../tests/functional/test_personal_dashboard.py | 29 +++++++++++-----------
.../allura/tests/functional/test_user_profile.py | 17 ++++++-------
Allura/allura/tests/model/test_project.py | 11 ++++----
Allura/allura/tests/test_commands.py | 4 +--
Allura/allura/tests/test_helpers.py | 5 ++--
Allura/allura/tests/test_tasks.py | 10 ++++----
Allura/allura/tests/test_webhooks.py | 5 ++--
Allura/allura/tests/unit/test_session.py | 28 ++++++++++-----------
ForgeActivity/forgeactivity/main.py | 2 +-
.../tests/unit/test_root_controller.py | 4 +--
ForgeImporters/forgeimporters/github/__init__.py | 2 +-
ForgeTracker/forgetracker/tracker_main.py | 2 +-
ruff.toml | 5 ++++
20 files changed, 71 insertions(+), 77 deletions(-)
diff --git a/Allura/allura/command/taskd_cleanup.py b/Allura/allura/command/taskd_cleanup.py
index 0f4ec913e..232773f93 100644
--- a/Allura/allura/command/taskd_cleanup.py
+++ b/Allura/allura/command/taskd_cleanup.py
@@ -154,7 +154,7 @@ class TaskdCleanupCommand(base.Command):
def _check_taskd_status(self, pid):
for i in range(self.options.num_retry):
- retry = False if i == 0 else True
+ retry = i != 0
status = self._taskd_status(pid, retry)
if ('taskd pid %s' % pid) in status:
return 'OK'
@@ -164,7 +164,7 @@ class TaskdCleanupCommand(base.Command):
def _check_task(self, taskd_pid, task):
for i in range(self.options.num_retry):
- retry = False if i == 0 else True
+ retry = i != 0
status = self._taskd_status(taskd_pid, retry)
line = 'taskd pid {} is currently handling task {}'.format(
taskd_pid, task)
diff --git a/Allura/allura/controllers/project.py b/Allura/allura/controllers/project.py
index f4d8c9865..f04fa5b62 100644
--- a/Allura/allura/controllers/project.py
+++ b/Allura/allura/controllers/project.py
@@ -564,7 +564,7 @@ class NeighborhoodAdminController:
set_nav(self.neighborhood)
c.overview_form = W.neighborhood_overview_form
allow_undelete = asbool(config.get('allow_project_undelete', True))
- allow_wiki_as_root = True if get_default_wiki_page() else False
+ allow_wiki_as_root = bool(get_default_wiki_page())
return dict(
neighborhood=self.neighborhood,
diff --git a/Allura/allura/controllers/rest.py b/Allura/allura/controllers/rest.py
index f077b3d40..255348208 100644
--- a/Allura/allura/controllers/rest.py
+++ b/Allura/allura/controllers/rest.py
@@ -189,7 +189,7 @@ class Oauth1Validator(oauthlib.oauth1.RequestValidator):
if request.environ.get('paste.testing'):
# test suite is running
return False
- elif asbool(config.get('debug')) and config['base_url'].startswith('http://'):
+ elif asbool(config.get('debug')) and config['base_url'].startswith('http://'): # noqa: SIM103
# development w/o https
return False
else:
diff --git a/Allura/allura/controllers/site_admin.py b/Allura/allura/controllers/site_admin.py
index 07009028b..252aea353 100644
--- a/Allura/allura/controllers/site_admin.py
+++ b/Allura/allura/controllers/site_admin.py
@@ -184,7 +184,7 @@ class SiteAdminController:
try:
end_dt = datetime.strptime(end_dt, '%Y/%m/%d %H:%M:%S')
except ValueError:
- end_dt = start_dt - timedelta(days=3) if not end_dt else end_dt
+ end_dt = end_dt if end_dt else start_dt - timedelta(days=3)
start = bson.ObjectId.from_datetime(start_dt)
end = bson.ObjectId.from_datetime(end_dt)
nb = M.Neighborhood.query.get(name='Users')
diff --git a/Allura/allura/lib/utils.py b/Allura/allura/lib/utils.py
index 0cf6b8c3c..de33f559b 100644
--- a/Allura/allura/lib/utils.py
+++ b/Allura/allura/lib/utils.py
@@ -337,7 +337,7 @@ class AntiSpam:
if params is None:
params = request.params
new_params = dict(params)
- if not request.method == 'GET':
+ if request.method != 'GET':
obj = None
try:
new_params.pop('timestamp', None)
diff --git a/Allura/allura/tests/decorators.py b/Allura/allura/tests/decorators.py
index 6b1ea527e..7311f58be 100644
--- a/Allura/allura/tests/decorators.py
+++ b/Allura/allura/tests/decorators.py
@@ -109,12 +109,8 @@ class raises:
def __exit__(self, exc_type, exc_val, exc_t):
if exc_type:
self.exc = exc_val
- if issubclass(exc_type, self.ExcType):
- # ok
- return True
- else:
- # root exception will be raised, untouched
- return False
+ # otherwise root exception will be raised, untouched
+ return issubclass(exc_type, self.ExcType)
else:
raise AssertionError('Did not raise %s' % self.ExcType)
diff --git a/Allura/allura/tests/functional/test_neighborhood.py b/Allura/allura/tests/functional/test_neighborhood.py
index ec1633172..e7e675920 100644
--- a/Allura/allura/tests/functional/test_neighborhood.py
+++ b/Allura/allura/tests/functional/test_neighborhood.py
@@ -342,7 +342,7 @@ class TestNeighborhood(TestController):
neighborhood = M.Neighborhood.query.get(name='Adobe')
neighborhood.features['css'] = 'picker'
r = self.app.get('/adobe/')
- while isinstance(r.response, HTTPFound) or isinstance(r.response, HTTPMovedPermanently):
+ while isinstance(r.response, (HTTPFound, HTTPMovedPermanently)):
r = r.follow()
assert test_css in r
r = self.app.get('/adobe/_admin/overview',
@@ -352,7 +352,7 @@ class TestNeighborhood(TestController):
neighborhood = M.Neighborhood.query.get(name='Adobe')
neighborhood.features['css'] = 'custom'
r = self.app.get('/adobe/')
- while isinstance(r.response, HTTPFound) or isinstance(r.response, HTTPMovedPermanently):
+ while isinstance(r.response, (HTTPFound, HTTPMovedPermanently)):
r = r.follow()
assert test_css in r
r = self.app.get('/adobe/_admin/overview',
diff --git a/Allura/allura/tests/functional/test_personal_dashboard.py b/Allura/allura/tests/functional/test_personal_dashboard.py
index 9a64ba7cc..b6a68eb3d 100644
--- a/Allura/allura/tests/functional/test_personal_dashboard.py
+++ b/Allura/allura/tests/functional/test_personal_dashboard.py
@@ -50,21 +50,20 @@ class TestPersonalDashboard(TestController):
return m
eps = list(map(ep, ['a', 'b', 'c', 'd']))
order = {'personal_dashboard_sections.order': 'b, d,c , f '}
- with mock.patch('allura.lib.helpers.iter_entry_points') as iep:
- with mock.patch.dict(tg.config, order):
- iep.return_value = eps
- sections = SectionsUtil.load_sections('personal_dashboard')
- assert sections == [
- eps[1].load(),
- eps[3].load(),
- eps[2].load(),
- eps[0].load()]
- r = self.app.get('/dashboard')
- assert 'Section a' in r.text
- assert 'Section b' in r.text
- assert 'Section c' in r.text
- assert 'Section d' in r.text
- assert 'Section f' not in r.text
+ with mock.patch('allura.lib.helpers.iter_entry_points') as iep, mock.patch.dict(tg.config, order):
+ iep.return_value = eps
+ sections = SectionsUtil.load_sections('personal_dashboard')
+ assert sections == [
+ eps[1].load(),
+ eps[3].load(),
+ eps[2].load(),
+ eps[0].load()]
+ r = self.app.get('/dashboard')
+ assert 'Section a' in r.text
+ assert 'Section b' in r.text
+ assert 'Section c' in r.text
+ assert 'Section d' in r.text
+ assert 'Section f' not in r.text
class TestTicketsSection(TrackerTestController):
diff --git a/Allura/allura/tests/functional/test_user_profile.py b/Allura/allura/tests/functional/test_user_profile.py
index e6de739d1..230bafb95 100644
--- a/Allura/allura/tests/functional/test_user_profile.py
+++ b/Allura/allura/tests/functional/test_user_profile.py
@@ -48,15 +48,14 @@ class TestUserProfileSections(TestController):
order = {'user_profile_sections.order': 'b, d,c , f '}
if hasattr(type(app), '_sections'):
delattr(type(app), '_sections')
- with mock.patch('allura.lib.helpers.iter_entry_points') as iep:
- with mock.patch.dict(tg.config, order):
- iep.return_value = eps
- sections = app.profile_sections
- assert sections == [
- eps[1].load(),
- eps[3].load(),
- eps[2].load(),
- eps[0].load()]
+ with mock.patch('allura.lib.helpers.iter_entry_points') as iep, mock.patch.dict(tg.config, order):
+ iep.return_value = eps
+ sections = app.profile_sections
+ assert sections == [
+ eps[1].load(),
+ eps[3].load(),
+ eps[2].load(),
+ eps[0].load()]
r = self.app.get('/u/test-user/profile')
assert 'Section a' in r.text
assert 'Section b' in r.text
diff --git a/Allura/allura/tests/model/test_project.py b/Allura/allura/tests/model/test_project.py
index b5850b87c..fc7b2bb02 100644
--- a/Allura/allura/tests/model/test_project.py
+++ b/Allura/allura/tests/model/test_project.py
@@ -120,12 +120,11 @@ class TestProjectModel:
def test_subproject(self):
project = M.Project.query.get(shortname='test')
- with td.raises(ToolError):
- with patch('allura.lib.plugin.ProjectRegistrationProvider') as Provider:
- Provider.get().shortname_validator.to_python.side_effect = Invalid(
- 'name', 'value', {})
- # name doesn't validate
- sp = project.new_subproject('test-proj-nose')
+ with td.raises(ToolError), patch('allura.lib.plugin.ProjectRegistrationProvider') as Provider:
+ Provider.get().shortname_validator.to_python.side_effect = Invalid(
+ 'name', 'value', {})
+ # name doesn't validate
+ sp = project.new_subproject('test-proj-nose')
sp = project.new_subproject('test-proj-nose')
spp = sp.new_subproject('spp')
ThreadLocalODMSession.flush_all()
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index 7ad19a1f5..ed304676c 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -435,14 +435,14 @@ def test_status_log_retries():
cmd._taskd_status.return_value = ''
cmd.options = Mock(num_retry=10)
cmd._check_taskd_status(123)
- expected_calls = [call(123, False if i == 0 else True) for i in range(10)]
+ expected_calls = [call(123, i != 0) for i in range(10)]
assert cmd._taskd_status.mock_calls == expected_calls
cmd._taskd_status = Mock()
cmd._taskd_status.return_value = ''
cmd.options = Mock(num_retry=3)
cmd._check_task(123, Mock())
- expected_calls = [call(123, False if i == 0 else True) for i in range(3)]
+ expected_calls = [call(123, i != 0) for i in range(3)]
assert cmd._taskd_status.mock_calls == expected_calls
diff --git a/Allura/allura/tests/test_helpers.py b/Allura/allura/tests/test_helpers.py
index 99eae9ce4..71318eef0 100644
--- a/Allura/allura/tests/test_helpers.py
+++ b/Allura/allura/tests/test_helpers.py
@@ -542,9 +542,8 @@ def test_login_overlay():
raise HTTPUnauthorized()
with h.login_overlay(exceptions=['foo']):
raise HTTPUnauthorized()
- with td.raises(HTTPUnauthorized):
- with h.login_overlay(exceptions=['foobar']):
- raise HTTPUnauthorized()
+ with td.raises(HTTPUnauthorized), h.login_overlay(exceptions=['foobar']):
+ raise HTTPUnauthorized()
class TestIterEntryPoints(TestCase):
diff --git a/Allura/allura/tests/test_tasks.py b/Allura/allura/tests/test_tasks.py
index 54e936ec6..81fb4123d 100644
--- a/Allura/allura/tests/test_tasks.py
+++ b/Allura/allura/tests/test_tasks.py
@@ -584,11 +584,11 @@ class TestNotificationTasks:
setup_global_objects()
def test_delivers_messages(self):
- with mock.patch.object(M.Mailbox, 'deliver') as deliver:
- with mock.patch.object(M.Mailbox, 'fire_ready') as fire_ready:
- notification_tasks.notify('42', ['52'], 'none')
- deliver.assert_called_with('42', ['52'], 'none')
- fire_ready.assert_called_with()
+ with mock.patch.object(M.Mailbox, 'deliver') as deliver, \
+ mock.patch.object(M.Mailbox, 'fire_ready') as fire_ready:
+ notification_tasks.notify('42', ['52'], 'none')
+ deliver.assert_called_with('42', ['52'], 'none')
+ fire_ready.assert_called_with()
@event_handler('my_event')
diff --git a/Allura/allura/tests/test_webhooks.py b/Allura/allura/tests/test_webhooks.py
index 63935734d..e8432415d 100644
--- a/Allura/allura/tests/test_webhooks.py
+++ b/Allura/allura/tests/test_webhooks.py
@@ -564,9 +564,8 @@ class TestRepoPushWebhookSender(TestWebhookBase):
def test_get_payload(self):
sender = RepoPushWebhookSender()
_ci = lambda x: MagicMock(webhook_info={'id': str(x)}, parent_ids=['0'])
- with patch.object(self.git.repo, 'commit', new=_ci):
- with h.push_config(c, app=self.git):
- result = sender.get_payload(commit_ids=['1', '2', '3'], ref='ref')
+ with patch.object(self.git.repo, 'commit', new=_ci), h.push_config(c, app=self.git):
+ result = sender.get_payload(commit_ids=['1', '2', '3'], ref='ref')
expected_result = {
'size': 3,
'commits': [{'id': '1'}, {'id': '2'}, {'id': '3'}],
diff --git a/Allura/allura/tests/unit/test_session.py b/Allura/allura/tests/unit/test_session.py
index e2042c039..282b5678e 100644
--- a/Allura/allura/tests/unit/test_session.py
+++ b/Allura/allura/tests/unit/test_session.py
@@ -46,14 +46,13 @@ def test_extensions_cm():
def test_extensions_cm_raises():
session = mock.Mock(_kwargs=dict(extensions=[]))
extension = mock.Mock()
- with td.raises(ValueError):
- with substitute_extensions(session, [extension]) as sess:
- session.flush.side_effect = AttributeError
- assert session.flush.call_count == 1
- assert session.close.call_count == 1
- assert sess == session
- assert sess._kwargs['extensions'] == [extension]
- raise ValueError('test')
+ with td.raises(ValueError), substitute_extensions(session, [extension]) as sess:
+ session.flush.side_effect = AttributeError
+ assert session.flush.call_count == 1
+ assert session.close.call_count == 1
+ assert sess == session
+ assert sess._kwargs['extensions'] == [extension]
+ raise ValueError('test')
assert session.flush.call_count == 1
assert session.close.call_count == 1
assert session._kwargs['extensions'] == []
@@ -62,13 +61,12 @@ def test_extensions_cm_raises():
def test_extensions_cm_flush_raises():
session = mock.Mock(_kwargs=dict(extensions=[]))
extension = mock.Mock()
- with td.raises(AttributeError):
- with substitute_extensions(session, [extension]) as sess:
- session.flush.side_effect = AttributeError
- assert session.flush.call_count == 1
- assert session.close.call_count == 1
- assert sess == session
- assert sess._kwargs['extensions'] == [extension]
+ with td.raises(AttributeError), substitute_extensions(session, [extension]) as sess:
+ session.flush.side_effect = AttributeError
+ assert session.flush.call_count == 1
+ assert session.close.call_count == 1
+ assert sess == session
+ assert sess._kwargs['extensions'] == [extension]
assert session.flush.call_count == 2
assert session.close.call_count == 1
assert session._kwargs['extensions'] == []
diff --git a/ForgeActivity/forgeactivity/main.py b/ForgeActivity/forgeactivity/main.py
index 5d85145da..de05a1944 100644
--- a/ForgeActivity/forgeactivity/main.py
+++ b/ForgeActivity/forgeactivity/main.py
@@ -168,7 +168,7 @@ class ForgeActivityController(BaseController):
followee=followee,
following=following,
timeline=filtered_timeline,
- noindex=False if filtered_timeline else True,
+ noindex=not filtered_timeline,
page=page,
limit=limit,
has_more=has_more,
diff --git a/ForgeFeedback/forgefeedback/tests/unit/test_root_controller.py b/ForgeFeedback/forgefeedback/tests/unit/test_root_controller.py
index fd04a04d2..0df97874d 100644
--- a/ForgeFeedback/forgefeedback/tests/unit/test_root_controller.py
+++ b/ForgeFeedback/forgefeedback/tests/unit/test_root_controller.py
@@ -35,10 +35,10 @@ class TestFeedbackApp(FeedbackTestWithModel):
def test_index(self):
reviews = feedback_main.RootController().index()
- assert True if not reviews['user_has_already_reviewed'] else False
+ assert bool(not reviews['user_has_already_reviewed'])
create_feedbacks()
reviews = feedback_main.RootController().index()
- assert True if reviews['user_has_already_reviewed'] else False
+ assert bool(reviews['user_has_already_reviewed'])
def test_feedback(self):
create_feedbacks()
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index f14d215f7..57deee4c4 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -208,7 +208,7 @@ def oauth_app_basic_auth(config):
def valid_access_token(access_token, scopes_required=None):
tok_details = access_token_details(access_token)
- if not tok_details.status_code == 200:
+ if tok_details.status_code != 200:
return False
if scopes_required and not all(scope_req in tok_details.json()['scopes']
for scope_req in scopes_required):
diff --git a/ForgeTracker/forgetracker/tracker_main.py b/ForgeTracker/forgetracker/tracker_main.py
index e70a61d7c..ad6ab614c 100644
--- a/ForgeTracker/forgetracker/tracker_main.py
+++ b/ForgeTracker/forgetracker/tracker_main.py
@@ -1329,7 +1329,7 @@ class changelog:
if key in self.originals:
orig_value = self.originals[key]
curr_value = self.data[key]
- if not orig_value == curr_value:
+ if orig_value != curr_value:
t.append((key, (orig_value, curr_value)))
return t
diff --git a/ruff.toml b/ruff.toml
index 4337590c5..6df46f41d 100644
--- a/ruff.toml
+++ b/ruff.toml
@@ -36,6 +36,11 @@ lint.select = [
"PLC",
"PLE",
"PLW",
+ "SIM101",
+ "SIM103",
+ "SIM117",
+ "SIM2",
+ "SIM401",
"FA", # future annotations (to ensure compatibility with `target-version`)
]
(allura) 03/05: [#8539] add bandit checks
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8539
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 99daa1fdfc121810da84be60ad75c7256f386949
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Wed Mar 20 16:37:16 2024 -0400
[#8539] add bandit checks
---
Allura/allura/command/script.py | 2 +-
Allura/allura/command/taskd.py | 2 +-
Allura/allura/config/middleware.py | 2 +-
Allura/allura/controllers/rest.py | 2 +-
Allura/allura/lib/decorators.py | 2 +-
Allura/allura/lib/helpers.py | 2 +-
Allura/allura/lib/import_api.py | 2 +-
Allura/allura/lib/phone/nexmo.py | 4 ++--
Allura/allura/lib/plugin.py | 5 +++--
Allura/allura/lib/project_create_helpers.py | 2 +-
Allura/allura/model/index.py | 2 +-
Allura/allura/model/notification.py | 4 ++--
Allura/allura/scripts/trac_export.py | 6 +++---
.../allura/tests/functional/test_neighborhood.py | 7 ++-----
Allura/allura/tests/unit/phone/test_nexmo.py | 16 ++++++++++++----
Allura/allura/websetup/bootstrap.py | 2 +-
Allura/ldap-setup.py | 2 +-
Allura/setup.py | 2 +-
ForgeImporters/forgeimporters/base.py | 2 +-
.../forgetracker/tests/unit/test_ticket_model.py | 2 +-
fuse/accessfs.py | 2 +-
ruff.toml | 22 +++++++++++++++++++---
scripts/ApacheAccessHandler.py | 2 +-
scripts/changelog.py | 2 +-
scripts/new_ticket.py | 2 +-
scripts/wiki-copy.py | 4 ++--
26 files changed, 63 insertions(+), 41 deletions(-)
diff --git a/Allura/allura/command/script.py b/Allura/allura/command/script.py
index 52e58b2fb..78edd8746 100644
--- a/Allura/allura/command/script.py
+++ b/Allura/allura/command/script.py
@@ -73,7 +73,7 @@ class ScriptCommand(base.Command):
pr = cProfile.Profile()
pr.enable()
- exec(code, ns)
+ exec(code, ns) # noqa: S102
if self.options.profile:
pr.disable()
diff --git a/Allura/allura/command/taskd.py b/Allura/allura/command/taskd.py
index 9d6c42c67..782c216cd 100644
--- a/Allura/allura/command/taskd.py
+++ b/Allura/allura/command/taskd.py
@@ -160,7 +160,7 @@ class TaskdCommand(base.Command):
if self.restart_when_done:
base.log.info('taskd pid %s restarting itself' % os.getpid())
- os.execv(sys.argv[0], sys.argv)
+ os.execv(sys.argv[0], sys.argv) # noqa: S606
class TaskCommand(base.Command):
diff --git a/Allura/allura/config/middleware.py b/Allura/allura/config/middleware.py
index e481a1621..edf28a968 100644
--- a/Allura/allura/config/middleware.py
+++ b/Allura/allura/config/middleware.py
@@ -91,7 +91,7 @@ def make_app(global_conf: dict, **app_conf):
class BeakerPickleSerializerWithLatin1(PickleSerializer):
def loads(self, data_string):
# need latin1 to decode py2 timestamps in py https://docs.python.org/3/library/pickle.html#pickle.Unpickler
- return pickle.loads(data_string, encoding='latin1')
+ return pickle.loads(data_string, encoding='latin1') # noqa: S301
def _make_core_app(root, global_conf: dict, **app_conf):
diff --git a/Allura/allura/controllers/rest.py b/Allura/allura/controllers/rest.py
index ef4ae4ca5..f077b3d40 100644
--- a/Allura/allura/controllers/rest.py
+++ b/Allura/allura/controllers/rest.py
@@ -251,7 +251,7 @@ class OAuthNegotiator:
return AlluraOauth1Server(Oauth1Validator())
def _authenticate(self):
- bearer_token_prefix = 'Bearer '
+ bearer_token_prefix = 'Bearer ' # noqa: S105
auth = request.headers.get('Authorization')
if auth and auth.startswith(bearer_token_prefix):
access_token = auth[len(bearer_token_prefix):]
diff --git a/Allura/allura/lib/decorators.py b/Allura/allura/lib/decorators.py
index eaf8016f8..a0fecc1f4 100644
--- a/Allura/allura/lib/decorators.py
+++ b/Allura/allura/lib/decorators.py
@@ -142,7 +142,7 @@ def reconfirm_auth(func, *args, **kwargs):
session.save()
kwargs.pop('password', None)
else:
- request.validation.errors['password'] = 'Invalid password.'
+ request.validation.errors['password'] = 'Invalid password.' # noqa: S105
allowed_timedelta = timedelta(seconds=asint(config.get('auth.reconfirm.seconds', 60)))
last_reconfirm = session.get('auth-reconfirmed', datetime.min)
diff --git a/Allura/allura/lib/helpers.py b/Allura/allura/lib/helpers.py
index f0675e443..5f5c41bdf 100644
--- a/Allura/allura/lib/helpers.py
+++ b/Allura/allura/lib/helpers.py
@@ -1080,7 +1080,7 @@ def urlopen(url: str | urllib.request.Request, retries=3, codes=(408, 500, 502,
attempts = 0
while True:
try:
- return urllib.request.urlopen(url, timeout=timeout)
+ return urllib.request.urlopen(url, timeout=timeout) # noqa: S310
except OSError as e:
no_retry = isinstance(e, urllib.error.HTTPError) and e.code not in codes
if attempts < retries and not no_retry:
diff --git a/Allura/allura/lib/import_api.py b/Allura/allura/lib/import_api.py
index 141c03c00..f6d6282ab 100644
--- a/Allura/allura/lib/import_api.py
+++ b/Allura/allura/lib/import_api.py
@@ -47,7 +47,7 @@ class AlluraImportApiClient:
while True:
try:
- result = six.moves.urllib.request.urlopen(url, six.ensure_binary(urlencode(params)))
+ result = six.moves.urllib.request.urlopen(url, six.ensure_binary(urlencode(params))) # noqa: S310
resp = result.read()
return json.loads(resp)
except six.moves.urllib.error.HTTPError as e:
diff --git a/Allura/allura/lib/phone/nexmo.py b/Allura/allura/lib/phone/nexmo.py
index 653159da2..630d7b94f 100644
--- a/Allura/allura/lib/phone/nexmo.py
+++ b/Allura/allura/lib/phone/nexmo.py
@@ -75,13 +75,13 @@ class NexmoPhoneService(PhoneService):
url = urljoin(url, 'json')
headers = {'Content-Type': 'application/json'}
params = self.add_common_params(params)
- log_params = dict(params, api_key='...', api_secret='...')
+ log_params = dict(params, api_key='...', api_secret='...') # noqa: S106
if 'number' in log_params:
log_params['number'] = phone_number_hash(log_params['number'])
post_params = json.dumps(params, sort_keys=True)
log.info('PhoneService (nexmo) request: %s %s', url, log_params)
try:
- resp = requests.post(url, data=post_params, headers=headers)
+ resp = requests.post(url, data=post_params, headers=headers, timeout=30)
log.info('PhoneService (nexmo) response: %s', resp.content)
resp = resp.json()
except Exception:
diff --git a/Allura/allura/lib/plugin.py b/Allura/allura/lib/plugin.py
index 03b6d61f2..d32227d13 100644
--- a/Allura/allura/lib/plugin.py
+++ b/Allura/allura/lib/plugin.py
@@ -27,7 +27,6 @@ import string
import crypt
import random
from contextlib import contextmanager
-from urllib.request import urlopen
from urllib.parse import urlparse
from io import BytesIO
from random import randint
@@ -36,6 +35,8 @@ from base64 import b64encode
from datetime import datetime, timedelta
import typing
import calendar
+
+import requests
import six
try:
@@ -1129,7 +1130,7 @@ class ProjectRegistrationProvider:
troves.append(
M.TroveCategory.query.get(trove_cat_id=trove_id)._id)
if 'icon' in project_template:
- icon_file = BytesIO(urlopen(project_template['icon']['url']).read())
+ icon_file = BytesIO(requests.get(project_template['icon']['url'], timeout=30).content)
p.save_icon(project_template['icon']['filename'], icon_file)
if user_project:
diff --git a/Allura/allura/lib/project_create_helpers.py b/Allura/allura/lib/project_create_helpers.py
index 77c2c5adf..387eb0f68 100644
--- a/Allura/allura/lib/project_create_helpers.py
+++ b/Allura/allura/lib/project_create_helpers.py
@@ -309,7 +309,7 @@ def create_project_with_attrs(p, nbhd, update=False, ensure_tools=False):
granted_by_neighborhood_id=nbhd._id)
if p.icon_url:
- req = requests.get(p.icon_url)
+ req = requests.get(p.icon_url, timeout=30)
req.raise_for_status()
project.save_icon(urlparse(p.icon_url).path,
BytesIO(req.content),
diff --git a/Allura/allura/model/index.py b/Allura/allura/model/index.py
index 374fcb9cf..c778db3b3 100644
--- a/Allura/allura/model/index.py
+++ b/Allura/allura/model/index.py
@@ -93,7 +93,7 @@ class ArtifactReference(MappedClass):
'''Look up the artifact referenced'''
aref = self.artifact_reference
try:
- cls = loads(bytes(aref.cls))
+ cls = loads(bytes(aref.cls)) # noqa: S301
with h.push_context(aref.project_id):
return cls.query.get(_id=aref.artifact_id)
except Exception:
diff --git a/Allura/allura/model/notification.py b/Allura/allura/model/notification.py
index d97f86ee3..164cee374 100644
--- a/Allura/allura/model/notification.py
+++ b/Allura/allura/model/notification.py
@@ -109,7 +109,7 @@ class Notification(MappedClass):
ref = RelationProperty('ArtifactReference')
- view = jinja2.Environment(
+ view = jinja2.Environment( # noqa: S701
loader=jinja2.PackageLoader('allura', 'templates'),
auto_reload=asbool(config.get('auto_reload_templates', True)),
)
@@ -688,7 +688,7 @@ class Mailbox(MappedClass):
class MailFooter:
- view = jinja2.Environment(
+ view = jinja2.Environment( # noqa: S701
loader=jinja2.PackageLoader('allura', 'templates'),
auto_reload=asbool(config.get('auto_reload_templates', True)),
)
diff --git a/Allura/allura/scripts/trac_export.py b/Allura/allura/scripts/trac_export.py
index e40aa85f4..ce9d2c9e2 100644
--- a/Allura/allura/scripts/trac_export.py
+++ b/Allura/allura/scripts/trac_export.py
@@ -143,7 +143,7 @@ class TracExport:
def csvopen(self, url):
self.log_url(url)
- f = urlopen(url)
+ f = urlopen(url) # noqa: S310
# Trac doesn't throw 403 error, just shows normal 200 HTML page
# telling that access denied. So, we'll emulate 403 ourselves.
# TODO: currently, any non-csv result treated as 403.
@@ -166,7 +166,7 @@ class TracExport:
html2text.BODY_WIDTH = 0
url = self.full_url(self.TICKET_URL % id)
self.log_url(url)
- d = BeautifulSoup(urlopen(url))
+ d = BeautifulSoup(urlopen(url)) # noqa: S310
self.clean_missing_wiki_links(d)
desc = d.find('div', 'description').find('div', 'searchable')
ticket['description'] = html2text.html2text(
@@ -195,7 +195,7 @@ class TracExport:
# Scrape HTML to get ticket attachments
url = self.full_url(self.ATTACHMENT_LIST_URL % id)
self.log_url(url)
- f = urlopen(url)
+ f = urlopen(url) # noqa: S310
soup = BeautifulSoup(f)
attach = soup.find('div', id='attachments')
list = []
diff --git a/Allura/allura/tests/functional/test_neighborhood.py b/Allura/allura/tests/functional/test_neighborhood.py
index 13f46dfff..ec1633172 100644
--- a/Allura/allura/tests/functional/test_neighborhood.py
+++ b/Allura/allura/tests/functional/test_neighborhood.py
@@ -670,9 +670,6 @@ class TestNeighborhood(TestController):
def test_project_template(self):
setup_trove_categories()
- icon_url = 'file://' + \
- os.path.join(allura.__path__[0], 'nf', 'allura',
- 'images', 'neo-icon-set-454545-256x350.png')
test_groups = [{
"name": "Viewer", # group will be created, all params are valid
"permissions": ["read"],
@@ -699,7 +696,7 @@ class TestNeighborhood(TestController):
project_template="""{
"private":true,
"icon":{
- "url":"%s",
+ "url":"https://httpbin.org/image/png",
"filename":"icon.png"
},
"tools":{
@@ -727,7 +724,7 @@ class TestNeighborhood(TestController):
"developmentstatus":[11]
},
"groups": %s
- }""" % (icon_url, json.dumps(test_groups))),
+ }""" % (json.dumps(test_groups))),
extra_environ=dict(username='root'))
r = self.app.post(
'/adobe/register',
diff --git a/Allura/allura/tests/unit/phone/test_nexmo.py b/Allura/allura/tests/unit/phone/test_nexmo.py
index f2e99dacd..8b27d4de1 100644
--- a/Allura/allura/tests/unit/phone/test_nexmo.py
+++ b/Allura/allura/tests/unit/phone/test_nexmo.py
@@ -95,7 +95,9 @@ class TestPhoneService:
req.post.assert_called_once_with(
'https://api.nexmo.com/verify/json',
data=data,
- headers=headers)
+ headers=headers,
+ timeout=30,
+ )
req.post.reset_mock()
req.post.return_value.json.return_value = {
@@ -108,7 +110,9 @@ class TestPhoneService:
req.post.assert_called_once_with(
'https://api.nexmo.com/verify/json',
data=data,
- headers=headers)
+ headers=headers,
+ timeout=30,
+ )
@patch('allura.lib.phone.nexmo.requests', autospec=True)
def test_verify_exception(self, req):
@@ -138,7 +142,9 @@ class TestPhoneService:
req.post.assert_called_once_with(
'https://api.nexmo.com/verify/check/json',
data=data,
- headers=headers)
+ headers=headers,
+ timeout=30,
+ )
req.post.reset_mock()
req.post.return_value.json.return_value = {
@@ -151,7 +157,9 @@ class TestPhoneService:
req.post.assert_called_once_with(
'https://api.nexmo.com/verify/check/json',
data=data,
- headers=headers)
+ headers=headers,
+ timeout=30,
+ )
@patch('allura.lib.phone.nexmo.requests', autospec=True)
def test_check_exception(self, req):
diff --git a/Allura/allura/websetup/bootstrap.py b/Allura/allura/websetup/bootstrap.py
index 765c91228..eadd141e9 100644
--- a/Allura/allura/websetup/bootstrap.py
+++ b/Allura/allura/websetup/bootstrap.py
@@ -307,7 +307,7 @@ def clear_all_database_tables():
db.drop_collection(coll)
-def create_user(display_name, username=None, password='foo', make_project=False):
+def create_user(display_name, username=None, password='foo', make_project=False): # noqa: S107
if not username:
username = display_name.lower().replace(' ', '-')
user = M.User.register(dict(username=username,
diff --git a/Allura/ldap-setup.py b/Allura/ldap-setup.py
index baf5432ac..e91343048 100644
--- a/Allura/ldap-setup.py
+++ b/Allura/ldap-setup.py
@@ -95,7 +95,7 @@ def get_value(key, default):
def run(command):
- rc = os.system(command)
+ rc = os.system(command) # noqa: S605
if rc != 0:
log.error('Error running %s', command)
assert rc == 0
diff --git a/Allura/setup.py b/Allura/setup.py
index e5c549376..63613d66a 100644
--- a/Allura/setup.py
+++ b/Allura/setup.py
@@ -17,7 +17,7 @@
from setuptools import setup, find_packages
-exec(open('allura/version.py').read())
+exec(open('allura/version.py').read()) # noqa: S102
PROJECT_DESCRIPTION = '''
Allura is an open source implementation of a software "forge", a web site
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index e66cce270..79984d011 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -167,7 +167,7 @@ class ProjectExtractor:
@staticmethod
def urlopen(url, retries=3, codes=(408, 500, 502, 503, 504), timeout=120, unredirected_hdrs=None, **kw):
- req = urllib.request.Request(url, **kw)
+ req = urllib.request.Request(url, **kw) # noqa: S310
if unredirected_hdrs:
for key, val in unredirected_hdrs.items():
req.add_unredirected_header(key, val)
diff --git a/ForgeTracker/forgetracker/tests/unit/test_ticket_model.py b/ForgeTracker/forgetracker/tests/unit/test_ticket_model.py
index afd604678..9ea0c37a0 100644
--- a/ForgeTracker/forgetracker/tests/unit/test_ticket_model.py
+++ b/ForgeTracker/forgetracker/tests/unit/test_ticket_model.py
@@ -349,7 +349,7 @@ class TestTicketModel(TrackerTestWithModel):
ticket.summary = 'test ticket'
ticket.description = 'test description'
assert len(ticket.attachments) == 0
- f = six.moves.urllib.request.urlopen('file://%s' % __file__)
+ f = six.moves.urllib.request.urlopen('file://%s' % __file__) # noqa: S310
TicketAttachment.save_attachment(
'test_ticket_model.py', ResettableStream(f),
artifact_id=ticket._id)
diff --git a/fuse/accessfs.py b/fuse/accessfs.py
index 4d43e7e2e..d5e843622 100644
--- a/fuse/accessfs.py
+++ b/fuse/accessfs.py
@@ -341,7 +341,7 @@ class PermissionCache:
repo_path=path,
username=uname)))
print(f'Checking access for {uname} at {url} ({path})')
- fp = six.moves.urllib.request.urlopen(url)
+ fp = six.moves.urllib.request.urlopen(url) # noqa: S310
result = json.load(fp)
print(result)
entry = 0
diff --git a/ruff.toml b/ruff.toml
index 2bfa1ec9e..7da944339 100644
--- a/ruff.toml
+++ b/ruff.toml
@@ -29,7 +29,7 @@ lint.select = [
"ISC001", # NIC001 in flake8 codes
"B",
"PGH", # https://github.com/pre-commit/pygrep-hooks
- "S307", # eval
+ "S", # bandit
"G010", # logging.warn
"T10", # debugger breakpoints
"T20", # print()
@@ -47,13 +47,29 @@ lint.ignore = [
'B007', # Loop control variable not used within the loop body
'B904', # use raise from
'B905', # zip(strict=True) would be good, but need to closely evaluate all existing cases first
+ 'S101', # assert
+ 'S103', # permissive filesystem mask
+ 'S104', # network binding, which doesn't apply to our code, and gives false-positive on test IPs
+ 'S108', # temp files
+ 'S110', # try-except-pass, too many exist already
+ 'S303', # md5 & sha1
+ 'S311', # random
+ 'S324', # md5 & sha1
+ 'S603', # subprocess
+ 'S607', # partial path
]
[lint.per-file-ignores]
'__init__.py' = ['F403'] # import *
-'**/{alluratest,tests}/*' = [
+'{**/alluratest/*,**/tests/*,run_tests}' = [
'B011', # assert False
'T20', # print
+ 'S105', # hard-coded secrets
+ 'S106', # hard-coded secrets
+ 'S107', # hard-coded password
+ 'S314', # XML parsing
+ 'S602', # subprocess shell=True
+ 'S604', # other shell=True
]
'{scripts,Allura/allura/scripts,Allura/allura/command}/*' = ['T20'] # print
-'{run_tests,fuse/accessfs.py,ForgeSVN/setup.py}' = ['T20'] # print
+'{fuse/accessfs.py,ForgeSVN/setup.py}' = ['T20'] # print
diff --git a/scripts/ApacheAccessHandler.py b/scripts/ApacheAccessHandler.py
index 63ed8431f..217e4ecf6 100644
--- a/scripts/ApacheAccessHandler.py
+++ b/scripts/ApacheAccessHandler.py
@@ -49,7 +49,7 @@ def load_requests_lib(req):
if virtualenv_path:
activate_this = '%s/bin/activate_this.py' % virtualenv_path
try:
- exec(compile(open(activate_this, "rb").read(), activate_this, 'exec'), {'__file__': activate_this})
+ exec(compile(open(activate_this, "rb").read(), activate_this, 'exec'), {'__file__': activate_this}) # noqa: S102
except Exception as e:
log(req, "Couldn't activate venv via {}: {}".format(activate_this, repr(e)))
global requests
diff --git a/scripts/changelog.py b/scripts/changelog.py
index bdada653f..6d4c2a7e9 100755
--- a/scripts/changelog.py
+++ b/scripts/changelog.py
@@ -56,7 +56,7 @@ def get_tickets(from_ref, to_ref):
def get_ticket_summaries(tickets):
summaries = {}
- r = requests.get(API_URL.format(' '.join(tickets)))
+ r = requests.get(API_URL.format(' '.join(tickets)), timeout=30)
if r.status_code != 200:
raise ValueError(f'Unexpected response code: {r.status_code}')
for ticket in r.json()['tickets']:
diff --git a/scripts/new_ticket.py b/scripts/new_ticket.py
index 7588dbfd8..6dbf5cc73 100755
--- a/scripts/new_ticket.py
+++ b/scripts/new_ticket.py
@@ -51,7 +51,7 @@ if __name__ == '__main__':
'access_token': access_token,
'ticket_form.summary': summary,
'ticket_form.description': description,
- })
+ }, timeout=30)
if r.status_code == 200:
print('Ticket created at: %s' % r.url)
pprint(r.json())
diff --git a/scripts/wiki-copy.py b/scripts/wiki-copy.py
index ddb3187d3..c023684b0 100644
--- a/scripts/wiki-copy.py
+++ b/scripts/wiki-copy.py
@@ -41,12 +41,12 @@ def main():
base_url = options.to_wiki.split('/rest/')[0]
oauth_client = make_oauth_client(base_url)
- wiki_json = requests.get(options.from_wiki).json()['pages']
+ wiki_json = requests.get(options.from_wiki, timeout=30).json()['pages']
for p in wiki_json:
from_url = options.from_wiki.rstrip('/') + '/' + p
to_url = options.to_wiki.rstrip('/') + '/' + p
try:
- page_json = requests.get(from_url).json()
+ page_json = requests.get(from_url, timeout=30).json()
if options.debug:
print(page_json['text'])
break
(allura) 01/05: [#8539] upgrade ruff
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8539
in repository https://gitbox.apache.org/repos/asf/allura.git
commit c5a60409d0cf6ea1f184bb8069b8ea8075187763
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Tue Mar 19 18:08:38 2024 -0400
[#8539] upgrade ruff
---
.pre-commit-config.yaml | 2 +-
requirements.txt | 2 +-
ruff.toml | 10 ++++++----
3 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0e3a5168f..7d3eea1f2 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -80,7 +80,7 @@ repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
- rev: v0.1.11
+ rev: v0.3.3
hooks:
- id: ruff
types: [python]
diff --git a/requirements.txt b/requirements.txt
index 471b477ba..656376a0e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -201,7 +201,7 @@ requests==2.31.0
# requests-oauthlib
requests-oauthlib==1.3.1
# via -r requirements.in
-ruff==0.1.13
+ruff==0.3.2
# via -r requirements.in
setproctitle==1.3.3
# via -r requirements.in
diff --git a/ruff.toml b/ruff.toml
index b2e7c55cb..2bfa1ec9e 100644
--- a/ruff.toml
+++ b/ruff.toml
@@ -16,10 +16,10 @@
# under the License.
line-length = 119
-show-source = true
+output-format = "full"
target-version = "py38"
-select = [
+lint.select = [
# all flake8 & pep8 (except 'ignore' below)
"F",
"E",
@@ -29,12 +29,14 @@ select = [
"ISC001", # NIC001 in flake8 codes
"B",
"PGH", # https://github.com/pre-commit/pygrep-hooks
+ "S307", # eval
+ "G010", # logging.warn
"T10", # debugger breakpoints
"T20", # print()
"FA", # future annotations (to ensure compatibility with `target-version`)
]
-ignore = [
+lint.ignore = [
'F401', # Imported but unused,
'F811', # Redefinition of unused
'F841', # Assigned to but never used
@@ -47,7 +49,7 @@ ignore = [
'B905', # zip(strict=True) would be good, but need to closely evaluate all existing cases first
]
-[per-file-ignores]
+[lint.per-file-ignores]
'__init__.py' = ['F403'] # import *
'**/{alluratest,tests}/*' = [
'B011', # assert False
(allura) 02/05: [#8539] remove old teamforge import script, which had lots of hardcoded specifics
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8539
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 2dc7b9fbdf79cae0cbd43fbc72a169724997ca47
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Wed Mar 20 16:20:08 2024 -0400
[#8539] remove old teamforge import script, which had lots of hardcoded specifics
---
.gitignore | 1 -
.pre-commit-config.yaml | 1 -
Allura/docs/getting_started/administration.rst | 11 -
scripts/teamforge-import.py | 1126 ------------------------
4 files changed, 1139 deletions(-)
diff --git a/.gitignore b/.gitignore
index c620e794e..97d6ae99e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,7 +34,6 @@ Allura/allura/templates/home
Allura/allura/templates/var
Allura/production.ini
Allura/forced_upgrade.ini
-scripts/teamforge-export/
/node_modules
report.clonedigger
.ropeproject
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7d3eea1f2..bfda2f671 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -33,7 +33,6 @@ repos:
(?x)^(
Allura/allura/eventslistener.py|
Allura/allura/lib/.*|
- scripts/teamforge-import.py
)$
diff --git a/Allura/docs/getting_started/administration.rst b/Allura/docs/getting_started/administration.rst
index 03b413f21..5d9bff267 100644
--- a/Allura/docs/getting_started/administration.rst
+++ b/Allura/docs/getting_started/administration.rst
@@ -314,17 +314,6 @@ scrub-allura-data.py
:prog: paster script development.ini ../scripts/scrub-allura-data.py --
-teamforge-import.py
--------------------
-
-*Cannot currently be run as a background task.*
-
-Extract data from a TeamForge site (via its web API), and import directly into Allura. There are some hard-coded
-and extra functions in this script, which should be removed or updated before being used again.
-Requires running: :command:`pip install suds` first. ::
-
- usage: paster script development.ini ../scripts/teamforge-import.py -- --help
-
.. _site-notifications:
Site Notifications
diff --git a/scripts/teamforge-import.py b/scripts/teamforge-import.py
deleted file mode 100644
index 15b669d38..000000000
--- a/scripts/teamforge-import.py
+++ /dev/null
@@ -1,1126 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import logging
-from ast import literal_eval
-from getpass import getpass
-from optparse import OptionParser
-from tg import tmpl_context as c
-import re
-import os
-from time import mktime
-import time
-import json
-from urllib.parse import urlparse
-import six.moves.urllib.request
-import six.moves.urllib.parse
-import six.moves.urllib.error
-from http.cookiejar import CookieJar
-from datetime import datetime
-from configparser import ConfigParser
-import random
-import string
-
-import sqlalchemy
-from suds.client import Client
-from ming.odm.odmsession import ThreadLocalODMSession
-from ming.base import Object
-
-from allura import model as M
-from allura.lib import helpers as h
-from allura.lib import utils
-import six
-
-log = logging.getLogger('teamforge-import')
-
-'''
-
-http://help.collab.net/index.jsp?topic=/teamforge520/reference/api-services.html
-
-http://www.open.collab.net/nonav/community/cif/csfe/50/javadoc/index.html?com/collabnet/ce/soap50/webservices/page/package-summary.html
-
-'''
-
-options = None
-s = None # security token
-client = None # main api client
-users = {}
-
-cj = CookieJar()
-loggedInOpener = six.moves.urllib.request.build_opener(six.moves.urllib.request.HTTPCookieProcessor(cj))
-
-
-def make_client(api_url, app):
- return Client(api_url + app + '?wsdl', location=api_url + app)
-
-
-def main():
- global options, s, client, users
- defaults = dict(
- api_url=None,
- attachment_url='/sf/%s/do/%s/',
- default_wiki_text='PRODUCT NAME HERE',
- username=None,
- password=None,
- output_dir='teamforge-export/',
- list_project_ids=False,
- neighborhood=None,
- neighborhood_shortname=None,
- use_thread_import_id_when_reloading=False,
- skip_wiki=False,
- skip_frs_download=False,
- skip_unsupported_check=False)
- optparser = get_parser(defaults)
- options, project_ids = optparser.parse_args()
- if options.config_file:
- config = ConfigParser()
- config.read(options.config_file)
- defaults.update(
- (k, literal_eval(v)) for k, v in config.items('teamforge-import'))
- optparser = get_parser(defaults)
- options, project_ids = optparser.parse_args()
-
- # neither specified, so do both
- if not options.extract and not options.load:
- options.extract = True
- options.load = True
-
- if options.extract:
- client = make_client(options.api_url, 'CollabNet')
- api_v = client.service.getApiVersion()
- if not api_v.startswith('5.4.'):
- log.warning('Unexpected API Version %s. May not work correctly.' %
- api_v)
-
- s = client.service.login(
- options.username, options.password or getpass('Password: '))
- teamforge_v = client.service.getVersion(s)
- if not teamforge_v.startswith('5.4.'):
- log.warning(
- 'Unexpected TeamForge Version %s. May not work correctly.' %
- teamforge_v)
-
- if options.load:
- if not options.neighborhood:
- log.error('You must specify a neighborhood when loading')
- return
- try:
- nbhd = M.Neighborhood.query.get(name=options.neighborhood)
- except Exception:
- log.exception('error querying mongo')
- log.error(
- 'This should be run as "paster script production.ini ../scripts/teamforge-import.py -- ...options.."')
- return
- assert nbhd
-
- if not project_ids:
- if not options.extract:
- log.error('You must specify project ids')
- return
- projects = client.service.getProjectList(s)
- project_ids = [p.id for p in projects.dataRows]
-
- if options.list_project_ids:
- print(' '.join(project_ids))
- return
-
- if not os.path.exists(options.output_dir):
- os.makedirs(options.output_dir)
- for pid in project_ids:
- if options.extract:
- try:
- project = client.service.getProjectData(s, pid)
- log.info('Project: %s %s %s' %
- (project.id, project.title, project.path))
- out_dir = os.path.join(options.output_dir, project.id)
- if not os.path.exists(out_dir):
- os.mkdir(out_dir)
-
- get_project(project)
- get_files(project)
- if not options.skip_wiki:
- get_homepage_wiki(project)
- get_discussion(project)
- get_news(project)
- if not options.skip_unsupported_check:
- check_unsupported_tools(project)
- with open(os.path.join(options.output_dir, 'users.json'), 'w', encoding='utf-8') as user_file:
- json.dump(users, user_file, default=str)
- except Exception:
- log.exception('Error extracting %s' % pid)
-
- if options.load:
- try:
- project = create_project(pid, nbhd)
- except Exception:
- log.exception('Error creating %s' % pid)
-
-
-def load_users():
- ''' load the users data from file, if it hasn't been already '''
- global users
- user_filename = os.path.join(options.output_dir, 'users.json')
- if not users and os.path.exists(user_filename):
- with open(user_filename) as user_file:
- # Object for attribute access
- users = json.load(user_file, object_hook=Object)
-
-
-def save_user(usernames):
- if isinstance(usernames, str):
- usernames = [usernames]
-
- load_users()
-
- for username in usernames:
- if username not in users:
- user_data = client.service.getUserData(s, username)
- users[username] = Object(user_data)
- if users[username].status != 'Active':
- log.warning('user: %s status: %s' %
- (username, users[username].status))
-
-
-def get_project(project):
- global client
- cats = make_client(options.api_url, 'CategorizationApp')
-
- data = client.service.getProjectData(s, project.id)
- access_level = {1: 'public', 4: 'private', 3: 'gated community'}[
- client.service.getProjectAccessLevel(s, project.id)
- ]
- admins = client.service.listProjectAdmins(s, project.id).dataRows
- members = client.service.getProjectMemberList(s, project.id).dataRows
- groups = client.service.getProjectGroupList(s, project.id).dataRows
- categories = cats.service.getProjectCategories(s, project.id).dataRows
- save(json.dumps(dict(
- data=dict(data),
- access_level=access_level,
- admins=list(map(dict, admins)),
- members=list(map(dict, members)),
- groups=list(map(dict, groups)),
- categories=list(map(dict, categories)),
- ), default=str),
- project, project.id + '.json')
-
- if len(groups):
- log.warning('Project has groups %s' % groups)
- for u in admins:
- if not u.status != 'active':
- log.warning('inactive admin %s' % u)
- if u.superUser:
- log.warning('super user admin %s' % u)
-
- save_user(data.createdBy)
- save_user(u.userName for u in admins)
- save_user(u.userName for u in members)
-
-
-def get_user(orig_username):
- 'returns an allura User object'
- sf_username = make_valid_sf_username(orig_username)
-
- u = M.User.by_username(sf_username)
-
- if not u:
- load_users()
- user = users[orig_username]
- if user.status != 'Active':
- log.warning(f'Inactive user {orig_username} {user.status}')
-
- if not 3 <= len(user.fullName) <= 32:
- raise Exception('invalid fullName length: %s' % user.fullName)
- if '@' not in user.email:
- raise Exception('invalid email: %s' % user.email)
- # FIXME: hardcoded SFX integration
- from sfx.model import tables as T
- nu = T.users.insert()
- nu.execute(user_name=sf_username.encode('utf-8'),
- email=user.email.lower().encode('utf-8'),
- realname=user.fullName.encode('utf-8'),
- status='A' if user.status == 'Active' else 'D',
- language=275, # english trove id
- timezone=user.timeZone,
- user_pw=''.join(random.sample(string.printable, 32)),
- unix_pw=''.join(random.sample(string.printable, 32)),
- user_pw_modtime=int(time.time()),
- mail_siteupdates=0,
- add_date=int(time.time()),
- )
- user_id = sqlalchemy.select(
- [T.users.c.user_id], T.users.c.user_name == sf_username).execute().fetchone().user_id
- npref = T.user_preferences.insert()
- npref.execute(user_id=user_id, preference_name='country',
- preference_value='US')
- npref.execute(user_id=user_id,
- preference_name='opt_research', preference_value=0)
- npref.execute(user_id=user_id,
- preference_name='opt_thirdparty', preference_value=0)
-
- new_audit = T.audit_trail_user.insert()
- new_audit.execute(
- date=int(time.time()),
- username='nobody',
- ip_address='(imported)',
- operation_resource=user_id,
- operation='%s user account created by TeamForge import script' % user.status,
- operation_target='',
- )
-
- u = M.User.by_username(sf_username)
- assert u
- return u
-
-
-def convert_project_shortname(teamforge_path):
- 'convert from TeamForge to SF, and validate early'
- tf_shortname = teamforge_path.split('.')[-1]
- sf_shortname = tf_shortname.replace('_', '-')
-
- # FIXME hardcoded translations
- sf_shortname = {
- 'i1': 'motorola-i1',
- 'i9': 'motorola-i9',
- 'devplatformforocap': 'ocap-dev-pltfrm',
- 'sitewide': '--init--',
- }.get(sf_shortname, sf_shortname)
-
- if not 3 <= len(sf_shortname) <= 15:
- raise ValueError(
- 'Project name length must be between 3 & 15, inclusive: %s (%s)' %
- (sf_shortname, len(sf_shortname)))
- return sf_shortname
-
-
-# FIXME hardcoded
-skip_perms_usernames = {
- 'username1', 'username2', 'username3'
-}
-
-
-def create_project(pid, nbhd):
- M.session.artifact_orm_session._get().skip_mod_date = True
- data = loadjson(pid, pid + '.json')
- # pprint(data)
- log.info(f'Loading: {pid} {data.data.title} {data.data.path}')
- shortname = convert_project_shortname(data.data.path)
-
- project = M.Project.query.get(
- shortname=shortname, neighborhood_id=nbhd._id)
- if not project:
- private = (data.access_level == 'private')
- log.debug(f'Creating {shortname} private={private}')
- one_admin = [
- u.userName for u in data.admins if u.status == 'Active'][0]
- project = nbhd.register_project(shortname,
- get_user(one_admin),
- project_name=data.data.title,
- private_project=private)
- project.notifications_disabled = True
- project.short_description = data.data.description
- project.last_updated = datetime.strptime(
- data.data.lastModifiedDate, '%Y-%m-%d %H:%M:%S')
- M.main_orm_session.flush(project)
- # TODO: push last_updated to gutenberg?
- # TODO: try to set createdDate?
-
- role_admin = M.ProjectRole.by_name('Admin', project)
- admin_usernames = set()
- for admin in data.admins:
- # FIXME: skip non-active users
- if admin.userName in skip_perms_usernames:
- continue
- admin_usernames.add(admin.userName)
- user = get_user(admin.userName)
- c.user = user
- pr = M.ProjectRole.by_user(user, project=project, upsert=True)
- pr.roles = [role_admin._id]
- ThreadLocalODMSession.flush_all()
- role_developer = M.ProjectRole.by_name('Developer', project)
- for member in data.members:
- # FIXME: skip non-active users
- if member.userName in skip_perms_usernames:
- continue
- if member.userName in admin_usernames:
- continue
- user = get_user(member.userName)
- pr = M.ProjectRole.by_user(user, project=project, upsert=True)
- pr.roles = [role_developer._id]
- ThreadLocalODMSession.flush_all()
- project.labels = [cat.path.split('projects/categorization.root.')[1]
- for cat in data.categories]
- icon_file = 'emsignia-MOBILITY-red.png'
- if 'nsn' in project.labels or 'msi' in project.labels:
- icon_file = 'emsignia-SOLUTIONS-blue.gif'
- if project.icon:
- M.ProjectFile.remove(dict(project_id=project._id, category='icon'))
- with open(os.path.join('..', 'scripts', icon_file), 'rb') as fp:
- M.ProjectFile.save_image(
- icon_file, fp, content_type=utils.guess_mime_type(icon_file),
- square=True, thumbnail_size=(48, 48),
- thumbnail_meta=dict(project_id=project._id, category='icon'))
- ThreadLocalODMSession.flush_all()
-
- dirs = os.listdir(os.path.join(options.output_dir, pid))
-
- frs_mapping = loadjson(pid, 'frs_mapping.json')
-
- if not options.skip_wiki and 'wiki' in dirs:
- import_wiki(project, pid, nbhd)
- if not options.skip_frs_download and not project.app_instance('downloads'):
- project.install_app('Downloads', 'downloads')
- if 'forum' in dirs:
- import_discussion(project, pid, frs_mapping, shortname, nbhd)
- if 'news' in dirs:
- import_news(project, pid, frs_mapping, shortname, nbhd)
-
- project.notifications_disabled = False
- ThreadLocalODMSession.flush_all()
- return project
-
-
-def import_wiki(project, pid, nbhd):
- from forgewiki import model as WM
-
- def upload_attachments(page, pid, beginning):
- dirpath = os.path.join(options.output_dir, pid, 'wiki', beginning)
- if not os.path.exists(dirpath):
- return
- files = os.listdir(dirpath)
- for f in files:
- with open(os.path.join(options.output_dir, pid, 'wiki', beginning, f)) as fp:
- page.attach(f, fp, content_type=utils.guess_mime_type(f))
- pages = os.listdir(os.path.join(options.output_dir, pid, 'wiki'))
- # handle the homepage content
- if 'homepage_text.markdown' in pages:
- home_app = project.app_instance('home')
- h.set_context(project.shortname, 'home', neighborhood=nbhd)
- # set permissions and config options
- role_admin = M.ProjectRole.by_name('Admin')._id
- role_anon = M.ProjectRole.by_name('*anonymous')._id
- home_app.config.options['show_discussion'] = False
- home_app.config.options['show_left_bar'] = False
- home_app.config.options['show_right_bar'] = False
- home_app.config.acl = [
- M.ACE.allow(role_anon, 'read'),
- M.ACE.allow(role_admin, 'create'),
- M.ACE.allow(role_admin, 'edit'),
- M.ACE.allow(role_admin, 'delete'),
- M.ACE.allow(role_admin, 'moderate'),
- M.ACE.allow(role_admin, 'configure'),
- M.ACE.allow(role_admin, 'admin')]
- p = WM.Page.upsert('Home')
- p.text = wiki2markdown(load(pid, 'wiki', 'homepage_text.markdown'))
- upload_attachments(p, pid, 'homepage')
- if 'HomePage.json' in pages and 'HomePage.markdown' in pages:
- wiki_app = project.app_instance('wiki')
- if not wiki_app:
- wiki_app = project.install_app('Wiki', 'wiki')
- h.set_context(project.shortname, 'wiki', neighborhood=nbhd)
- # set permissions and config options
- role_admin = M.ProjectRole.by_name('Admin')._id
- role_anon = M.ProjectRole.by_name('*anonymous')._id
- wiki_app.config.options['show_discussion'] = False
- wiki_app.config.options['show_left_bar'] = False
- wiki_app.config.options['show_right_bar'] = False
- wiki_app.config.acl = [
- M.ACE.allow(role_anon, 'read'),
- M.ACE.allow(role_admin, 'create'),
- M.ACE.allow(role_admin, 'edit'),
- M.ACE.allow(role_admin, 'delete'),
- M.ACE.allow(role_admin, 'moderate'),
- M.ACE.allow(role_admin, 'configure'),
- M.ACE.allow(role_admin, 'admin')]
- # make all the wiki pages
- for page in pages:
- ending = page[-5:]
- beginning = page[:-5]
- markdown_file = '%s.markdown' % beginning
- if '.json' == ending and markdown_file in pages:
- page_data = loadjson(pid, 'wiki', page)
- content = load(pid, 'wiki', markdown_file)
- if page == 'HomePage.json':
- globals = WM.Globals.query.get(
- app_config_id=wiki_app.config._id)
- if globals is not None:
- globals.root = page_data.title
- else:
- globals = WM.Globals(
- app_config_id=wiki_app.config._id, root=page_data.title)
- p = WM.Page.upsert(page_data.title)
- p.text = wiki2markdown(content)
- # upload attachments
- upload_attachments(p, pid, beginning)
- if not p.history().first():
- p.commit()
- ThreadLocalODMSession.flush_all()
-
-
-def import_discussion(project, pid, frs_mapping, sf_project_shortname, nbhd):
- from forgediscussion import model as DM
- discuss_app = project.app_instance('discussion')
- if not discuss_app:
- discuss_app = project.install_app('Discussion', 'discussion')
- h.set_context(project.shortname, 'discussion', neighborhood=nbhd)
- assert c.app
- # set permissions and config options
- role_admin = M.ProjectRole.by_name('Admin')._id
- role_developer = M.ProjectRole.by_name('Developer')._id
- role_auth = M.ProjectRole.by_name('*authenticated')._id
- role_anon = M.ProjectRole.by_name('*anonymous')._id
- discuss_app.config.acl = [
- M.ACE.allow(role_anon, 'read'),
- M.ACE.allow(role_auth, 'post'),
- M.ACE.allow(role_auth, 'unmoderated_post'),
- M.ACE.allow(role_developer, 'moderate'),
- M.ACE.allow(role_admin, 'configure'),
- M.ACE.allow(role_admin, 'admin')]
- ThreadLocalODMSession.flush_all()
- DM.Forum.query.remove(
- dict(app_config_id=discuss_app.config._id, shortname='general'))
- forums = os.listdir(os.path.join(options.output_dir, pid, 'forum'))
- for forum in forums:
- ending = forum[-5:]
- forum_name = forum[:-5]
- if '.json' == ending and forum_name in forums:
- forum_data = loadjson(pid, 'forum', forum)
- fo = DM.Forum.query.get(
- shortname=forum_name, app_config_id=discuss_app.config._id)
- if not fo:
- fo = DM.Forum(app_config_id=discuss_app.config._id,
- shortname=forum_name)
- fo.name = forum_data.title
- fo.description = forum_data.description
- fo_num_topics = 0
- fo_num_posts = 0
- topics = os.listdir(os.path.join(options.output_dir, pid, 'forum',
- forum_name))
- for topic in topics:
- ending = topic[-5:]
- topic_name = topic[:-5]
- if '.json' == ending and topic_name in topics:
- fo_num_topics += 1
- topic_data = loadjson(pid, 'forum', forum_name, topic)
- thread_query = dict(
- subject=topic_data.title,
- discussion_id=fo._id,
- app_config_id=discuss_app.config._id)
- if not options.skip_thread_import_id_when_reloading:
- # temporary/transitional. Just needed the first time
- # running with this new code against an existing import
- # that didn't have import_ids
- thread_query['import_id'] = topic_data.id
- to = DM.ForumThread.query.get(**thread_query)
- if not to:
- to = DM.ForumThread.new(
- subject=topic_data.title,
- discussion_id=fo._id,
- import_id=topic_data.id,
- app_config_id=discuss_app.config._id)
- to.import_id = topic_data.id
- to_num_replies = 0
- oldest_post = None
- newest_post = None
- posts = sorted(
- os.listdir(os.path.join(options.output_dir, pid, 'forum', forum_name, topic_name)))
- for post in posts:
- ending = post[-5:]
- post_name = post[:-5]
- if '.json' == ending:
- to_num_replies += 1
- post_data = loadjson(pid, 'forum',
- forum_name, topic_name, post)
- p = DM.ForumPost.query.get(
- _id='{}{}@import'.format(
- post_name, str(discuss_app.config._id)),
- thread_id=to._id,
- discussion_id=fo._id,
- app_config_id=discuss_app.config._id)
-
- if not p:
- p = DM.ForumPost(
- _id='{}{}@import'.format(
- post_name, str(
- discuss_app.config._id)),
- thread_id=to._id,
- discussion_id=fo._id,
- app_config_id=discuss_app.config._id)
- create_date = datetime.strptime(
- post_data.createdDate, '%Y-%m-%d %H:%M:%S')
- p.timestamp = create_date
- p.author_id = str(
- get_user(post_data.createdByUserName)._id)
- p.text = convert_post_content(
- frs_mapping, sf_project_shortname, post_data.content, nbhd)
- p.status = 'ok'
- if post_data.replyToId:
- p.parent_id = '{}{}@import'.format(
- post_data.replyToId, str(discuss_app.config._id))
- slug, full_slug = p.make_slugs(
- parent=p.parent, timestamp=create_date)
- p.slug = slug
- p.full_slug = full_slug
- if oldest_post is None or oldest_post.timestamp > create_date:
- oldest_post = p
- if newest_post is None or newest_post.timestamp < create_date:
- newest_post = p
- ThreadLocalODMSession.flush_all()
- to.num_replies = to_num_replies
- to.first_post_id = oldest_post._id
- to.last_post_date = newest_post.timestamp
- to.mod_date = newest_post.timestamp
- fo_num_posts += to_num_replies
- fo.num_topics = fo_num_topics
- fo.num_posts = fo_num_posts
- ThreadLocalODMSession.flush_all()
-
-
-def import_news(project, pid, frs_mapping, sf_project_shortname, nbhd):
- from forgeblog import model as BM
- posts = os.listdir(os.path.join(options.output_dir, pid, 'news'))
- if len(posts):
- news_app = project.app_instance('news')
- if not news_app:
- news_app = project.install_app('blog', 'news', mount_label='News')
- h.set_context(project.shortname, 'news', neighborhood=nbhd)
- # make all the blog posts
- for post in posts:
- if '.json' == post[-5:]:
- post_data = loadjson(pid, 'news', post)
- create_date = datetime.strptime(
- post_data.createdOn, '%Y-%m-%d %H:%M:%S')
- p = BM.BlogPost.query.get(title=post_data.title,
- timestamp=create_date,
- app_config_id=news_app.config._id)
- if not p:
- p = BM.BlogPost(title=post_data.title,
- timestamp=create_date,
- app_config_id=news_app.config._id)
- p.text = convert_post_content(
- frs_mapping, sf_project_shortname, post_data.body, nbhd)
- p.mod_date = create_date
- p.state = 'published'
- if not p.slug:
- p.make_slug()
- if not p.history().first():
- p.commit()
- ThreadLocalODMSession.flush_all()
- M.Thread.new(discussion_id=p.app_config.discussion_id,
- ref_id=p.index_id(),
- subject='%s discussion' % p.title)
- user = get_user(post_data.createdByUsername)
- p.history().first().author = dict(
- id=user._id,
- username=user.username,
- display_name=user.get_pref('display_name'))
- ThreadLocalODMSession.flush_all()
-
-
-def check_unsupported_tools(project):
- docs = make_client(options.api_url, 'DocumentApp')
- doc_count = 0
- for doc in docs.service.getDocumentFolderList(s, project.id, recursive=True).dataRows:
- if doc.title == 'Root Folder':
- continue
- doc_count += 1
- if doc_count:
- log.warning('Migrating documents is not supported, but found %s docs' %
- doc_count)
-
- scm = make_client(options.api_url, 'ScmApp')
- for repo in scm.service.getRepositoryList(s, project.id).dataRows:
- log.warning('Migrating SCM repos is not supported, but found %s' %
- repo.repositoryPath)
-
- tasks = make_client(options.api_url, 'TaskApp')
- task_count = len(
- tasks.service.getTaskList(s, project.id, filters=None).dataRows)
- if task_count:
- log.warning('Migrating tasks is not supported, but found %s tasks' %
- task_count)
-
- tracker = make_client(options.api_url, 'TrackerApp')
- tracker_count = len(
- tracker.service.getArtifactList(s, project.id, filters=None).dataRows)
- if tracker_count:
- log.warning(
- 'Migrating trackers is not supported, but found %s tracker artifacts' %
- task_count)
-
-
-def load(project_id, *paths):
- in_file = os.path.join(options.output_dir, project_id, *paths)
- with open(in_file, encoding='utf-8') as input:
- content = input.read()
- return content
-
-
-def loadjson(*args):
- # Object for attribute access
- return json.loads(load(*args), object_hook=Object)
-
-
-def save(content, project, *paths):
- out_file = os.path.join(options.output_dir, project.id, *paths)
- if not os.path.exists(os.path.dirname(out_file)):
- os.makedirs(os.path.dirname(out_file))
- with open(out_file, 'w', encoding='utf-8') as out:
- out.write(content.encode('utf-8'))
-
-
-def download_file(tool, url_path, *filepaths):
- if tool == 'wiki':
- action = 'viewAttachment'
- elif tool == 'frs':
- action = 'downloadFile'
- else:
- raise ValueError('tool %s not supported' % tool)
- action_url = options.attachment_url % (tool, action)
-
- out_file = os.path.join(options.output_dir, *filepaths)
- if not os.path.exists(os.path.dirname(out_file)):
- os.makedirs(os.path.dirname(out_file))
-
- if '://' in url_path:
- url = url_path
- else:
- hostname = urlparse(options.api_url).hostname
- scheme = urlparse(options.api_url).scheme
- url = scheme + '://' + hostname + action_url + six.moves.urllib.parse.quote(url_path)
- log.debug('fetching %s' % url)
-
- resp = loggedInOpener.open(url)
- # if not logged in and this is private, you will get an html response instead of the file
- # log in to make sure the file should really be html
- if resp.headers.type == 'text/html':
- # log in and save the file
- resp = loggedInOpener.open(scheme + '://' + hostname + "/sf/sfmain/do/login", six.moves.urllib.parse.urlencode(
- {'username': options.username, 'password': options.password, 'returnToUrl': url, 'sfsubmit': 'submit'}))
- with open(out_file, 'w', encoding='utf-8') as out:
- out.write(resp.fp.read())
- return out_file
-
-
-bracket_macro = re.compile(r'\[(.*?)\]')
-h1 = re.compile(r'^!!!', re.MULTILINE)
-h2 = re.compile(r'^!!', re.MULTILINE)
-h3 = re.compile(r'^!', re.MULTILINE)
-re_stats = re.compile(r'#+ .* [Ss]tatistics\n+(.*\[sf:.*?Statistics\].*)+')
-
-
-def wiki2markdown(markup):
- '''
- Partial implementation of http://help.collab.net/index.jsp?topic=/teamforge520/reference/wiki-wikisyntax.html
- TODO: __ for bold
- TODO: quote filenames with spaces, e.g. [[img src="foo bar.jpg"]]
- '''
- def bracket_handler(matchobj):
- snippet = matchobj.group(1)
- ext = snippet.rsplit('.')[-1].lower()
- # TODO: support [foo|bar.jpg]
- if snippet.startswith('sf:'):
- # can't handle these macros
- return matchobj.group(0)
- elif ext in ('jpg', 'gif', 'png'):
- filename = snippet.split('/')[-1]
- return '[[img src=%s]]' % filename
- elif '|' in snippet:
- text, link = snippet.split('|', 1)
- return f'[{text}]({link})'
- else:
- # regular link
- return '<%s>' % snippet
- markup = bracket_macro.sub(bracket_handler, markup or '')
- markup = h1.sub('#', markup)
- markup = h2.sub('##', markup)
- markup = h3.sub('###', markup)
-
- markup = re_stats.sub('', markup)
- return markup
-
-
-re_rel = re.compile(r'\b(rel\d+)\b')
-
-
-def convert_post_content(frs_mapping, sf_project_shortname, text, nbhd):
- def rel_handler(matchobj):
- relno = matchobj.group(1)
- path = frs_mapping.get(relno)
- if path:
- return '<a href="/projects/{}.{}/files/{}">{}</a>'.format(
- sf_project_shortname, nbhd.url_prefix.strip('/'), path, path)
- else:
- return relno
- text = re_rel.sub(rel_handler, text or '')
- return text
-
-
-def find_image_references(markup):
- 'yields filenames'
- for matchobj in bracket_macro.finditer(markup):
- snippet = matchobj.group(1)
- ext = snippet.rsplit('.')[-1].lower()
- if ext in ('jpg', 'gif', 'png'):
- yield snippet
-
-
-def get_news(project):
- '''
- Extracts news posts
- '''
- app = make_client(options.api_url, 'NewsApp')
-
- # find the forums
- posts = app.service.getNewsPostList(s, project.id)
- for post in posts.dataRows:
- save(json.dumps(dict(post), default=str),
- project, 'news', post.id + '.json')
- save_user(post.createdByUsername)
-
-
-def get_discussion(project):
- '''
- Extracts discussion forums and posts
- '''
- app = make_client(options.api_url, 'DiscussionApp')
-
- # find the forums
- forums = app.service.getForumList(s, project.id)
- for forum in forums.dataRows:
- forumname = forum.path.split('.')[-1]
- log.info('Retrieving data for forum: %s' % forumname)
- save(json.dumps(dict(forum), default=str), project, 'forum',
- forumname + '.json')
- # topic in this forum
- topics = app.service.getTopicList(s, forum.id)
- for topic in topics.dataRows:
- save(json.dumps(dict(topic), default=str), project, 'forum',
- forumname, topic.id + '.json')
- # posts in this topic
- posts = app.service.getPostList(s, topic.id)
- for post in posts.dataRows:
- save(json.dumps(dict(post), default=str), project, 'forum',
- forumname, topic.id, post.id + '.json')
- save_user(post.createdByUserName)
-
-
-def get_homepage_wiki(project):
- '''
- Extracts home page and wiki pages
- '''
- wiki = make_client(options.api_url, 'WikiApp')
-
- pages = {}
- wiki_pages = wiki.service.getWikiPageList(s, project.id)
- for wiki_page in wiki_pages.dataRows:
- wiki_page = wiki.service.getWikiPageData(s, wiki_page.id)
- pagename = wiki_page.path.split('/')[-1]
- save(json.dumps(dict(wiki_page), default=str),
- project, 'wiki', pagename + '.json')
- if not wiki_page.wikiText:
- log.debug('skip blank wiki page %s' % wiki_page.path)
- continue
- pages[pagename] = wiki_page.wikiText
-
- # PageApp does not provide a useful way to determine the Project Home special wiki page
- # so use some heuristics
- homepage = None
- if '$ProjectHome' in pages and options.default_wiki_text not in pages['$ProjectHome']:
- homepage = pages.pop('$ProjectHome')
- elif 'HomePage' in pages and options.default_wiki_text not in pages['HomePage']:
- homepage = pages.pop('HomePage')
- elif '$ProjectHome' in pages:
- homepage = pages.pop('$ProjectHome')
- elif 'HomePage' in pages:
- homepage = pages.pop('HomePage')
- else:
- log.warning('did not find homepage')
-
- if homepage:
- save(homepage, project, 'wiki', 'homepage_text.markdown')
- for img_ref in find_image_references(homepage):
- filename = img_ref.split('/')[-1]
- if '://' in img_ref:
- img_url = img_ref
- else:
- img_url = project.path + '/wiki/' + img_ref
- download_file('wiki', img_url, project.id,
- 'wiki', 'homepage', filename)
-
- for path, text in pages.items():
- if options.default_wiki_text in text:
- log.debug('skipping default wiki page %s' % path)
- else:
- save(text, project, 'wiki', path + '.markdown')
- for img_ref in find_image_references(text):
- filename = img_ref.split('/')[-1]
- if '://' in img_ref:
- img_url = img_ref
- else:
- img_url = project.path + '/wiki/' + img_ref
- download_file('wiki', img_url, project.id,
- 'wiki', path, filename)
-
-
-def _dir_sql(created_on, project, dir_name, rel_path):
- assert options.neighborhood_shortname
- if not rel_path:
- parent_directory = "'1'"
- else:
- parent_directory = "(SELECT pfs_path FROM pfs_path WHERE path_name = '%s/')" % rel_path
- sql = """
- UPDATE pfs
- SET file_crtime = '{}'
- WHERE source_pk = (SELECT project.project FROM project WHERE project.project_name = '{}.{}')
- AND source_table = 'project'
- AND pfs_type = 'd'
- AND pfs_name = '{}'
- AND parent_directory = {};
- """.format(created_on, convert_project_shortname(project.path), options.neighborhood_shortname,
- dir_name, parent_directory)
- return sql
-
-
-def get_files(project):
- frs = make_client(options.api_url, 'FrsApp')
- valid_pfs_filename = re.compile(
- r'(?![. ])[-_ +.,=#~@!()\[\]a-zA-Z0-9]+(?<! )$')
- pfs_output_dir = os.path.join(
- os.path.abspath(options.output_dir), 'PFS', convert_project_shortname(project.path))
- sql_updates = ''
-
- def handle_path(obj, prev_path):
- path_component = obj.title.strip().replace(
- '/', ' ').replace('&', '').replace(':', '')
- path = os.path.join(prev_path, path_component)
- if not valid_pfs_filename.match(path_component):
- log.error('Invalid filename: "%s"' % path)
- save(json.dumps(dict(obj), default=str),
- project, 'frs', path + '.json')
- return path
-
- frs_mapping = {}
-
- for pkg in frs.service.getPackageList(s, project.id).dataRows:
- pkg_path = handle_path(pkg, '')
- pkg_details = frs.service.getPackageData(s, pkg.id) # download count
- save(json.dumps(dict(pkg_details), default=str),
- project, 'frs', pkg_path + '_details.json')
-
- for rel in frs.service.getReleaseList(s, pkg.id).dataRows:
- rel_path = handle_path(rel, pkg_path)
- frs_mapping[rel['id']] = rel_path
- # download count
- rel_details = frs.service.getReleaseData(s, rel.id)
- save(json.dumps(dict(rel_details), default=str),
- project, 'frs', rel_path + '_details.json')
-
- for file in frs.service.getFrsFileList(s, rel.id).dataRows:
- details = frs.service.getFrsFileData(s, file.id)
-
- file_path = handle_path(file, rel_path)
- save(json.dumps(dict(file,
- lastModifiedBy=details.lastModifiedBy,
- lastModifiedDate=details.lastModifiedDate,
- ),
- default=str),
- project,
- 'frs',
- file_path + '.json'
- )
- if not options.skip_frs_download:
- download_file('frs', rel.path + '/' + file.id,
- pfs_output_dir, file_path)
- mtime = int(mktime(details.lastModifiedDate.timetuple()))
- os.utime(os.path.join(pfs_output_dir, file_path),
- (mtime, mtime))
-
- # releases
- created_on = int(mktime(rel.createdOn.timetuple()))
- mtime = int(mktime(rel.lastModifiedOn.timetuple()))
- if os.path.exists(os.path.join(pfs_output_dir, rel_path)):
- os.utime(os.path.join(pfs_output_dir, rel_path),
- (mtime, mtime))
- sql_updates += _dir_sql(created_on, project,
- rel.title.strip(), pkg_path)
- # packages
- created_on = int(mktime(pkg.createdOn.timetuple()))
- mtime = int(mktime(pkg.lastModifiedOn.timetuple()))
- if os.path.exists(os.path.join(pfs_output_dir, pkg_path)):
- os.utime(os.path.join(pfs_output_dir, pkg_path), (mtime, mtime))
- sql_updates += _dir_sql(created_on, project, pkg.title.strip(), '')
- # save pfs update sql for this project
- with open(os.path.join(options.output_dir, 'pfs_updates.sql'), 'a') as out:
- out.write('/* %s */' % project.id)
- out.write(sql_updates)
- save(json.dumps(frs_mapping), project, 'frs_mapping.json')
-
-
-def get_parser(defaults):
- optparser = OptionParser(
- usage=('%prog [--options] [projID projID projID]\n'
- 'If no project ids are given, all projects will be migrated'))
- optparser.set_defaults(**defaults)
-
- # Command-line-only options
- optparser.add_option(
- '--extract-only', action='store_true', dest='extract',
- help='Store data from the TeamForge API on the local filesystem; not load into Allura')
- optparser.add_option(
- '--load-only', action='store_true', dest='load',
- help='Load into Allura previously-extracted data')
- optparser.add_option(
- '--config-file', dest='config_file',
- help='Load options from config file')
-
- # Command-line options with defaults in config file
- optparser.add_option(
- '--api-url', dest='api_url', help='e.g. https://hostname/ce-soap50/services/')
- optparser.add_option(
- '--attachment-url', dest='attachment_url')
- optparser.add_option(
- '--default-wiki-text', dest='default_wiki_text',
- help='used in determining if a wiki page text is default or changed')
- optparser.add_option(
- '-u', '--username', dest='username')
- optparser.add_option(
- '-p', '--password', dest='password')
- optparser.add_option(
- '-o', '--output-dir', dest='output_dir')
- optparser.add_option(
- '--list-project-ids', action='store_true', dest='list_project_ids')
- optparser.add_option(
- '-n', '--neighborhood', dest='neighborhood',
- help='Neighborhood full name, to load in to')
- optparser.add_option(
- '--n-shortname', dest='neighborhood_shortname',
- help='Neighborhood shortname, for PFS extract SQL')
- optparser.add_option(
- '--skip-thread-import-id-when-reloading', action='store_true',
- dest='skip_thread_import_id_when_reloading'
- )
- optparser.add_option(
- '--skip-frs-download', action='store_true', dest='skip_frs_download')
- optparser.add_option(
- '--skip-wiki', action='store_true', dest='skip_wiki')
- optparser.add_option(
- '--skip-unsupported-check', action='store_true', dest='skip_unsupported_check')
-
- return optparser
-
-
-re_username = re.compile(r"^[a-z\-0-9]+$")
-
-
-def make_valid_sf_username(orig_username):
- sf_username = orig_username.replace('_', '-').lower()
-
- # FIXME username translation is hardcoded here:
- sf_username = dict(
- rlevy='ramilevy',
- mkeisler='mkeisler',
- bthale='bthale',
- mmuller='mattjustmull',
- MalcolmDwyer='slagheap',
- tjyang='tjyang',
- manaic='maniac76',
- srinid='cnudav',
- es='est016',
- david_peyer='david-mmi',
- okruse='ottokruse',
- jvp='jvpmoto',
- dmorelli='dmorelli',
- ).get(sf_username, sf_username + '-mmi')
-
- if not re_username.match(sf_username):
- adjusted_username = ''.join(
- ch for ch in sf_username[:-4]
- if ch.isalnum() or ch == '-') + '-mmi'
- log.error('invalid sf_username characters: %s Changing it to %s',
- sf_username, adjusted_username)
- sf_username = adjusted_username
- if len(sf_username) > 15:
- adjusted_username = sf_username[0:15 - 4] + '-mmi'
- log.error('invalid sf_username length: %s Changing it to %s',
- sf_username, adjusted_username)
- sf_username = adjusted_username
- return sf_username
-
-
-if __name__ == '__main__':
- logging.basicConfig(level=logging.WARN)
- log.setLevel(logging.DEBUG)
- main()
-
-
-def test_make_valid_sf_username():
- tests = {
- # basic
- 'foo': 'foo-mmi',
- # lookup
- 'rlevy': 'ramilevy',
- # too long
- 'u012345678901234567890': 'u0123456789-mmi',
- 'foo^213': 'foo213-mmi'
- }
- for k, v in tests.items():
- assert make_valid_sf_username(k) == v
-
-
-def test_convert_post_content():
- nbhd = Object()
- nbhd.url_prefix = '/motorola/'
- text = '''rel100? or ?rel101 or rel102 or rel103a or rel104'''
- mapping = dict(
- rel100='rel/100/',
- rel101='rel/101/',
- rel102='rel/102/',
- rel103='rel/103/',
- rel104='rel/104/')
- converted = convert_post_content(mapping, 'foo', text, nbhd)
- assert 'href="/projects/foo.motorola/files/rel/100' in converted, converted
- assert 'href="/projects/foo.motorola/files/rel/101' in converted, converted
- assert 'href="/projects/foo.motorola/files/rel/102' in converted, converted
- assert 'href="/projects/foo.motorola/files/rel/103' not in converted, converted
- assert 'href="/projects/foo.motorola/files/rel/104' in converted, converted
-
-
-def test_convert_markup():
-
- markup = '''
-!this is the first headline
-Please note that this project is for distributing, discussing, and supporting the open source software we release.
-
-[http://www.google.com]
-
-[SourceForge |http://www.sf.net]
-
-[$ProjectHome/myimage.jpg]
-[$ProjectHome/anotherimage.jpg]
-
-!!! Project Statistics
-
-|[sf:frsStatistics]|[sf:artifactStatistics]|
- '''
-
- new_markup = wiki2markdown(markup)
- assert '\n[[img src=myimage.jpg]]\n[[img src=anotherimage.jpg]]\n' in new_markup
- assert '\n###this is the first' in new_markup
- assert '<http://www.google.com>' in new_markup
- assert '[SourceForge ](http://www.sf.net)' in new_markup
- assert '\n# Project Statistics' not in new_markup
- assert '[sf:frsStatistics]' not in new_markup