You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2020/10/02 21:44:55 UTC
[allura] branch db/8380 created (now 3bc8d6f)
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a change to branch db/8380
in repository https://gitbox.apache.org/repos/asf/allura.git.
at 3bc8d6f [#8380] expose /rest/*/add_project to admins
This branch includes the following new commits:
new f67d9d9 [#8380] avoid "MongoClient opened before fork" error since pymongo upgrade by removing multiprocessing support. Users can make separate batches of import files and run this script in parallel themselves http://api.mongodb.org/python/current/faq.html#is-pymongo-fork-safe
new 1109c73 [#8380] factor out reusable core from project-import.py
new 3bc8d6f [#8380] expose /rest/*/add_project to admins
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[allura] 03/03: [#8380] expose /rest/*/add_project to admins
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8380
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 3bc8d6f57cc698ebb6f48218db7cae59b178aa7f
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Thu Oct 1 19:21:28 2020 -0400
[#8380] expose /rest/*/add_project to admins
---
Allura/allura/controllers/rest.py | 44 +++++++++-
Allura/allura/lib/plugin.py | 4 +-
Allura/allura/lib/project_create_helpers.py | 21 ++++-
Allura/allura/tests/functional/test_rest.py | 129 ++++++++++++++++++++++++++++
Allura/docs/api-rest/api.raml | 45 ++++++++--
AlluraTest/alluratest/controller.py | 3 +-
AlluraTest/alluratest/validation.py | 2 +-
scripts/project-import.json | 3 +-
scripts/project-import.py | 4 +
9 files changed, 237 insertions(+), 18 deletions(-)
diff --git a/Allura/allura/controllers/rest.py b/Allura/allura/controllers/rest.py
index 2686a54..25c4ce5 100644
--- a/Allura/allura/controllers/rest.py
+++ b/Allura/allura/controllers/rest.py
@@ -20,6 +20,7 @@
"""REST Controller"""
from __future__ import unicode_literals
from __future__ import absolute_import
+import json
import logging
from six.moves.urllib.parse import unquote
@@ -29,16 +30,16 @@ from webob import exc
from tg import expose, flash, redirect, config
from tg import tmpl_context as c, app_globals as g
from tg import request, response
-
+import colander
from ming.orm import session
-from ming.utils import LazyProperty
from allura import model as M
from allura.lib import helpers as h
from allura.lib import security
from allura.lib import plugin
-from allura.lib.exceptions import Invalid
+from allura.lib.exceptions import Invalid, ForgeError
from allura.lib.decorators import require_post
+from allura.lib.project_create_helpers import make_newproject_schema, deserialize_project, create_project_with_attrs
from allura.lib.security import has_access
import six
@@ -361,6 +362,7 @@ def nbhd_lookup_first_path(nbhd, name, current_user, remainder, api=False):
class NeighborhoodRestController(object):
def __init__(self, neighborhood):
+ # type: (M.Neighborhood) -> None
self._neighborhood = neighborhood
@expose('json:')
@@ -374,6 +376,42 @@ class NeighborhoodRestController(object):
c.project, remainder = nbhd_lookup_first_path(self._neighborhood, name, c.user, remainder, api=True)
return ProjectRestController(), remainder
+ @expose('json:')
+ @require_post()
+ def add_project(self, **kw):
+ # TODO: currently limited to 'admin' permissions instead of 'register' since not enough validation is in place.
+ # There is sanity checks and validation that the user may create a project, but not on project fields
+ # for example: tool_data, admins, awards, etc can be set arbitrarily right now
+ # and normal fields like description, summary, external_homepage, troves etc don't have validation on length,
+ # quantity, value etc. which match the HTML web form validations
+ # if/when this is handled better, the following line can be updated. Also update api.raml docs
+ # security.require_access(self._neighborhood, 'register')
+ security.require_access(self._neighborhood, 'admin')
+
+ project_reg = plugin.ProjectRegistrationProvider.get()
+
+ jsondata = json.loads(request.body)
+ projectSchema = make_newproject_schema(self._neighborhood)
+ try:
+ pdata = deserialize_project(jsondata, projectSchema)
+ shortname = pdata.shortname or pdata.name.shortname
+ project_reg.validate_project(self._neighborhood, shortname, pdata.name.name, c.user,
+ user_project=False, private_project=pdata.private)
+ except (colander.Invalid, ForgeError) as e:
+ response.status_int = 400
+ return {
+ 'error': six.text_type(e) or repr(e),
+ }
+
+ project = create_project_with_attrs(pdata, self._neighborhood)
+ response.status_int = 201
+ response.location = '/rest' + project.url()
+ return {
+ "status": "success",
+ "html_url": h.absurl(project.url()),
+ "url": h.absurl('/rest' + project.url()),
+ }
+
class ProjectRestController(object):
diff --git a/Allura/allura/lib/plugin.py b/Allura/allura/lib/plugin.py
index b67abdc..d6175aa 100644
--- a/Allura/allura/lib/plugin.py
+++ b/Allura/allura/lib/plugin.py
@@ -972,8 +972,8 @@ class ProjectRegistrationProvider(object):
# Check for private project rights
if neighborhood.features['private_projects'] is False and private_project:
- raise ValueError(
- "You can't create private projects for %s neighborhood" %
+ raise forge_exc.ForgeError(
+ "You can't create private projects in the %s neighborhood" %
neighborhood.name)
# Check for project limit creation
diff --git a/Allura/allura/lib/project_create_helpers.py b/Allura/allura/lib/project_create_helpers.py
index 28a6500..97f384b 100644
--- a/Allura/allura/lib/project_create_helpers.py
+++ b/Allura/allura/lib/project_create_helpers.py
@@ -18,13 +18,17 @@
from __future__ import unicode_literals
from __future__ import absolute_import
import re
+from io import BytesIO
+
import datetime
import string
import logging
import colander as col
import bson
+import requests
import six
+from six.moves.urllib.parse import urlparse
from allura.model import Neighborhood
from ming.base import Object
@@ -170,6 +174,9 @@ class Labels(col.SequenceSchema):
class NewProjectSchema(col.MappingSchema):
+ def schema_type(self, **kw):
+ return col.Mapping(unknown='raise')
+
name = col.SchemaNode(ProjectNameType())
summary = col.SchemaNode(col.Str(), missing='')
description = col.SchemaNode(col.Str(), missing='')
@@ -188,8 +195,8 @@ class NewProjectSchema(col.MappingSchema):
trove_natlanguages = TroveTranslations(missing=None)
trove_environments = TroveUIs(missing=None)
tool_data = col.SchemaNode(col.Mapping(unknown='preserve'), missing={})
- icon = col.SchemaNode(col.Str(), missing=None)
- # more fields are added dynamically to the schema in main()
+ icon_url = col.SchemaNode(col.Str(), missing=None, validator=col.url)
+ # more fields are added dynamically to the schema in make_newproject_schema()
def trove_ids(orig, new_):
@@ -201,7 +208,7 @@ def trove_ids(orig, new_):
def make_newproject_schema(nbhd, update=False):
# type: (Neighborhood, bool) -> NewProjectSchema
# dynamically add to the schema (e.g. if needs nbhd)
- projectSchema = NewProjectSchema()
+ projectSchema = NewProjectSchema(unknown='raise')
projectSchema.add(col.SchemaNode(col.Sequence(),
col.SchemaNode(Award(nbhd)),
name='awards', missing=[]))
@@ -282,7 +289,13 @@ def create_project_with_attrs(p, nbhd, update=False, ensure_tools=False):
granted_to_project_id=project._id,
granted_by_neighborhood_id=nbhd._id)
- if p.icon:
+ if p.icon_url:
+ req = requests.get(p.icon_url)
+ req.raise_for_status()
+ project.save_icon(urlparse(p.icon_url).path,
+ BytesIO(req.content),
+ content_type=req.headers.get('Content-Type'))
+ elif getattr(p, 'icon'):
with open(p.icon, 'rb') as icon_file:
project.save_icon(p.icon, icon_file)
diff --git a/Allura/allura/tests/functional/test_rest.py b/Allura/allura/tests/functional/test_rest.py
index b885ed4..c66686b 100644
--- a/Allura/allura/tests/functional/test_rest.py
+++ b/Allura/allura/tests/functional/test_rest.py
@@ -19,6 +19,11 @@
from __future__ import unicode_literals
from __future__ import absolute_import
+
+import json
+
+import tg
+from bson import ObjectId
from tg import app_globals as g
import mock
from nose.tools import assert_equal, assert_in, assert_not_in
@@ -408,6 +413,130 @@ class TestRestHome(TestRestApiBase):
assert r.json == {}
+class TestRestNbhdAddProject(TestRestApiBase):
+
+ def setUp(self):
+ super(TestRestNbhdAddProject, self).setUp()
+ # create some troves we'll need
+ M.TroveCategory(fullname="Root", trove_cat_id=1, trove_parent_id=0)
+ M.TroveCategory(fullname="License", trove_cat_id=2, trove_parent_id=1)
+ M.TroveCategory(fullname="Apache License V2.0", fullpath="License :: Apache License V2.0",
+ trove_cat_id=4, trove_parent_id=2)
+ M.TroveCategory(fullname="Public Domain", fullpath="License :: Public Domain",
+ trove_cat_id=5, trove_parent_id=2)
+ p_nbhd = M.Neighborhood.query.get(url_prefix='/p/')
+ p_nbhd.features['private_projects'] = False
+
+ def test_add_project_disallowed(self):
+ self.api_post('/rest/p/add_project', user='*anonymous', status=401)
+ self.api_post('/rest/adobe/add_project', user='test-user', status=403)
+ self.api_post('/rest/p/add_project', user='test-admin', status=403) # not a nbhd admin
+
+ def test_add_project_missing_data(self):
+ project_data = {
+ "shortname": "my-new-proj",
+ }
+ r = self.api_post('/rest/p/add_project',
+ params=json.dumps(project_data),
+ user='root',
+ status=400)
+ assert_in('Required', r.json['error'])
+
+ def test_add_project_bad_data(self):
+ project_data = {
+ "shortname": "my-new-proj",
+ "admin": 'test-admin',
+ "name": "asdf",
+ "trove_licenses": ["asdfasdf"]
+ }
+ r = self.api_post('/rest/p/add_project',
+ params=json.dumps(project_data),
+ user='root',
+ status=400)
+ assert_in('is not a valid trove category', r.json['error'])
+
+ # local icon paths are not allowed through web (have to use icon_url)
+ project_data = {
+ "shortname": "my-new-proj",
+ "admin": 'test-admin',
+ "name": "asdf",
+ "icon": "/local/file/path.png"
+ }
+ r = self.api_post('/rest/p/add_project',
+ params=json.dumps(project_data),
+ user='root',
+ status=400)
+ assert_in('Unrecognized keys in mapping', r.json['error'])
+
+ def test_add_project_name_error(self):
+ project_data = {
+ "shortname": "test",
+ }
+ r = self.api_post('/rest/p/add_project',
+ params=json.dumps(project_data),
+ user='root',
+ status=400)
+ assert_equal('This project name is taken.', r.json['error'])
+
+ project_data = {
+ "shortname": "/?xyz",
+ }
+ r = self.api_post('/rest/p/add_project',
+ params=json.dumps(project_data),
+ user='root',
+ status=400)
+ assert_in('Please use', r.json['error'])
+
+ def test_add_project_create_error(self):
+ project_data = {
+ "shortname": "private-project",
+ "name": "private-project",
+ "admin": 'test-admin',
+ "private": True
+ }
+ r = self.api_post('/rest/p/add_project',
+ params=json.dumps(project_data),
+ user='root',
+ status=400)
+ assert_equal("You can't create private projects in the Projects neighborhood", r.json['error'])
+
+ def test_add_project(self):
+ project_data = {
+ "shortname": "my-new-proj",
+ "admin": "test-admin",
+ "name": "My New Project",
+ "summary": "My summary.",
+ "description": "My description.",
+ "external_homepage": "http://wwww.example.com/test",
+ "labels": ["label1", "label2"],
+ "trove_licenses": [
+ "Apache License V2.0",
+ "Public Domain"],
+ "awards": [],
+ "tool_data": {
+ "allura": {
+ "grouping_threshold": 5
+ }
+ },
+ "icon_url": "https://via.placeholder.com/140x100",
+ }
+ r = self.api_post('/rest/p/add_project',
+ params=json.dumps(project_data),
+ user='root',
+ status=201)
+ assert_equal(r.json, {
+ 'status': 'success',
+ 'html_url': 'http://localhost/p/my-new-proj/',
+ 'url': 'http://localhost/rest/p/my-new-proj/',
+ })
+ p = M.Project.query.get(shortname='my-new-proj')
+ assert_equal(p.name, 'My New Project')
+ assert_equal(len(p.trove_license), 2)
+ assert isinstance(p.trove_license[0], ObjectId), p.trove_license[0]
+ # these are sensitive fields only admins should be able to set:
+ assert_equal(p.get_tool_data('allura', 'grouping_threshold'), 5)
+ assert_equal(p.admins()[0].username, 'test-admin')
+
class TestDoap(TestRestApiBase):
validate_skip = True
diff --git a/Allura/docs/api-rest/api.raml b/Allura/docs/api-rest/api.raml
index f53de14..a36c35e 100755
--- a/Allura/docs/api-rest/api.raml
+++ b/Allura/docs/api-rest/api.raml
@@ -15,6 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+#
+# http://apiworkbench.com/ is a useful tool to edit this file
---
title: Apache Allura
version: 1
@@ -58,6 +61,36 @@ documentation:
default: read
enum: [read, admin, create, update, register]
+ /add_project:
+ description: |
+ Create a new project and set its admins, tools and various fields. Limited to admins. currently
+ post:
+ body:
+ application/json:
+ example: |
+ {
+ "name": "Template Test 1",
+ "shortname": "template-test-1",
+ "summary": "My summary.",
+ "description": "My description.",
+ "admin": "admin1",
+ "external_homepage": "http://wwww.example.com/test",
+ "labels": ["label1", "label2"],
+ "trove_audiences": [
+ "Information Technology",
+ "Developers"],
+ "trove_licenses": [
+ "Apache License V2.0",
+ "GNU General Public License version 2.0 (GPLv2)",
+ "Public Domain"],
+ "awards": [],
+ "tool_data": {
+ "allura": {
+ "grouping_threshold": 5
+ }
+ },
+ "icon": "https://via.placeholder.com/140x100"
+ }
/{project}:
description: |
@@ -66,7 +99,7 @@ documentation:
project:
description: The Project short name.
example: "allura"
- pattern: ([a-zA-Z]+)
+ pattern: ([a-zA-Z0-9-]+)
type: {
project: {
@@ -285,13 +318,13 @@ documentation:
ticket_form.labels:
type: string
required: false
- description: comma-separated list of ticket labels
+ description: comma-separated list of ticket labels
ticket_form.attachment:
type: file
- description: (optional) attachment
+ description: (optional) attachment
required: false
- ticket_form.custom field name:
- description: custom field value
+ ticket_form.custom_field_name:
+ description: custom field value
type: string
required: false
@@ -771,4 +804,4 @@ documentation:
example: "/p/test-project"
tool_name:
description: tool's name for current page
- example: "Git"
\ No newline at end of file
+ example: "Git"
diff --git a/AlluraTest/alluratest/controller.py b/AlluraTest/alluratest/controller.py
index 4372584..4f9258f 100644
--- a/AlluraTest/alluratest/controller.py
+++ b/AlluraTest/alluratest/controller.py
@@ -255,7 +255,8 @@ class TestRestApiBase(TestController):
params = {wrap_args: params}
if status is None:
status = [200, 201, 301, 302]
- params = variabledecode.variable_encode(params, add_repetitions=False)
+ if not isinstance(params, six.string_types):
+ params = variabledecode.variable_encode(params, add_repetitions=False)
token = self.token(user).api_key
headers = {
diff --git a/AlluraTest/alluratest/validation.py b/AlluraTest/alluratest/validation.py
index 11fb131..bddfaa2 100644
--- a/AlluraTest/alluratest/validation.py
+++ b/AlluraTest/alluratest/validation.py
@@ -257,7 +257,7 @@ class PostParamCheckingTestApp(AntiSpamTestApp):
if not params:
return
# params can be raw data (json data post, for example)
- if isinstance(params, six.binary_type):
+ if isinstance(params, (six.binary_type, six.string_types)):
return
# params can be a list or a dict
if hasattr(params, 'items'):
diff --git a/scripts/project-import.json b/scripts/project-import.json
index 543c998..7dd4fa1 100644
--- a/scripts/project-import.json
+++ b/scripts/project-import.json
@@ -18,5 +18,6 @@
"allura": {
"grouping_threshold": 5
}
- }
+ },
+ "icon": "../Allura/allura/public/nf/images/up.png"
}]
diff --git a/scripts/project-import.py b/scripts/project-import.py
index 914d453..0cba5db 100644
--- a/scripts/project-import.py
+++ b/scripts/project-import.py
@@ -23,6 +23,7 @@ import sys
from io import open
from ming.orm import session
+import colander
from allura import model as M
from allura.lib.project_create_helpers import create_project_with_attrs, make_newproject_schema, deserialize_project
@@ -43,6 +44,9 @@ def main(options):
data = json.load(open(options.file, 'r'))
projectSchema = make_newproject_schema(nbhd, options.update)
+ # allow 'icon' as a local filesystem path via this script only
+ projectSchema.add(colander.SchemaNode(colander.Str(), name='icon', missing=None))
+
projects = []
for datum in data:
try:
[allura] 02/03: [#8380] factor out reusable core from
project-import.py
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8380
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 1109c7380efbb33f968b785e735b0f47a0ed8b0e
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Thu Oct 1 13:40:03 2020 -0400
[#8380] factor out reusable core from project-import.py
---
.../allura/lib/project_create_helpers.py | 163 +++--------
scripts/project-import.py | 310 ++-------------------
2 files changed, 66 insertions(+), 407 deletions(-)
diff --git a/scripts/project-import.py b/Allura/allura/lib/project_create_helpers.py
similarity index 70%
copy from scripts/project-import.py
copy to Allura/allura/lib/project_create_helpers.py
index a121dfc..28a6500 100644
--- a/scripts/project-import.py
+++ b/Allura/allura/lib/project_create_helpers.py
@@ -17,28 +17,33 @@
from __future__ import unicode_literals
from __future__ import absolute_import
-import bson
-import datetime
-import json
-import logging
import re
+import datetime
import string
-import sys
+import logging
import colander as col
-from ming.orm import session, ThreadLocalORMSession
+import bson
+import six
+
+from allura.model import Neighborhood
+from ming.base import Object
+from ming.orm import ThreadLocalORMSession
from tg import tmpl_context as c, app_globals as g
from allura import model as M
from allura.lib import helpers as h
from allura.lib.plugin import ProjectRegistrationProvider
-import six
-from io import open
-from six.moves import range
+
log = logging.getLogger(__name__)
+"""
+Validators and helpers for adding projects via the API or project-import.py script
+"""
+
+
class TroveCategory():
def __init__(self, root_type=''):
@@ -164,7 +169,7 @@ class Labels(col.SequenceSchema):
label = col.SchemaNode(col.Str())
-class Project(col.MappingSchema):
+class NewProjectSchema(col.MappingSchema):
name = col.SchemaNode(ProjectNameType())
summary = col.SchemaNode(col.Str(), missing='')
description = col.SchemaNode(col.Str(), missing='')
@@ -187,62 +192,56 @@ class Project(col.MappingSchema):
# more fields are added dynamically to the schema in main()
-def valid_shortname(project):
- if project.shortname:
- # already validated in ProjectShortnameType validator
- return True
- elif 3 <= len(project.name.shortname) <= 15:
- return True
- else:
- return 'Project shortname "%s" must be between 3 and 15 characters' \
- % project.name.shortname
-
-
-class Projects(col.SequenceSchema):
- project = Project(validator=col.Function(valid_shortname))
-
-
-class Object(object):
-
- def __init__(self, d):
- self.__dict__.update(d)
-
-
def trove_ids(orig, new_):
if new_ is None:
return orig
return list(set(t._id for t in list(new_)))
-def create_project(p, nbhd, options):
+def make_newproject_schema(nbhd, update=False):
+ # type: (Neighborhood, bool) -> NewProjectSchema
+ # dynamically add to the schema (e.g. if needs nbhd)
+ projectSchema = NewProjectSchema()
+ projectSchema.add(col.SchemaNode(col.Sequence(),
+ col.SchemaNode(Award(nbhd)),
+ name='awards', missing=[]))
+ projectSchema.add(col.SchemaNode(ProjectShortnameType(nbhd, update),
+ name='shortname', missing=None))
+ return projectSchema
+
+
+def deserialize_project(datum, projectSchema):
+ # type: (dict, NewProjectSchema) -> object
+ p = projectSchema.deserialize(datum)
+ p = Object(p) # convert from dict to something with attr-access
+ return p
+
+
+def create_project_with_attrs(p, nbhd, update=False, ensure_tools=False):
+ # type: (object, M.Neighborhood, bool, bool) -> Union[M.Project|bool]
M.session.artifact_orm_session._get().skip_mod_date = True
shortname = p.shortname or p.name.shortname
project = M.Project.query.get(shortname=shortname,
neighborhood_id=nbhd._id)
project_template = nbhd.get_project_template()
- if project and not (options.update and p.shortname):
+ if project and not (update and p.shortname):
log.warning('Skipping existing project "%s". To update an existing '
'project you must provide the project shortname and run '
'this script with --update.' % (shortname))
- return 0
+ return False
if not project:
- log.info('Creating project "%s".' % (shortname))
- try:
- project = nbhd.register_project(shortname,
- p.admin,
- project_name=p.name.name,
- private_project=p.private)
- except Exception as e:
- log.exception('%s' % (str(e)))
- return 0
+ project = nbhd.register_project(shortname,
+ p.admin,
+ project_name=p.name.name,
+ private_project=p.private)
else:
log.info('Updating project "%s".' % (shortname))
project.notifications_disabled = True
- if options.ensure_tools and 'tools' in project_template:
+ if ensure_tools and 'tools' in project_template:
for i, tool in enumerate(six.iterkeys(project_template['tools'])):
tool_config = project_template['tools'][tool]
if project.app_instance(tool_config['mount_point']):
@@ -291,78 +290,4 @@ def create_project(p, nbhd, options):
with h.push_config(c, project=project, user=p.admin):
ThreadLocalORMSession.flush_all()
g.post_event('project_updated')
- session(project).clear()
- return 0
-
-
-def create_projects(projects, nbhd, options):
- for p in projects:
- r = create_project(Object(p), nbhd, options)
- if r != 0:
- sys.exit(r)
-
-
-def main(options):
- log.addHandler(logging.StreamHandler(sys.stdout))
- log.setLevel(getattr(logging, options.log_level.upper()))
- log.debug(options)
-
- nbhd = M.Neighborhood.query.get(name=options.neighborhood)
- if not nbhd:
- return 'Invalid neighborhood "%s".' % options.neighborhood
-
- data = json.load(open(options.file, 'r'))
-
- # dynamically add to the schema (e.g. if needs nbhd)
- project = Project()
- project.add(col.SchemaNode(col.Sequence(),
- col.SchemaNode(Award(nbhd)),
- name='awards', missing=[]))
- project.add(col.SchemaNode(ProjectShortnameType(nbhd, options.update),
- name='shortname', missing=None))
-
- projects = []
- for datum in data:
- try:
- projects.append(project.deserialize(datum))
- except Exception:
- keep_going = options.validate_only
- log.error('Error on %s\n%s', datum['shortname'], datum, exc_info=keep_going)
- if not keep_going:
- raise
-
- log.debug(projects)
-
- if options.validate_only:
- return
-
- create_projects(projects, nbhd, options)
-
- return 0
-
-
-def parse_options():
- import argparse
- parser = argparse.ArgumentParser(
- description='Import Allura project(s) from JSON file')
- parser.add_argument('file', metavar='JSON_FILE', type=str,
- help='Path to JSON file containing project data.')
- parser.add_argument('neighborhood', metavar='NEIGHBORHOOD', type=str,
- help='Destination Neighborhood shortname.')
- parser.add_argument('--log', dest='log_level', default='INFO',
- help='Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL).')
- parser.add_argument('--update', dest='update', default=False,
- action='store_true',
- help='Update existing projects. Without this option, existing '
- 'projects will be skipped.')
- parser.add_argument('--ensure-tools', dest='ensure_tools', default=False,
- action='store_true',
- help='Check nbhd project template for default tools, and install '
- 'them on the project(s) if not already installed.')
- parser.add_argument('--validate-only', '-v', action='store_true', dest='validate_only',
- help='Validate ALL records, make no changes')
- return parser.parse_args()
-
-
-if __name__ == '__main__':
- sys.exit(main(parse_options()))
+ return project
diff --git a/scripts/project-import.py b/scripts/project-import.py
index a121dfc..914d453 100644
--- a/scripts/project-import.py
+++ b/scripts/project-import.py
@@ -17,294 +17,23 @@
from __future__ import unicode_literals
from __future__ import absolute_import
-import bson
-import datetime
import json
import logging
-import re
-import string
import sys
+from io import open
-import colander as col
-from ming.orm import session, ThreadLocalORMSession
-from tg import tmpl_context as c, app_globals as g
+from ming.orm import session
from allura import model as M
-from allura.lib import helpers as h
-from allura.lib.plugin import ProjectRegistrationProvider
-import six
-from io import open
-from six.moves import range
+from allura.lib.project_create_helpers import create_project_with_attrs, make_newproject_schema, deserialize_project
log = logging.getLogger(__name__)
-class TroveCategory():
-
- def __init__(self, root_type=''):
- self.root_type = root_type
-
- def deserialize(self, node, cstruct):
- if cstruct is col.null:
- return col.null
- cat = M.TroveCategory.query.get(fullpath=cstruct)
- if not cat:
- cat = M.TroveCategory.query.get(fullname=cstruct)
- if not cat:
- raise col.Invalid(node,
- '"%s" is not a valid trove category.' % cstruct)
- if not cat.fullpath.startswith(self.root_type):
- raise col.Invalid(node,
- '"%s" is not a valid "%s" trove category.' %
- (cstruct, self.root_type))
- return cat
-
-
-class User():
-
- def deserialize(self, node, cstruct):
- if cstruct is col.null:
- return col.null
- user = M.User.by_username(cstruct)
- if not user:
- raise col.Invalid(node,
- 'Invalid username "%s".' % cstruct)
- return user
-
-
-class ProjectName(object):
-
- def __init__(self, name, shortname):
- self.name = name
- self.shortname = shortname
-
-
-class ProjectNameType():
-
- def deserialize(self, node, cstruct):
- if cstruct is col.null:
- return col.null
- name = cstruct
- shortname = re.sub("[^A-Za-z0-9 ]", "", name).lower()
- shortname = re.sub(" ", "-", shortname)
- return ProjectName(name, shortname)
-
-
-class ProjectShortnameType():
-
- def __init__(self, nbhd, update):
- self.nbhd = nbhd
- self.update = update
-
- def deserialize(self, node, cstruct):
- if cstruct is col.null:
- return col.null
- return ProjectRegistrationProvider.get().shortname_validator.to_python(cstruct,
- check_allowed=not self.update,
- neighborhood=self.nbhd)
-
-
-class Award():
-
- def __init__(self, nbhd):
- self.nbhd = nbhd
-
- def deserialize(self, node, cstruct):
- if cstruct is col.null:
- return col.null
- award = M.Award.query.find(dict(short=cstruct,
- created_by_neighborhood_id=self.nbhd._id)).first()
- if not award:
- # try to look up the award by _id
- award = M.Award.query.find(dict(_id=bson.ObjectId(cstruct),
- created_by_neighborhood_id=self.nbhd._id)).first()
- if not award:
- raise col.Invalid(node,
- 'Invalid award "%s".' % cstruct)
- return award
-
-
-class TroveTopics(col.SequenceSchema):
- trove_topics = col.SchemaNode(TroveCategory("Topic"))
-
-
-class TroveLicenses(col.SequenceSchema):
- trove_license = col.SchemaNode(TroveCategory("License"))
-
-
-class TroveDatabases(col.SequenceSchema):
- trove_databases = col.SchemaNode(TroveCategory("Database Environment"))
-
-
-class TroveStatuses(col.SequenceSchema):
- trove_statuses = col.SchemaNode(TroveCategory("Development Status"))
-
-
-class TroveAudiences(col.SequenceSchema):
- trove_audience = col.SchemaNode(TroveCategory("Intended Audience"))
-
-
-class TroveOSes(col.SequenceSchema):
- trove_oses = col.SchemaNode(TroveCategory("Operating System"))
-
-
-class TroveLanguages(col.SequenceSchema):
- trove_languages = col.SchemaNode(TroveCategory("Programming Language"))
-
-
-class TroveTranslations(col.SequenceSchema):
- trove_translations = col.SchemaNode(TroveCategory("Translations"))
-
-
-class TroveUIs(col.SequenceSchema):
- trove_uis = col.SchemaNode(TroveCategory("User Interface"))
-
-
-class Labels(col.SequenceSchema):
- label = col.SchemaNode(col.Str())
-
-
-class Project(col.MappingSchema):
- name = col.SchemaNode(ProjectNameType())
- summary = col.SchemaNode(col.Str(), missing='')
- description = col.SchemaNode(col.Str(), missing='')
- admin = col.SchemaNode(User())
- private = col.SchemaNode(col.Bool(), missing=False)
- labels = Labels(missing=[])
- external_homepage = col.SchemaNode(col.Str(), missing='')
- video_url = col.SchemaNode(col.Str(), missing='')
- trove_root_databases = TroveDatabases(missing=None)
- trove_developmentstatuses = TroveStatuses(validator=col.Length(max=6), missing=None)
- trove_audiences = TroveAudiences(validator=col.Length(max=6), missing=None)
- trove_licenses = TroveLicenses(validator=col.Length(max=6), missing=None)
- trove_oses = TroveOSes(missing=None)
- trove_languages = TroveLanguages(validator=col.Length(max=6), missing=None)
- trove_topics = TroveTopics(validator=col.Length(max=3), missing=None)
- trove_natlanguages = TroveTranslations(missing=None)
- trove_environments = TroveUIs(missing=None)
- tool_data = col.SchemaNode(col.Mapping(unknown='preserve'), missing={})
- icon = col.SchemaNode(col.Str(), missing=None)
- # more fields are added dynamically to the schema in main()
-
-
-def valid_shortname(project):
- if project.shortname:
- # already validated in ProjectShortnameType validator
- return True
- elif 3 <= len(project.name.shortname) <= 15:
- return True
- else:
- return 'Project shortname "%s" must be between 3 and 15 characters' \
- % project.name.shortname
-
-
-class Projects(col.SequenceSchema):
- project = Project(validator=col.Function(valid_shortname))
-
-
-class Object(object):
-
- def __init__(self, d):
- self.__dict__.update(d)
-
-
-def trove_ids(orig, new_):
- if new_ is None:
- return orig
- return list(set(t._id for t in list(new_)))
-
-
-def create_project(p, nbhd, options):
- M.session.artifact_orm_session._get().skip_mod_date = True
- shortname = p.shortname or p.name.shortname
- project = M.Project.query.get(shortname=shortname,
- neighborhood_id=nbhd._id)
- project_template = nbhd.get_project_template()
-
- if project and not (options.update and p.shortname):
- log.warning('Skipping existing project "%s". To update an existing '
- 'project you must provide the project shortname and run '
- 'this script with --update.' % (shortname))
- return 0
-
- if not project:
- log.info('Creating project "%s".' % (shortname))
- try:
- project = nbhd.register_project(shortname,
- p.admin,
- project_name=p.name.name,
- private_project=p.private)
- except Exception as e:
- log.exception('%s' % (str(e)))
- return 0
- else:
- log.info('Updating project "%s".' % (shortname))
-
- project.notifications_disabled = True
-
- if options.ensure_tools and 'tools' in project_template:
- for i, tool in enumerate(six.iterkeys(project_template['tools'])):
- tool_config = project_template['tools'][tool]
- if project.app_instance(tool_config['mount_point']):
- continue
- tool_options = tool_config.get('options', {})
- for k, v in six.iteritems(tool_options):
- if isinstance(v, six.string_types):
- tool_options[k] = string.Template(v).safe_substitute(
- project.root_project.__dict__.get('root_project', {}))
- project.install_app(tool,
- mount_label=tool_config['label'],
- mount_point=tool_config['mount_point'],
- **tool_options)
-
- project.summary = p.summary
- project.short_description = p.description
- project.external_homepage = p.external_homepage
- project.video_url = p.video_url
- project.last_updated = datetime.datetime.utcnow()
- # These properties may have been populated by nbhd template defaults in
- # register_project(). Overwrite if we have data, otherwise keep defaults.
- project.labels = p.labels or project.labels
- project.trove_root_database = trove_ids(project.trove_root_database, p.trove_root_databases)
- project.trove_developmentstatus = trove_ids(project.trove_developmentstatus, p.trove_developmentstatuses)
- project.trove_audience = trove_ids(project.trove_audience, p.trove_audiences)
- project.trove_license = trove_ids(project.trove_license, p.trove_licenses)
- project.trove_os = trove_ids(project.trove_os, p.trove_oses)
- project.trove_language = trove_ids(project.trove_language, p.trove_languages)
- project.trove_topic = trove_ids(project.trove_topic, p.trove_topics)
- project.trove_natlanguage = trove_ids(project.trove_natlanguage, p.trove_natlanguages)
- project.trove_environment = trove_ids(project.trove_environment, p.trove_environments)
-
- project.tool_data.update(p.tool_data)
-
- for a in p.awards:
- M.AwardGrant(app_config_id=bson.ObjectId(),
- award_id=a._id,
- granted_to_project_id=project._id,
- granted_by_neighborhood_id=nbhd._id)
-
- if p.icon:
- with open(p.icon, 'rb') as icon_file:
- project.save_icon(p.icon, icon_file)
-
- project.notifications_disabled = False
- with h.push_config(c, project=project, user=p.admin):
- ThreadLocalORMSession.flush_all()
- g.post_event('project_updated')
- session(project).clear()
- return 0
-
-
-def create_projects(projects, nbhd, options):
- for p in projects:
- r = create_project(Object(p), nbhd, options)
- if r != 0:
- sys.exit(r)
-
-
def main(options):
- log.addHandler(logging.StreamHandler(sys.stdout))
- log.setLevel(getattr(logging, options.log_level.upper()))
+ root_logger = logging.getLogger()
+ root_logger.addHandler(logging.StreamHandler(sys.stdout))
+ root_logger.setLevel(getattr(logging, options.log_level.upper()))
log.debug(options)
nbhd = M.Neighborhood.query.get(name=options.neighborhood)
@@ -313,18 +42,11 @@ def main(options):
data = json.load(open(options.file, 'r'))
- # dynamically add to the schema (e.g. if needs nbhd)
- project = Project()
- project.add(col.SchemaNode(col.Sequence(),
- col.SchemaNode(Award(nbhd)),
- name='awards', missing=[]))
- project.add(col.SchemaNode(ProjectShortnameType(nbhd, options.update),
- name='shortname', missing=None))
-
+ projectSchema = make_newproject_schema(nbhd, options.update)
projects = []
for datum in data:
try:
- projects.append(project.deserialize(datum))
+ projects.append(deserialize_project(datum, projectSchema))
except Exception:
keep_going = options.validate_only
log.error('Error on %s\n%s', datum['shortname'], datum, exc_info=keep_going)
@@ -336,8 +58,20 @@ def main(options):
if options.validate_only:
return
- create_projects(projects, nbhd, options)
+ for p in projects:
+ shortname = p.shortname or p.name.shortname
+ log.info('Creating%s project "%s".' % ('/updating' if options.update else '', shortname))
+ try:
+ project = create_project_with_attrs(p, nbhd, update=options.update, ensure_tools=options.ensure_tools)
+ except Exception as e:
+ log.exception('%s' % (str(e)))
+ project = False
+ if not project:
+ log.warning('Stopping due to error.')
+ return 1
+ session(project).clear()
+ log.warning('Done.')
return 0
@@ -348,7 +82,7 @@ def parse_options():
parser.add_argument('file', metavar='JSON_FILE', type=str,
help='Path to JSON file containing project data.')
parser.add_argument('neighborhood', metavar='NEIGHBORHOOD', type=str,
- help='Destination Neighborhood shortname.')
+ help='Neighborhood name like "Projects"')
parser.add_argument('--log', dest='log_level', default='INFO',
help='Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL).')
parser.add_argument('--update', dest='update', default=False,
[allura] 01/03: [#8380] avoid "MongoClient opened before fork"
error since pymongo upgrade by removing multiprocessing support. Users can
make separate batches of import files and run this script in parallel
themselves
http://api.mongodb.org/python/current/faq.html#is-pymongo-fork-safe
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8380
in repository https://gitbox.apache.org/repos/asf/allura.git
commit f67d9d9ed6ed177274d74723b8bf94f0745e1d74
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Wed Sep 30 16:44:30 2020 -0400
[#8380] avoid "MongoClient opened before fork" error since pymongo upgrade by removing multiprocessing support. Users can make separate batches of import files and run this script in parallel themselves http://api.mongodb.org/python/current/faq.html#is-pymongo-fork-safe
---
scripts/project-import.py | 30 +++++++-----------------------
1 file changed, 7 insertions(+), 23 deletions(-)
diff --git a/scripts/project-import.py b/scripts/project-import.py
index 6c153fc..a121dfc 100644
--- a/scripts/project-import.py
+++ b/scripts/project-import.py
@@ -21,7 +21,6 @@ import bson
import datetime
import json
import logging
-import multiprocessing
import re
import string
import sys
@@ -216,7 +215,6 @@ def trove_ids(orig, new_):
def create_project(p, nbhd, options):
- worker_name = multiprocessing.current_process().name
M.session.artifact_orm_session._get().skip_mod_date = True
shortname = p.shortname or p.name.shortname
project = M.Project.query.get(shortname=shortname,
@@ -224,23 +222,23 @@ def create_project(p, nbhd, options):
project_template = nbhd.get_project_template()
if project and not (options.update and p.shortname):
- log.warning('[%s] Skipping existing project "%s". To update an existing '
+ log.warning('Skipping existing project "%s". To update an existing '
'project you must provide the project shortname and run '
- 'this script with --update.' % (worker_name, shortname))
+ 'this script with --update.' % (shortname))
return 0
if not project:
- log.info('[%s] Creating project "%s".' % (worker_name, shortname))
+ log.info('Creating project "%s".' % (shortname))
try:
project = nbhd.register_project(shortname,
p.admin,
project_name=p.name.name,
private_project=p.private)
except Exception as e:
- log.exception('[%s] %s' % (worker_name, str(e)))
+ log.exception('%s' % (str(e)))
return 0
else:
- log.info('[%s] Updating project "%s".' % (worker_name, shortname))
+ log.info('Updating project "%s".' % (shortname))
project.notifications_disabled = True
@@ -338,18 +336,8 @@ def main(options):
if options.validate_only:
return
- chunks = [projects[i::options.nprocs] for i in range(options.nprocs)]
- jobs = []
- for i in range(options.nprocs):
- p = multiprocessing.Process(target=create_projects,
- args=(chunks[i], nbhd, options), name='worker-' + str(i + 1))
- jobs.append(p)
- p.start()
-
- for j in jobs:
- j.join()
- if j.exitcode != 0:
- return j.exitcode
+ create_projects(projects, nbhd, options)
+
return 0
@@ -371,10 +359,6 @@ def parse_options():
action='store_true',
help='Check nbhd project template for default tools, and install '
'them on the project(s) if not already installed.')
- parser.add_argument(
- '--nprocs', '-n', action='store', dest='nprocs', type=int,
- help='Number of processes to divide the work among.',
- default=multiprocessing.cpu_count())
parser.add_argument('--validate-only', '-v', action='store_true', dest='validate_only',
help='Validate ALL records, make no changes')
return parser.parse_args()