You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by ke...@apache.org on 2018/02/07 19:09:53 UTC
[1/2] allura git commit: Add tool_data field,
use ProjectRegistrationProvider shortname validator, cleanup
Repository: allura
Updated Branches:
refs/heads/master 6b6fbf582 -> d9a4fc1ec
Add tool_data field, use ProjectRegistrationProvider shortname validator, cleanup
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/3e8e548e
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/3e8e548e
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/3e8e548e
Branch: refs/heads/master
Commit: 3e8e548e3954139da1701abf4af738e02593e2f1
Parents: 166f869
Author: Dave Brondsema <da...@brondsema.net>
Authored: Tue Feb 6 10:39:24 2018 -0500
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Tue Feb 6 15:22:34 2018 -0500
----------------------------------------------------------------------
Allura/allura/command/create_neighborhood.py | 2 +-
Allura/allura/lib/widgets/forms.py | 3 +-
scripts/project-import.json | 9 ++-
scripts/project-import.py | 87 +++++++++++++----------
4 files changed, 59 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/3e8e548e/Allura/allura/command/create_neighborhood.py
----------------------------------------------------------------------
diff --git a/Allura/allura/command/create_neighborhood.py b/Allura/allura/command/create_neighborhood.py
index d099c46..dc4b2cd 100644
--- a/Allura/allura/command/create_neighborhood.py
+++ b/Allura/allura/command/create_neighborhood.py
@@ -43,7 +43,7 @@ class CreateNeighborhoodCommand(base.Command):
name=shortname,
url_prefix='/' + shortname + '/',
features=dict(private_projects=False,
- max_projects=500,
+ max_projects=None,
css='none',
google_analytics=False))
project_reg = plugin.ProjectRegistrationProvider.get()
http://git-wip-us.apache.org/repos/asf/allura/blob/3e8e548e/Allura/allura/lib/widgets/forms.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/widgets/forms.py b/Allura/allura/lib/widgets/forms.py
index d92471c..80d3877 100644
--- a/Allura/allura/lib/widgets/forms.py
+++ b/Allura/allura/lib/widgets/forms.py
@@ -75,8 +75,7 @@ class NeighborhoodProjectShortNameValidator(fev.FancyValidator):
otherwise disallowed.
"""
if neighborhood is None:
- neighborhood = M.Neighborhood.query.get(
- name=state.full_dict['neighborhood'])
+ neighborhood = M.Neighborhood.query.get(name=state.full_dict['neighborhood'])
value = h.really_unicode(value or '').encode('utf-8').lower()
self._validate_shortname(value, neighborhood, state)
if check_allowed:
http://git-wip-us.apache.org/repos/asf/allura/blob/3e8e548e/scripts/project-import.json
----------------------------------------------------------------------
diff --git a/scripts/project-import.json b/scripts/project-import.json
index 47c9505..2b94de1 100644
--- a/scripts/project-import.json
+++ b/scripts/project-import.json
@@ -13,5 +13,10 @@
"Apache License V2.0",
"GNU General Public License (GPL)",
"Public Domain"],
- "awards": ["What?"]
- }]
+ "awards": ["What?"],
+ "tool_data": {
+ "allura": {
+ "grouping_threshold": 5
+ }
+ }
+}]
http://git-wip-us.apache.org/repos/asf/allura/blob/3e8e548e/scripts/project-import.py
----------------------------------------------------------------------
diff --git a/scripts/project-import.py b/scripts/project-import.py
index b42a6fd..587d091 100644
--- a/scripts/project-import.py
+++ b/scripts/project-import.py
@@ -25,13 +25,12 @@ import string
import sys
import colander as col
-
from ming.orm import session, ThreadLocalORMSession
from pylons import tmpl_context as c, app_globals as g
-from tg import config
from allura import model as M
from allura.lib import helpers as h
+from allura.lib.plugin import ProjectRegistrationProvider
log = logging.getLogger(__name__)
@@ -89,15 +88,16 @@ class ProjectNameType():
class ProjectShortnameType():
+ def __init__(self, nbhd, update):
+ self.nbhd = nbhd
+ self.update = update
+
def deserialize(self, node, cstruct):
if cstruct is col.null:
return col.null
- col.Length(min=3, max=15)(node, cstruct)
- col.Regex(r'^[A-z][-A-z0-9]{2,}$',
- msg='Project shortname must begin with a letter, can '
- 'contain letters, numbers, and dashes, and must be '
- '3-15 characters in length.')(node, cstruct)
- return cstruct.lower()
+ return ProjectRegistrationProvider.get().shortname_validator.to_python(cstruct,
+ check_allowed=not self.update,
+ neighborhood=self.nbhd)
class Award():
@@ -162,7 +162,6 @@ class Labels(col.SequenceSchema):
class Project(col.MappingSchema):
name = col.SchemaNode(ProjectNameType())
- shortname = col.SchemaNode(ProjectShortnameType(), missing=None)
summary = col.SchemaNode(col.Str(), missing='')
description = col.SchemaNode(col.Str(), missing='')
admin = col.SchemaNode(User())
@@ -170,8 +169,7 @@ class Project(col.MappingSchema):
labels = Labels(missing=[])
external_homepage = col.SchemaNode(col.Str(), missing='')
trove_root_databases = TroveDatabases(missing=None)
- trove_developmentstatuses = TroveStatuses(
- validator=col.Length(max=6), missing=None)
+ trove_developmentstatuses = TroveStatuses(validator=col.Length(max=6), missing=None)
trove_audiences = TroveAudiences(validator=col.Length(max=6), missing=None)
trove_licenses = TroveLicenses(validator=col.Length(max=6), missing=None)
trove_oses = TroveOSes(missing=None)
@@ -179,6 +177,8 @@ class Project(col.MappingSchema):
trove_topics = TroveTopics(validator=col.Length(max=3), missing=None)
trove_natlanguages = TroveTranslations(missing=None)
trove_environments = TroveUIs(missing=None)
+ tool_data = col.SchemaNode(col.Mapping(unknown='preserve'), missing={})
+ # more fields are added dynamically to the schema in main()
def valid_shortname(project):
@@ -208,7 +208,7 @@ def trove_ids(orig, new_):
return list(set(t._id for t in list(new_)))
-def create_project(p, nbhd, user, options):
+def create_project(p, nbhd, options):
worker_name = multiprocessing.current_process().name
M.session.artifact_orm_session._get().skip_mod_date = True
shortname = p.shortname or p.name.shortname
@@ -225,10 +225,10 @@ def create_project(p, nbhd, user, options):
if not project:
log.info('[%s] Creating project "%s".' % (worker_name, shortname))
try:
- project = nbhd.register_project(shortname,
- p.admin,
- project_name=p.name.name,
- private_project=p.private)
+ project = nbhd.register_project(shortname,
+ p.admin,
+ project_name=p.name.name,
+ private_project=p.private)
except Exception, e:
log.exception('[%s] %s' % (worker_name, str(e)))
return 0
@@ -259,21 +259,17 @@ def create_project(p, nbhd, user, options):
# These properties may have been populated by nbhd template defaults in
# register_project(). Overwrite if we have data, otherwise keep defaults.
project.labels = p.labels or project.labels
- project.trove_root_database = trove_ids(
- project.trove_root_database, p.trove_root_databases)
- project.trove_developmentstatus = trove_ids(
- project.trove_developmentstatus, p.trove_developmentstatuses)
- project.trove_audience = trove_ids(
- project.trove_audience, p.trove_audiences)
+ project.trove_root_database = trove_ids(project.trove_root_database, p.trove_root_databases)
+ project.trove_developmentstatus = trove_ids(project.trove_developmentstatus, p.trove_developmentstatuses)
+ project.trove_audience = trove_ids(project.trove_audience, p.trove_audiences)
project.trove_license = trove_ids(project.trove_license, p.trove_licenses)
project.trove_os = trove_ids(project.trove_os, p.trove_oses)
- project.trove_language = trove_ids(
- project.trove_language, p.trove_languages)
+ project.trove_language = trove_ids(project.trove_language, p.trove_languages)
project.trove_topic = trove_ids(project.trove_topic, p.trove_topics)
- project.trove_natlanguage = trove_ids(
- project.trove_natlanguage, p.trove_natlanguages)
- project.trove_environment = trove_ids(
- project.trove_environment, p.trove_environments)
+ project.trove_natlanguage = trove_ids(project.trove_natlanguage, p.trove_natlanguages)
+ project.trove_environment = trove_ids(project.trove_environment, p.trove_environments)
+
+ project.tool_data.update(p.tool_data)
for a in p.awards:
M.AwardGrant(app_config_id=bson.ObjectId(),
@@ -281,16 +277,16 @@ def create_project(p, nbhd, user, options):
granted_to_project_id=project._id,
granted_by_neighborhood_id=nbhd._id)
project.notifications_disabled = False
- with h.push_config(c, project=project, user=user):
+ with h.push_config(c, project=project, user=p.admin):
ThreadLocalORMSession.flush_all()
g.post_event('project_updated')
session(project).clear()
return 0
-def create_projects(projects, nbhd, user, options):
+def create_projects(projects, nbhd, options):
for p in projects:
- r = create_project(Object(p), nbhd, user, options)
+ r = create_project(Object(p), nbhd, options)
if r != 0:
sys.exit(r)
@@ -303,29 +299,43 @@ def main(options):
nbhd = M.Neighborhood.query.get(name=options.neighborhood)
if not nbhd:
return 'Invalid neighborhood "%s".' % options.neighborhood
- admin = M.User.query.get(
- username=config.get('sfx.api.siteadmin', 'sf-robot'))
data = json.load(open(options.file, 'r'))
+
+ # dynamically add to the schema (e.g. if needs nbhd)
project = Project()
project.add(col.SchemaNode(col.Sequence(),
col.SchemaNode(Award(nbhd)),
name='awards', missing=[]))
- schema = col.SchemaNode(col.Sequence(), project, name='project')
- projects = schema.deserialize(data)
+ project.add(col.SchemaNode(ProjectShortnameType(nbhd, options.update),
+ name='shortname', missing=None))
+
+ projects = []
+ for datum in data:
+ try:
+ projects.append(project.deserialize(datum))
+ except Exception:
+ keep_going = options.validate_only
+ log.error('Error on %s\n%s', datum['shortname'], datum, exc_info=keep_going)
+ if not keep_going:
+ raise
+
log.debug(projects)
+ if options.validate_only:
+ return
+
chunks = [projects[i::options.nprocs] for i in range(options.nprocs)]
jobs = []
for i in range(options.nprocs):
p = multiprocessing.Process(target=create_projects,
- args=(chunks[i], nbhd, admin, options), name='worker-' + str(i + 1))
+ args=(chunks[i], nbhd, options), name='worker-' + str(i + 1))
jobs.append(p)
p.start()
for j in jobs:
j.join()
- if j.exitcode <> 0:
+ if j.exitcode != 0:
return j.exitcode
return 0
@@ -352,7 +362,10 @@ def parse_options():
'--nprocs', '-n', action='store', dest='nprocs', type=int,
help='Number of processes to divide the work among.',
default=multiprocessing.cpu_count())
+ parser.add_argument('--validate-only', '-v', action='store_true', dest='validate_only',
+ help='Validate ALL records, make no changes')
return parser.parse_args()
+
if __name__ == '__main__':
sys.exit(main(parse_options()))
[2/2] allura git commit: Merge commit
'3e8e548e3954139da1701abf4af738e02593e2f1'
Posted by ke...@apache.org.
Merge commit '3e8e548e3954139da1701abf4af738e02593e2f1'
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/d9a4fc1e
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/d9a4fc1e
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/d9a4fc1e
Branch: refs/heads/master
Commit: d9a4fc1ec64fb52906ab1038dbd3ae590385752b
Parents: 6b6fbf5 3e8e548
Author: Kenton Taylor <kt...@slashdotmedia.com>
Authored: Wed Feb 7 17:36:45 2018 +0000
Committer: Kenton Taylor <kt...@slashdotmedia.com>
Committed: Wed Feb 7 17:36:45 2018 +0000
----------------------------------------------------------------------
Allura/allura/command/create_neighborhood.py | 2 +-
Allura/allura/lib/widgets/forms.py | 3 +-
scripts/project-import.json | 9 ++-
scripts/project-import.py | 87 +++++++++++++----------
4 files changed, 59 insertions(+), 42 deletions(-)
----------------------------------------------------------------------