You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by ke...@apache.org on 2018/04/12 20:10:19 UTC
[1/2] allura git commit: refreshrepo.py option to control creating
activity, firing webhooks, etc
Repository: allura
Updated Branches:
refs/heads/master 487c804f1 -> 6eb39f5e2
refreshrepo.py option to control creating activity, firing webhooks, etc
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/6eb39f5e
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/6eb39f5e
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/6eb39f5e
Branch: refs/heads/master
Commit: 6eb39f5e280192f37a87948973717fdb5b1bbd66
Parents: dfd206c
Author: Dave Brondsema <da...@brondsema.net>
Authored: Thu Apr 12 14:30:54 2018 -0400
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Thu Apr 12 15:21:32 2018 -0400
----------------------------------------------------------------------
Allura/allura/model/repo_refresh.py | 7 +++++--
Allura/allura/model/repository.py | 4 ++--
Allura/allura/scripts/refreshrepo.py | 10 ++++++++--
3 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/6eb39f5e/Allura/allura/model/repo_refresh.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repo_refresh.py b/Allura/allura/model/repo_refresh.py
index 5d8415f..bc24d89 100644
--- a/Allura/allura/model/repo_refresh.py
+++ b/Allura/allura/model/repo_refresh.py
@@ -44,7 +44,10 @@ log = logging.getLogger(__name__)
QSIZE = 100
-def refresh_repo(repo, all_commits=False, notify=True, new_clone=False):
+def refresh_repo(repo, all_commits=False, notify=True, new_clone=False, commits_are_new=None):
+ if commits_are_new is None:
+ commits_are_new = not all_commits and not new_clone
+
all_commit_ids = commit_ids = list(repo.all_commit_ids())
if not commit_ids:
# the repo is empty, no need to continue
@@ -111,7 +114,7 @@ def refresh_repo(repo, all_commits=False, notify=True, new_clone=False):
repo.get_branches()
repo.get_tags()
- if not all_commits and not new_clone:
+ if commits_are_new:
for commit in commit_ids:
new = repo.commit(commit)
user = User.by_email_address(new.committed.email)
http://git-wip-us.apache.org/repos/asf/allura/blob/6eb39f5e/Allura/allura/model/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index a859c6d..b2c6c7e 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -699,13 +699,13 @@ class Repository(Artifact, ActivityObject):
from allura.model.repo_refresh import unknown_commit_ids as unknown_commit_ids_repo
return unknown_commit_ids_repo(self.all_commit_ids())
- def refresh(self, all_commits=False, notify=True, new_clone=False):
+ def refresh(self, all_commits=False, notify=True, new_clone=False, commits_are_new=None):
'''Find any new commits in the repository and update'''
try:
from allura.model.repo_refresh import refresh_repo
log.info('... %r analyzing', self)
self.set_status('analyzing')
- refresh_repo(self, all_commits, notify, new_clone)
+ refresh_repo(self, all_commits, notify, new_clone, commits_are_new)
finally:
log.info('... %s ready', self)
self.set_status('ready')
http://git-wip-us.apache.org/repos/asf/allura/blob/6eb39f5e/Allura/allura/scripts/refreshrepo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/refreshrepo.py b/Allura/allura/scripts/refreshrepo.py
index dc7a175..7410b4c 100644
--- a/Allura/allura/scripts/refreshrepo.py
+++ b/Allura/allura/scripts/refreshrepo.py
@@ -20,6 +20,7 @@ import logging
import faulthandler
from datetime import datetime
+from paste.util.converters import asbool
from pylons import tmpl_context as c
from ming.orm import ThreadLocalORMSession
@@ -119,11 +120,12 @@ class RefreshRepo(ScriptTask):
if options.profile:
import cProfile
cProfile.runctx(
- 'c.app.repo.refresh(options.all, notify=options.notify)',
+ 'c.app.repo.refresh(options.all, notify=options.notify, '
+ ' commits_are_new=options.commits_are_new)',
globals(), locals(), 'refresh.profile')
else:
c.app.repo.refresh(
- options.all, notify=options.notify)
+ options.all, notify=options.notify, commits_are_new=options.commits_are_new)
except:
log.exception('Error refreshing %r', c.app.repo)
ThreadLocalORMSession.flush_all()
@@ -175,6 +177,10 @@ class RefreshRepo(ScriptTask):
help='Refresh all commits (not just the ones that are new).')
parser.add_argument('--notify', action='store_true', dest='notify',
default=False, help='Send email notifications of new commits.')
+ parser.add_argument('--commits-are-new', dest='commits_are_new',
+ type=asbool, metavar='true/false', default=None,
+ help='Specify true/false to override smart default. Controls creating activity entries, '
+ 'stats, sending webhook etc.')
parser.add_argument('--dry-run', action='store_true', dest='dry_run',
default=False, help='Log names of projects that would have their '
'repos refreshed, but do not perform the actual refresh.')
[2/2] allura git commit: Option in refreshrepo.py to clean commits
after certain date
Posted by ke...@apache.org.
Option in refreshrepo.py to clean commits after certain date
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/dfd206c2
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/dfd206c2
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/dfd206c2
Branch: refs/heads/master
Commit: dfd206c2e88ac526813d2e09e6a787fe10ebbf5a
Parents: 487c804
Author: Dave Brondsema <da...@brondsema.net>
Authored: Thu Apr 12 14:02:45 2018 -0400
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Thu Apr 12 15:21:32 2018 -0400
----------------------------------------------------------------------
Allura/allura/scripts/refreshrepo.py | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/dfd206c2/Allura/allura/scripts/refreshrepo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/refreshrepo.py b/Allura/allura/scripts/refreshrepo.py
index 81fdfc5..dc7a175 100644
--- a/Allura/allura/scripts/refreshrepo.py
+++ b/Allura/allura/scripts/refreshrepo.py
@@ -17,8 +17,9 @@
import argparse
import logging
-
import faulthandler
+from datetime import datetime
+
from pylons import tmpl_context as c
from ming.orm import ThreadLocalORMSession
@@ -65,11 +66,17 @@ class RefreshRepo(ScriptTask):
c.app.repo.tool.lower())
continue
+ ci_ids = []
if options.clean:
ci_ids = list(c.app.repo.all_commit_ids())
+ elif options.clean_after:
+ for ci in M.repository.CommitDoc.m.find({'repo_ids': c.app.repo._id,
+ 'committed.date': {'$gt': options.clean_after}}):
+ ci_ids.append(ci._id)
+
+ if ci_ids:
log.info("Deleting mongo data for %i commits...",
len(ci_ids))
-
# delete these in chunks, otherwise the query doc can
# exceed the max BSON size limit (16MB at the moment)
for ci_ids_chunk in chunked_list(ci_ids, 3000):
@@ -133,6 +140,8 @@ class RefreshRepo(ScriptTask):
repo_types.append(repo_type)
return repo_types
+ date_format = '%Y-%m-%dT%H:%M:%S'
+
parser = argparse.ArgumentParser(description='Scan repos on filesytem and '
'update repo metadata in MongoDB. Run for all repos (no args), '
'or restrict by neighborhood, project, or code tool mount point.')
@@ -156,6 +165,11 @@ class RefreshRepo(ScriptTask):
parser.add_argument('--clean', action='store_true', dest='clean',
default=False, help='Remove repo-related mongo docs (for '
'project(s) being refreshed only) before doing the refresh.')
+ parser.add_argument('--clean-after', metavar='DATETIME', dest='clean_after',
+ type=lambda d: datetime.strptime(d, date_format),
+ help='Like --clean but only docs for commits after date ({} format)'.format(
+ date_format.replace('%', '%%')
+ ))
parser.add_argument(
'--all', action='store_true', dest='all', default=False,
help='Refresh all commits (not just the ones that are new).')