You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2019/03/13 16:25:39 UTC
[allura] branch db/8271 created (now 4b3cc44)
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a change to branch db/8271
in repository https://gitbox.apache.org/repos/asf/allura.git.
at 4b3cc44 [#8271] remove unused CommitRunDoc and CommitRunBuilder
This branch includes the following new commits:
new 4b3cc44 [#8271] remove unused CommitRunDoc and CommitRunBuilder
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[allura] 01/01: [#8271] remove unused CommitRunDoc and
CommitRunBuilder
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8271
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 4b3cc44ed23e9442f525e63a22d94ae7aaacde67
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Wed Mar 13 12:03:08 2019 -0400
[#8271] remove unused CommitRunDoc and CommitRunBuilder
---
Allura/allura/model/repo.py | 4 +-
Allura/allura/model/repo_refresh.py | 136 -----------------------
Allura/allura/model/repository.py | 10 --
Allura/allura/scripts/refreshrepo.py | 6 -
Allura/allura/tests/model/test_repo.py | 34 +-----
Allura/allura/tests/unit/test_repo.py | 50 ---------
ForgeSVN/forgesvn/tests/model/test_repository.py | 3 -
7 files changed, 3 insertions(+), 240 deletions(-)
diff --git a/Allura/allura/model/repo.py b/Allura/allura/model/repo.py
index 4cbbea3..6231ba4 100644
--- a/Allura/allura/model/repo.py
+++ b/Allura/allura/model/repo.py
@@ -21,11 +21,11 @@
from .repository import SUser, SObjType
from .repository import QSIZE, README_RE, VIEWABLE_EXTENSIONS, PYPELINE_EXTENSIONS, DIFF_SIMILARITY_THRESHOLD
-from .repository import CommitDoc, TreeDoc, LastCommitDoc, CommitRunDoc
+from .repository import CommitDoc, TreeDoc, LastCommitDoc
from .repository import RepoObject, Commit, Tree, Blob, LastCommit
from .repository import ModelCache
__all__ = [
'SUser', 'SObjType', 'QSIZE', 'README_RE', 'VIEWABLE_EXTENSIONS', 'PYPELINE_EXTENSIONS',
- 'DIFF_SIMILARITY_THRESHOLD', 'CommitDoc', 'TreeDoc', 'LastCommitDoc', 'CommitRunDoc', 'RepoObject',
+ 'DIFF_SIMILARITY_THRESHOLD', 'CommitDoc', 'TreeDoc', 'LastCommitDoc', 'RepoObject',
'Commit', 'Tree', 'Blob', 'LastCommit', 'ModelCache']
diff --git a/Allura/allura/model/repo_refresh.py b/Allura/allura/model/repo_refresh.py
index bc24d89..b6677b6 100644
--- a/Allura/allura/model/repo_refresh.py
+++ b/Allura/allura/model/repo_refresh.py
@@ -33,7 +33,6 @@ from ming.orm import mapper, session, ThreadLocalORMSession
from allura.lib import utils
from allura.lib import helpers as h
from allura.model.repository import CommitDoc
-from allura.model.repository import CommitRunDoc
from allura.model.repository import Commit, Tree, LastCommit, ModelCache
from allura.model.index import ArtifactReferenceDoc, ShortlinkDoc
from allura.model.auth import User
@@ -82,25 +81,6 @@ def refresh_repo(repo, all_commits=False, notify=True, new_clone=False, commits_
log.info('Refresh child info %d for parents of %s',
(i + 1), ci._id)
- if repo._refresh_precompute:
- # Refresh commit runs
- commit_run_ids = commit_ids
- # Check if the CommitRuns for the repo are in a good state by checking for
- # a CommitRunDoc that contains the last known commit. If there isn't one,
- # the CommitRuns for this repo are in a bad state - rebuild them
- # entirely.
- if commit_run_ids != all_commit_ids:
- last_commit = last_known_commit_id(all_commit_ids, new_commit_ids)
- log.info('Last known commit id: %s', last_commit)
- if not CommitRunDoc.m.find(dict(commit_ids=last_commit)).count():
- log.info('CommitRun incomplete, rebuilding with all commits')
- commit_run_ids = all_commit_ids
- log.info('Starting CommitRunBuilder for %s', repo.full_fs_path)
- rb = CommitRunBuilder(commit_run_ids)
- rb.run()
- rb.cleanup()
- log.info('Finished CommitRunBuilder for %s', repo.full_fs_path)
-
# Clear any existing caches for branches/tags
if repo.cached_branches:
repo.cached_branches = []
@@ -195,122 +175,6 @@ def refresh_children(ci):
multi=True)
-class CommitRunBuilder(object):
-
- '''Class used to build up linear runs of single-parent commits'''
-
- def __init__(self, commit_ids):
- self.commit_ids = commit_ids
- self.run_index = {} # by commit ID
- self.runs = {} # by run ID
- self.reasons = {} # reasons to stop merging runs
-
- def run(self):
- '''Build up the runs'''
- for oids in utils.chunked_iter(self.commit_ids, QSIZE):
- oids = list(oids)
- for ci in CommitDoc.m.find(dict(_id={'$in': oids})):
- if ci._id in self.run_index:
- continue
- self.run_index[ci._id] = ci._id
- self.runs[ci._id] = CommitRunDoc(dict(
- _id=ci._id,
- parent_commit_ids=ci.parent_ids,
- commit_ids=[ci._id],
- commit_times=[ci.authored['date']]))
- self.merge_runs()
- log.info('%d runs', len(self.runs))
- for rid, run in sorted(self.runs.items()):
- log.info('%32s: %r', self.reasons.get(rid, 'none'), run._id)
- for run in self.runs.itervalues():
- run.m.save()
- return self.runs
-
- def _all_runs(self):
- '''Find all runs containing this builder's commit IDs'''
- runs = {}
- for oids in utils.chunked_iter(self.commit_ids, QSIZE):
- oids = list(oids)
- for run in CommitRunDoc.m.find(dict(commit_ids={'$in': oids})):
- runs[run._id] = run
- for run in CommitRunDoc.m.find(dict(parent_commit_ids={'$in': oids})):
- runs[run._id] = run
- seen_run_ids = set()
- runs = runs.values()
- while runs:
- run = runs.pop()
- if run._id in seen_run_ids:
- continue
- seen_run_ids.add(run._id)
- yield run
- for run in CommitRunDoc.m.find(
- dict(commit_ids={'$in': run.parent_commit_ids})):
- runs.append(run)
-
- def cleanup(self):
- '''Delete non-maximal runs and merge any new runs with existing runs'''
- runs = dict(
- (run['commit_ids'][0], run)
- for run in self._all_runs())
- for rid, run in runs.items():
- p_cis = run['parent_commit_ids']
- if len(p_cis) != 1:
- continue
- parent_run = runs.get(p_cis[0], None)
- if parent_run is None:
- continue
- run['commit_ids'] += parent_run['commit_ids']
- run['commit_times'] += parent_run['commit_times']
- run['parent_commit_ids'] = parent_run['parent_commit_ids']
- run.m.save()
- parent_run.m.delete()
- del runs[p_cis[0]]
- for run1 in runs.values():
- # if run1 is a subset of another run, delete it
- if CommitRunDoc.m.find(dict(commit_ids={'$all': run1.commit_ids},
- _id={'$ne': run1._id})).count():
- log.info('... delete %r (subset of another run)', run1)
- run1.m.delete()
- continue
- for run2 in CommitRunDoc.m.find(dict(
- commit_ids=run1.commit_ids[0])):
- if run1._id == run2._id:
- continue
- log.info('... delete %r (part of %r)', run2, run1)
- run2.m.delete()
-
- def merge_runs(self):
- '''Find partial runs that may be merged and merge them'''
- while True:
- for run_id, run in self.runs.iteritems():
- if len(run.parent_commit_ids) != 1:
- self.reasons[run_id] = '%d parents' % len(
- run.parent_commit_ids)
- continue
- p_oid = run.parent_commit_ids[0]
- p_run_id = self.run_index.get(p_oid)
- if p_run_id is None:
- self.reasons[run_id] = 'parent commit not found'
- continue
- p_run = self.runs.get(p_run_id)
- if p_run is None:
- self.reasons[run_id] = 'parent run not found'
- continue
- if p_run.commit_ids[0] != p_oid:
- self.reasons[
- run_id] = 'parent does not start with parent commit'
- continue
- run.commit_ids += p_run.commit_ids
- run.commit_times += p_run.commit_times
- run.parent_commit_ids = p_run.parent_commit_ids
- for oid in p_run.commit_ids:
- self.run_index[oid] = run_id
- break
- else:
- break
- del self.runs[p_run_id]
-
-
def unknown_commit_ids(all_commit_ids):
'''filter out all commit ids that have already been cached'''
result = []
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index f68580e..dc97527 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -1018,16 +1018,6 @@ LastCommitDoc = collection(
commit_id=str)]))
-# List of commit runs (a run is a linear series of single-parent commits)
-# CommitRunDoc.commit_ids = [ CommitDoc._id, ... ]
-CommitRunDoc = collection(
- 'repo_commitrun', main_doc_session,
- Field('_id', str),
- Field('parent_commit_ids', [str], index=True),
- Field('commit_ids', [str], index=True),
- Field('commit_times', [datetime]))
-
-
class RepoObject(object):
def __repr__(self): # pragma no cover
diff --git a/Allura/allura/scripts/refreshrepo.py b/Allura/allura/scripts/refreshrepo.py
index 7410b4c..9050400 100644
--- a/Allura/allura/scripts/refreshrepo.py
+++ b/Allura/allura/scripts/refreshrepo.py
@@ -102,12 +102,6 @@ class RefreshRepo(ScriptTask):
M.repository.LastCommitDoc.m.remove(
dict(commit_id={'$in': ci_ids_chunk}))
- i = M.repository.CommitRunDoc.m.find(
- {"commit_ids": {"$in": ci_ids_chunk}}).count()
- if i:
- log.info("Deleting %i CommitRunDoc docs...", i)
- M.repository.CommitRunDoc.m.remove(
- {"commit_ids": {"$in": ci_ids_chunk}})
del ci_ids
try:
diff --git a/Allura/allura/tests/model/test_repo.py b/Allura/allura/tests/model/test_repo.py
index efbbb4d..1020031 100644
--- a/Allura/allura/tests/model/test_repo.py
+++ b/Allura/allura/tests/model/test_repo.py
@@ -62,39 +62,7 @@ class TestGitLikeTree(object):
class RepoImplTestBase(object):
- def test_commit_run(self):
- M.repository.CommitRunDoc.m.remove()
- commit_ids = list(self.repo.all_commit_ids())
- # simulate building up a commit run from multiple pushes
- for c_id in commit_ids:
- crb = M.repo_refresh.CommitRunBuilder([c_id])
- crb.run()
- crb.cleanup()
- runs = M.repository.CommitRunDoc.m.find().all()
- self.assertEqual(len(runs), 1)
- run = runs[0]
- self.assertEqual(run.commit_ids, commit_ids)
- self.assertEqual(len(run.commit_ids), len(run.commit_times))
- self.assertEqual(run.parent_commit_ids, [])
-
- def test_repair_commit_run(self):
- commit_ids = list(self.repo.all_commit_ids())
- # simulate building up a commit run from multiple pushes, but skip the
- # last commit to simulate a broken commit run
- for c_id in commit_ids[:-1]:
- crb = M.repo_refresh.CommitRunBuilder([c_id])
- crb.run()
- crb.cleanup()
- # now repair the commitrun by rebuilding with all commit ids
- crb = M.repo_refresh.CommitRunBuilder(commit_ids)
- crb.run()
- crb.cleanup()
- runs = M.repository.CommitRunDoc.m.find().all()
- self.assertEqual(len(runs), 1)
- run = runs[0]
- self.assertEqual(run.commit_ids, commit_ids)
- self.assertEqual(len(run.commit_ids), len(run.commit_times))
- self.assertEqual(run.parent_commit_ids, [])
+ pass
class RepoTestBase(unittest.TestCase):
diff --git a/Allura/allura/tests/unit/test_repo.py b/Allura/allura/tests/unit/test_repo.py
index eb78df4..33c10d3 100644
--- a/Allura/allura/tests/unit/test_repo.py
+++ b/Allura/allura/tests/unit/test_repo.py
@@ -27,58 +27,8 @@ from allura import model as M
from allura.controllers.repository import topo_sort
from allura.model.repository import zipdir, prefix_paths_union
from allura.model.repo_refresh import (
- CommitRunDoc,
- CommitRunBuilder,
_group_commits,
)
-from alluratest.controller import setup_unit_test
-
-
-class TestCommitRunBuilder(unittest.TestCase):
-
- def setUp(self):
- setup_unit_test()
- commits = [
- M.repository.CommitDoc.make(dict(
- _id=str(i)))
- for i in range(10)]
- for p, com in zip(commits, commits[1:]):
- p.child_ids = [com._id]
- com.parent_ids = [p._id]
- for ci in commits:
- ci.m.save()
- self.commits = commits
-
- def test_single_pass(self):
- crb = CommitRunBuilder(
- [ci._id for ci in self.commits])
- crb.run()
- self.assertEqual(CommitRunDoc.m.count(), 1)
-
- def test_two_pass(self):
- crb = CommitRunBuilder(
- [ci._id for ci in self.commits[:5]])
- crb.run()
- crb = CommitRunBuilder(
- [ci._id for ci in self.commits[5:]])
- crb.run()
- self.assertEqual(CommitRunDoc.m.count(), 2)
- crb.cleanup()
- self.assertEqual(CommitRunDoc.m.count(), 1)
-
- def test_svn_like(self):
- for ci in self.commits:
- crb = CommitRunBuilder([ci._id])
- crb.run()
- crb.cleanup()
- self.assertEqual(CommitRunDoc.m.count(), 1)
-
- def test_reversed(self):
- for ci in reversed(self.commits):
- crb = CommitRunBuilder([ci._id])
- crb.run()
- crb.cleanup()
- self.assertEqual(CommitRunDoc.m.count(), 1)
class TestTopoSort(unittest.TestCase):
diff --git a/ForgeSVN/forgesvn/tests/model/test_repository.py b/ForgeSVN/forgesvn/tests/model/test_repository.py
index a5634ea..f4c49e8 100644
--- a/ForgeSVN/forgesvn/tests/model/test_repository.py
+++ b/ForgeSVN/forgesvn/tests/model/test_repository.py
@@ -681,9 +681,6 @@ class _Test(unittest.TestCase):
def _make_log(self, ci):
session(ci).flush(ci)
- rb = M.repo_refresh.CommitRunBuilder([ci._id])
- rb.run()
- rb.cleanup()
def setUp(self):
setup_basic_test()