You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by jo...@apache.org on 2013/12/16 20:21:52 UTC
[1/2] git commit: [#6821] Refactored common last commit logic up and
removed merge commit skipping
Updated Branches:
refs/heads/cj/6821 ad87c1599 -> 77d18b129
[#6821] Refactored common last commit logic up and removed merge commit skipping
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/33796354
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/33796354
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/33796354
Branch: refs/heads/cj/6821
Commit: 33796354c9dd9514dfe012db789946fee25701a4
Parents: ad87c15
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Mon Dec 16 18:25:14 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Mon Dec 16 18:26:07 2013 +0000
----------------------------------------------------------------------
Allura/allura/model/repository.py | 46 +++++++++++++++++++++++++
ForgeGit/forgegit/model/git_repo.py | 59 ++++++--------------------------
2 files changed, 56 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/33796354/Allura/allura/model/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index fa36e73..169b365 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -32,6 +32,8 @@ from collections import defaultdict
from itertools import izip
from urlparse import urljoin
from urllib import quote
+from threading import Thread
+from Queue import Queue
import tg
from paste.deploy.converters import asbool, asint
@@ -245,6 +247,50 @@ class RepositoryImplementation(object):
def tags(self):
raise NotImplementedError, 'tags'
+ def last_commit_ids(self, commit, paths):
+ """
+ Find the ID of the last commit to touch each path.
+ """
+ if not paths:
+ return {}
+ timeout = float(tg.config.get('lcd_timeout', 60))
+ start_time = time()
+ paths = list(set(paths)) # remove dupes
+ result = {} # will be appended to from each thread
+ chunks = Queue()
+ lcd_chunk_size = asint(tg.config.get('lcd_thread_chunk_size', 10))
+ num_threads = 0
+ for s in range(0, len(paths), lcd_chunk_size):
+ chunks.put(paths[s:s+lcd_chunk_size])
+ num_threads += 1
+ def get_ids():
+ paths = set(chunks.get())
+ try:
+ commit_id = commit._id
+ while paths and commit_id:
+ if time() - start_time > timeout:
+ log.error('last_commit_ids timeout for %s on %s', commit._id, ', '.join(paths))
+ break
+ commit_id, changes = self._get_last_commit(commit._id, paths)
+ if commit_id is None:
+ break
+ changed = prefix_paths_union(paths, changes)
+ for path in changed:
+ result[path] = commit_id
+ paths -= changed
+ except Exception as e:
+ log.exception('Error in SCM thread: %s', e)
+ finally:
+ chunks.task_done()
+ if num_threads == 1:
+ get_ids()
+ else:
+ for i in range(num_threads):
+ t = Thread(target=get_ids)
+ t.start()
+ chunks.join()
+ return result
+
class Repository(Artifact, ActivityObject):
BATCH_SIZE=100
class __mongometa__:
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/33796354/ForgeGit/forgegit/model/git_repo.py
----------------------------------------------------------------------
diff --git a/ForgeGit/forgegit/model/git_repo.py b/ForgeGit/forgegit/model/git_repo.py
index f08fdd4..229f2bb 100644
--- a/ForgeGit/forgegit/model/git_repo.py
+++ b/ForgeGit/forgegit/model/git_repo.py
@@ -27,8 +27,6 @@ from datetime import datetime
from glob import glob
import gzip
from time import time
-from threading import Thread
-from Queue import Queue
import tg
import git
@@ -500,54 +498,17 @@ class GitImplementation(M.RepositoryImplementation):
self._repo.default_branch_name = name
session(self._repo).flush(self._repo)
- def last_commit_ids(self, commit, paths):
- """
- Find the ID of the last commit to touch each path.
- """
- if not paths:
- return {}
- timeout = float(tg.config.get('lcd_timeout', 60))
- start_time = time()
- paths = list(set(paths)) # remove dupes
- result = {} # will be appended to from each thread
- chunks = Queue()
- lcd_chunk_size = asint(tg.config.get('lcd_thread_chunk_size', 10))
- num_threads = 0
- for s in range(0, len(paths), lcd_chunk_size):
- chunks.put(paths[s:s+lcd_chunk_size])
- num_threads += 1
- def get_ids():
- paths = set(chunks.get())
- try:
- commit_id = commit._id
- while paths and commit_id:
- if time() - start_time > timeout:
- log.error('last_commit_ids timeout for %s on %s', commit._id, ', '.join(paths))
- break
- lines = self._git.git.log(
- commit._id, '--', *paths,
- pretty='format:%H',
- name_only=True,
- max_count=1,
- no_merges=True).split('\n')
- commit_id = lines[0]
- changes = set(lines[1:])
- changed = prefix_paths_union(paths, changes)
- for path in changed:
- result[path] = commit_id
- paths -= changed
- except Exception as e:
- log.exception('Error in Git thread: %s', e)
- finally:
- chunks.task_done()
- if num_threads == 1:
- get_ids()
+ def _get_last_commit(self, commit_id, paths):
+ output = self._git.git.log(
+ commit_id, '--', *paths,
+ pretty='format:%H',
+ name_only=True,
+ max_count=1)
+ if not output:
+ return None, set()
else:
- for i in range(num_threads):
- t = Thread(target=get_ids)
- t.start()
- chunks.join()
- return result
+ lines = output.split('\n')
+ return lines[0], set(lines[1:])
class _OpenedGitBlob(object):
CHUNK_SIZE=4096
[2/2] git commit: [#6821] Remove any possibility of indefinite
blocking and make test more clear
Posted by jo...@apache.org.
[#6821] Remove any possibility of indefinite blocking and make test more clear
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/77d18b12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/77d18b12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/77d18b12
Branch: refs/heads/cj/6821
Commit: 77d18b1293304d6a3c7474e9a8a1befba2b08514
Parents: 3379635
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Mon Dec 16 19:20:34 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Mon Dec 16 19:20:34 2013 +0000
----------------------------------------------------------------------
Allura/allura/model/repository.py | 11 ++++++++++-
ForgeGit/forgegit/tests/model/test_repository.py | 7 ++++---
2 files changed, 14 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/77d18b12/Allura/allura/model/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index 169b365..2aeef64 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -288,7 +288,16 @@ class RepositoryImplementation(object):
for i in range(num_threads):
t = Thread(target=get_ids)
t.start()
- chunks.join()
+ # reimplement chunks.join() but with a timeout
+ # see: http://bugs.python.org/issue9634
+ # (giving threads a bit of extra cleanup time in case they timeout)
+ chunks.all_tasks_done.acquire()
+ try:
+ endtime = time() + timeout + 0.5
+ while chunks.unfinished_tasks and endtime > time():
+ chunks.all_tasks_done.wait(endtime - time())
+ finally:
+ chunks.all_tasks_done.release()
return result
class Repository(Artifact, ActivityObject):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/77d18b12/ForgeGit/forgegit/tests/model/test_repository.py
----------------------------------------------------------------------
diff --git a/ForgeGit/forgegit/tests/model/test_repository.py b/ForgeGit/forgegit/tests/model/test_repository.py
index 8569025..4f4f56f 100644
--- a/ForgeGit/forgegit/tests/model/test_repository.py
+++ b/ForgeGit/forgegit/tests/model/test_repository.py
@@ -456,13 +456,14 @@ class TestGitImplementation(unittest.TestCase):
with h.push_config(tg.config, lcd_thread_chunk_size=1):
self.test_last_commit_ids()
- def test_last_commit_ids_threaded_error(self):
- with h.push_config(tg.config, lcd_thread_chunk_size=1):
+ @mock.patch('forgegit.model.git_repo.GitImplementation._git', new_callable=mock.PropertyMock)
+ def test_last_commit_ids_threaded_error(self, _git):
+ with h.push_config(tg.config, lcd_thread_chunk_size=1, lcd_timeout=2):
repo_dir = pkg_resources.resource_filename(
'forgegit', 'tests/data/testrename.git')
repo = mock.Mock(full_fs_path=repo_dir)
+ _git.side_effect = ValueError
impl = GM.git_repo.GitImplementation(repo)
- impl._git = None
lcds = impl.last_commit_ids(mock.Mock(_id='13951944969cf45a701bf90f83647b309815e6d5'), ['f2.txt', 'f3.txt'])
self.assertEqual(lcds, {})