You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2013/09/23 19:09:24 UTC
[1/3] git commit: [#6695] Added logging for LCD timeout and loop
detection
Updated Branches:
refs/heads/master 9f8788007 -> f236c1439
[#6695] Added logging for LCD timeout and loop detection
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/f236c143
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/f236c143
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/f236c143
Branch: refs/heads/master
Commit: f236c14395dee45086cc7c6375fdb7fd2c70b0c6
Parents: 9d029eb
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Mon Sep 23 15:11:42 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Mon Sep 23 17:09:04 2013 +0000
----------------------------------------------------------------------
Allura/allura/model/repository.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/f236c143/Allura/allura/model/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index b371295..f02f122 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -145,6 +145,7 @@ class RepositoryImplementation(object):
Return a mapping {path: commit_id} of the _id of the last
commit to touch each path, starting from the given commit.
'''
+ orig_commit = commit
timeout = asint(config.get('lcd_timeout', 60)) * 1000
start_time = time()
paths = set(paths)
@@ -152,9 +153,8 @@ class RepositoryImplementation(object):
seen_commits = set()
while paths and commit:
if time() - start_time > timeout:
+ log.error('last_commit_ids timeout for %s on %s', orig_commit._id, ', '.join(paths))
return result
- if commit._id in seen_commits:
- return result # sanity check for bad data (loops)
seen_commits.add(commit._id)
changed = paths & set(commit.changed_paths)
result.update({path: commit._id for path in changed})
@@ -164,6 +164,9 @@ class RepositoryImplementation(object):
# computed wrong (not including children of added trees).
# Can be removed once all projects have had diffs / LCDs refreshed.
parent = commit.get_parent()
+ if parent and parent._id in seen_commits:
+ log.error('last_commit_ids loop detected at %s for %s on %s', parent._id, orig_commit._id, ', '.join(paths))
+ return result # sanity check for bad data (loops)
if parent:
changed = set([path for path in paths if not parent.has_path(path)])
result.update({path: commit._id for path in changed})
[3/3] git commit: [#6695] Added timeout for computing LCDs to prevent
hung requests
Posted by br...@apache.org.
[#6695] Added timeout for computing LCDs to prevent hung requests
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/4ed71de8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/4ed71de8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/4ed71de8
Branch: refs/heads/master
Commit: 4ed71de8c041de612c87cc5868e76b94fe2a800c
Parents: 9f87880
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Fri Sep 20 21:23:44 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Mon Sep 23 17:09:04 2013 +0000
----------------------------------------------------------------------
Allura/allura/model/repo.py | 5 +++--
Allura/allura/model/repository.py | 7 ++++++-
Allura/allura/tests/model/test_repo.py | 15 +++++++++++++++
3 files changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/4ed71de8/Allura/allura/model/repo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repo.py b/Allura/allura/model/repo.py
index 06a0853..502961b 100644
--- a/Allura/allura/model/repo.py
+++ b/Allura/allura/model/repo.py
@@ -843,8 +843,9 @@ class LastCommit(RepoObject):
# (but only ask for the ones that we know we need)
entries = tree.commit.repo.last_commit_ids(tree.commit, unchanged)
if entries is None:
- # something strange went wrong; bail out and possibly try again later
- return None
+ # something strange went wrong; still show the list of files
+ # and possibly try again later
+ entries = {}
# paths are fully-qualified; shorten them back to just node names
entries = {os.path.basename(path):commit_id for path,commit_id in entries.iteritems()}
# update with the nodes changed in this tree's commit
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/4ed71de8/Allura/allura/model/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index 3fe5a80..32b9bf1 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -27,13 +27,14 @@ from subprocess import Popen, PIPE
from difflib import SequenceMatcher
from hashlib import sha1
from datetime import datetime
+from time import time
from collections import defaultdict
from itertools import izip
from urlparse import urljoin
from urllib import quote
import tg
-from paste.deploy.converters import asbool
+from paste.deploy.converters import asbool, asint
from pylons import tmpl_context as c
from pylons import app_globals as g
import pymongo.errors
@@ -144,9 +145,13 @@ class RepositoryImplementation(object):
Return a mapping {path: commit_id} of the _id of the last
commit to touch each path, starting from the given commit.
'''
+ timeout = asint(config.get('lcd_timeout', 60)) * 1000
+ start_time = time()
paths = set(paths)
result = {}
while paths and commit:
+ if time() - start_time > timeout:
+ return result
changed = paths & set(commit.changed_paths)
result.update({path: commit._id for path in changed})
paths = paths - changed
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/4ed71de8/Allura/allura/tests/model/test_repo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/model/test_repo.py b/Allura/allura/tests/model/test_repo.py
index 5d9e95e..b67be9f 100644
--- a/Allura/allura/tests/model/test_repo.py
+++ b/Allura/allura/tests/model/test_repo.py
@@ -17,15 +17,18 @@
from datetime import datetime
from collections import defaultdict, OrderedDict
+
import unittest
import mock
from nose.tools import assert_equal
from pylons import tmpl_context as c
from bson import ObjectId
from ming.orm import session
+from tg import config
from alluratest.controller import setup_basic_test, setup_global_objects
from allura import model as M
+from allura.lib import helpers as h
class TestGitLikeTree(object):
@@ -319,6 +322,18 @@ class TestLastCommit(unittest.TestCase):
result = self.repo.last_commit_ids(commit1, ['file1'])
assert_equal(result, {'file1': commit1._id})
+ def test_timeout(self):
+ commit1 = self._add_commit('Commit 1', ['file1'])
+ commit2 = self._add_commit('Commit 2', ['file1', 'dir1/file1'], ['dir1/file1'], [commit1])
+ commit3 = self._add_commit('Commit 3', ['file1', 'dir1/file1', 'file2'], ['file2'], [commit2])
+ with h.push_config(config, lcd_timeout=0):
+ lcd = M.repo.LastCommit.get(commit3.tree)
+ self.assertEqual(self.repo._commits[lcd.commit_id].message, commit3.message)
+ self.assertEqual(lcd.commit_id, commit3._id)
+ self.assertEqual(lcd.path, '')
+ self.assertEqual(len(lcd.entries), 1)
+ self.assertEqual(lcd.by_name['file2'], commit3._id)
+
class TestModelCache(unittest.TestCase):
def setUp(self):
[2/3] git commit: [#6695] Added sanity check in LCD for loops in
commit data
Posted by br...@apache.org.
[#6695] Added sanity check in LCD for loops in commit data
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/9d029ebe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/9d029ebe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/9d029ebe
Branch: refs/heads/master
Commit: 9d029ebeb70f04d839138d3d4836cae87f96cab8
Parents: 4ed71de
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Fri Sep 20 22:27:25 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Mon Sep 23 17:09:04 2013 +0000
----------------------------------------------------------------------
Allura/allura/model/repository.py | 4 ++++
Allura/allura/tests/model/test_repo.py | 14 ++++++++++++++
2 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9d029ebe/Allura/allura/model/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index 32b9bf1..b371295 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -149,9 +149,13 @@ class RepositoryImplementation(object):
start_time = time()
paths = set(paths)
result = {}
+ seen_commits = set()
while paths and commit:
if time() - start_time > timeout:
return result
+ if commit._id in seen_commits:
+ return result # sanity check for bad data (loops)
+ seen_commits.add(commit._id)
changed = paths & set(commit.changed_paths)
result.update({path: commit._id for path in changed})
paths = paths - changed
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9d029ebe/Allura/allura/tests/model/test_repo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/model/test_repo.py b/Allura/allura/tests/model/test_repo.py
index b67be9f..ef39e4b 100644
--- a/Allura/allura/tests/model/test_repo.py
+++ b/Allura/allura/tests/model/test_repo.py
@@ -334,6 +334,20 @@ class TestLastCommit(unittest.TestCase):
self.assertEqual(len(lcd.entries), 1)
self.assertEqual(lcd.by_name['file2'], commit3._id)
+ def test_loop(self):
+ commit1 = self._add_commit('Commit 1', ['file1'])
+ commit2 = self._add_commit('Commit 2', ['file1', 'dir1/file1'], ['dir1/file1'], [commit1])
+ commit3 = self._add_commit('Commit 3', ['file1', 'dir1/file1', 'file2'], ['file2'], [commit2])
+ commit2.parent_ids = [commit3._id]
+ session(commit2).flush(commit2)
+ lcd = M.repo.LastCommit.get(commit3.tree)
+ self.assertEqual(self.repo._commits[lcd.commit_id].message, commit3.message)
+ self.assertEqual(lcd.commit_id, commit3._id)
+ self.assertEqual(lcd.path, '')
+ self.assertEqual(len(lcd.entries), 2)
+ self.assertEqual(lcd.by_name['dir1'], commit2._id)
+ self.assertEqual(lcd.by_name['file2'], commit3._id)
+
class TestModelCache(unittest.TestCase):
def setUp(self):