You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by tv...@apache.org on 2014/01/09 21:48:15 UTC
[3/6] git commit: [#6905] ticket:503 Fix tarball generation
[#6905] ticket:503 Fix tarball generation
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/462bcfb5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/462bcfb5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/462bcfb5
Branch: refs/heads/master
Commit: 462bcfb5dbc49cac097f33690446960e1b5c6494
Parents: da8258d
Author: Mykola Kharechko <cr...@gmail.com>
Authored: Fri Jan 3 14:26:35 2014 +0200
Committer: Tim Van Steenburgh <tv...@gmail.com>
Committed: Thu Jan 9 18:41:35 2014 +0000
----------------------------------------------------------------------
Allura/allura/controllers/repository.py | 5 +--
Allura/allura/lib/decorators.py | 3 +-
Allura/allura/model/monq_model.py | 6 ++++
Allura/allura/model/repository.py | 27 +++++++++++---
Allura/allura/tasks/repo_tasks.py | 5 +--
Allura/allura/templates/repo/tarball.html | 10 +++---
Allura/development.ini | 7 ++++
.../tests/functional/test_controllers.py | 4 +--
.../forgegit/tests/model/test_repository.py | 37 +++++++++++++++++---
9 files changed, 83 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/allura/controllers/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/controllers/repository.py b/Allura/allura/controllers/repository.py
index d0e78e2..0dc1274 100644
--- a/Allura/allura/controllers/repository.py
+++ b/Allura/allura/controllers/repository.py
@@ -466,10 +466,11 @@ class CommitBrowser(BaseController):
raise exc.HTTPNotFound()
rev = self._commit.url().split('/')[-2]
status = c.app.repo.get_tarball_status(rev, path)
- if status is None and request.method == 'POST':
+ if status in (None, 'error') and request.method == 'POST':
allura.tasks.repo_tasks.tarball.post(revision=rev, path=path)
redirect('tarball' + '?path={0}'.format(path) if path else '')
- return dict(commit=self._commit, revision=rev, status=status or 'na')
+ status = 'na' if status in (None, 'error') else status
+ return dict(commit=self._commit, revision=rev, status=status)
@expose('json:')
def tarball_status(self, path=None, **kw):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/allura/lib/decorators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/decorators.py b/Allura/allura/lib/decorators.py
index 8604a92..5810fb0 100644
--- a/Allura/allura/lib/decorators.py
+++ b/Allura/allura/lib/decorators.py
@@ -59,7 +59,8 @@ def task(*args, **kw):
kw.get('notifications_disabled') else h.null_contextmanager)
with cm(project):
from allura import model as M
- return M.MonQTask.post(func, args, kwargs, delay=delay)
+ task_obj = M.MonQTask.post(func, args, kwargs, delay=delay)
+ return task_obj
# if decorating a class, have to make it a staticmethod
# or it gets a spurious cls argument
func.post = staticmethod(post) if inspect.isclass(func) else post
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/allura/model/monq_model.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/monq_model.py b/Allura/allura/model/monq_model.py
index faac14b..f38c073 100644
--- a/Allura/allura/model/monq_model.py
+++ b/Allura/allura/model/monq_model.py
@@ -76,6 +76,11 @@ class MonQTask(MappedClass):
# have an index on task_name
'state', 'task_name', 'time_queue'
],
+ [
+ # used while user quering snapsot
+ 'kwargs.revision', 'kwargs.path'
+ ],
+ 'args',
]
_id = FieldProperty(S.ObjectId)
@@ -246,6 +251,7 @@ class MonQTask(MappedClass):
old_cuser = getattr(c, 'user', None)
try:
func = self.function
+ func.task_id = str(self._id)
c.project = M.Project.query.get(_id=self.context.project_id)
c.app = None
if c.project:
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/allura/model/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index 06ade3f..93e012e 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -39,6 +39,7 @@ import tg
from paste.deploy.converters import asbool, asint
from pylons import tmpl_context as c
from pylons import app_globals as g
+import pymongo
import pymongo.errors
from ming import schema as S
@@ -56,6 +57,7 @@ from .notification import Notification
from .repo_refresh import refresh_repo, unknown_commit_ids as unknown_commit_ids_repo
from .repo import CommitRunDoc, QSIZE
from .timeline import ActivityObject
+from .monq_model import MonQTask
log = logging.getLogger(__name__)
config = utils.ConfigProxy(
@@ -355,15 +357,32 @@ class Repository(Artifact, ActivityObject):
filename)
return urljoin(tg.config.get('scm.repos.tarball.url_prefix', '/'), r)
- def get_tarball_status(self, revision, path=None):
+ def get_tarball_status(self, revision, path=None, task_id=None):
pathname = os.path.join(self.tarball_path, self.tarball_filename(revision, path))
filename = '%s%s' % (pathname, '.zip')
tmpfilename = '%s%s' % (pathname, '.tmp')
+ # check for state of task in MonQTask
+ path = '' if path is None else path
+ task_query = MonQTask.query.find({
+ 'task_name': 'allura.tasks.repo_tasks.tarball',
+ '$or': [
+ {'kwargs.path': path, 'kwargs.revision': revision},
+ {'args':[revision, path]}]})
+ task = task_query.sort(
+ [('time_queue', pymongo.DESCENDING),]).limit(1).first()
+
if os.path.isfile(filename):
- return 'ready'
- elif os.path.isfile(tmpfilename):
- return 'busy'
+ return 'complete'
+
+ if not task or \
+ (task.state == 'complete' and not os.path.isfile(filename)):
+ return None
+ if task.state == 'busy' and str(task._id) == task_id:
+ return 'self'
+
+ return task.state
+
def __repr__(self): # pragma no cover
return '<%s %s>' % (
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/allura/tasks/repo_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/repo_tasks.py b/Allura/allura/tasks/repo_tasks.py
index a987228..c4b9296 100644
--- a/Allura/allura/tasks/repo_tasks.py
+++ b/Allura/allura/tasks/repo_tasks.py
@@ -124,8 +124,9 @@ def tarball(revision=None, path=None):
log = logging.getLogger(__name__)
if revision:
repo = c.app.repo
- status = repo.get_tarball_status(revision, path)
- if status:
+ status = repo.get_tarball_status(revision, path,
+ task_id=tarball.task_id)
+ if status in ('busy', 'complete'):
log.info('Skipping snapshot for repository: %s:%s rev %s because it is already %s' %
(c.project.shortname, c.app.config.options.mount_point, revision, status))
else:
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/allura/templates/repo/tarball.html
----------------------------------------------------------------------
diff --git a/Allura/allura/templates/repo/tarball.html b/Allura/allura/templates/repo/tarball.html
index d68430d..21bc7d7 100644
--- a/Allura/allura/templates/repo/tarball.html
+++ b/Allura/allura/templates/repo/tarball.html
@@ -56,7 +56,7 @@ Commit <a href="{{commit.url()}}">{{commit.shorthand_id()}}</a> {{commit_labels(
var delay = 500;
function check_status() {
$.get('{{commit.url()}}tarball_status?path={{path}}', function(data) {
- if (data.status !== 'na') {
+ if (data.status === 'complete') {
spinner.stop();
$('#snapshot_status h2').hide();
$('#snapshot_status h2.' + data.status).show();
@@ -64,13 +64,13 @@ Commit <a href="{{commit.url()}}">{{commit.shorthand_id()}}</a> {{commit_labels(
window.location.href = '{{c.app.repo.tarball_url(revision, path)}}';
{% endif %}
} else {
+ if (data.status === 'na' || data.status === 'error') {
+ // something went wrong
+ $('#snapshot_status form').show();
+ }
if (delay < 60000){
delay = delay * 2;
}
- if (delay >= 16000) {
- // we've been waiting at least 15 seconds
- $('#snapshot_status form').show();
- }
window.setTimeout(check_status, delay);
}
});
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/development.ini
----------------------------------------------------------------------
diff --git a/Allura/development.ini b/Allura/development.ini
index edfb794..e5ed4f4 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -33,6 +33,13 @@ error_email_from = paste@localhost
# Used to uniquify references to static resources
build_key=1276635823
+# root directory for tarballs
+scm.repos.tarball.root = /tmp/tarball
+# enable/disable feature
+scm.repos.tarball.enable = false
+# url prefix for tarballs' download urls
+scm.repos.tarball.url_prefix = http://sf-fortytwo-7049.sb.sf.net/p/snapshots/
+
[server:main]
use = egg:Paste#http
host = 0.0.0.0
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/ForgeGit/forgegit/tests/functional/test_controllers.py
----------------------------------------------------------------------
diff --git a/ForgeGit/forgegit/tests/functional/test_controllers.py b/ForgeGit/forgegit/tests/functional/test_controllers.py
index 9a78579..495187a 100644
--- a/ForgeGit/forgegit/tests/functional/test_controllers.py
+++ b/ForgeGit/forgegit/tests/functional/test_controllers.py
@@ -352,9 +352,9 @@ class TestRootController(_TestCase):
M.MonQTask.run_ready()
ThreadLocalORMSession.flush_all()
r = self.app.get(ci + 'tarball_status')
- assert '{"status": "ready"}' in r
+ assert '{"status": "complete"}' in r
r = self.app.get('/p/test/src-git/ci/master/tarball_status')
- assert '{"status": "ready"}' in r
+ assert '{"status": "complete"}' in r
r = self.app.get('/p/test/src-git/ci/master/tarball')
assert 'Your download will begin shortly' in r
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/ForgeGit/forgegit/tests/model/test_repository.py
----------------------------------------------------------------------
diff --git a/ForgeGit/forgegit/tests/model/test_repository.py b/ForgeGit/forgegit/tests/model/test_repository.py
index 4f4f56f..3680846 100644
--- a/ForgeGit/forgegit/tests/model/test_repository.py
+++ b/ForgeGit/forgegit/tests/model/test_repository.py
@@ -24,6 +24,8 @@ import unittest
import pkg_resources
import datetime
+import pymongo
+
import mock
from pylons import tmpl_context as c, app_globals as g
import tg
@@ -34,6 +36,7 @@ from testfixtures import TempDirectory
from alluratest.controller import setup_basic_test, setup_global_objects
from allura.lib import helpers as h
+from allura.tasks.repo_tasks import tarball
from allura.tests import decorators as td
from allura.tests.model.test_repo import RepoImplTestBase
from allura import model as M
@@ -394,13 +397,37 @@ class TestGitRepo(unittest.TestCase, RepoImplTestBase):
if os.path.isdir(os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD/")):
os.removedirs(os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD/"))
self.repo.tarball('HEAD')
+ assert_equal(self.repo.get_tarball_status('HEAD'), 'complete')
+
+ os.remove(os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD.zip"))
+ assert_equal(self.repo.get_tarball_status('HEAD'), None)
+
+ def test_tarball_status_task(self):
+ assert_equal(self.repo.get_tarball_status('HEAD'), None)
+
+ # create tarball task in MonQTask and check get_tarball_status
+ tarball.post('HEAD', '')
+
+ # task created
assert_equal(self.repo.get_tarball_status('HEAD'), 'ready')
- os.rename(os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD.zip"),
- os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD.tmp"))
+
+ task = M.MonQTask.query.find({
+ 'task_name': 'allura.tasks.repo_tasks.tarball',
+ '$or': [
+ {'kwargs':
+ {'path': '',
+ 'revision': 'HEAD'}},
+ {'args':['HEAD', '']}]
+ }).sort([('time_queue', pymongo.DESCENDING),]).limit(1).first()
+
+ # task is running
+ task.state = 'busy'
+ task.query.session.flush_all()
assert_equal(self.repo.get_tarball_status('HEAD'), 'busy')
- os.remove(os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD.tmp"))
- assert_equal(self.repo.get_tarball_status('HEAD'), None)
- os.makedirs(os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD"))
+
+ # when state is complete, but file don't exists, then status is None
+ task.state = 'complete'
+ task.query.session.flush_all()
assert_equal(self.repo.get_tarball_status('HEAD'), None)
def test_is_empty(self):