You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by tv...@apache.org on 2014/01/09 21:36:58 UTC
[11/18] git commit: [#4397] Background timeline aggregation
[#4397] Background timeline aggregation
Fires off background tasks to aggregate affected timelines after a new
activity is created. Multiple aggregations for the same node are
prevented from occuring concurrently. On-demand aggregation will still
happen if necessary (if a timeline is requested and it's aggregation
is stale) but should be rare.
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/dca06665
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/dca06665
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/dca06665
Branch: refs/heads/tv/6905
Commit: dca066651ec43fb6541075bacda767151fb7562c
Parents: 131f06f
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Thu Dec 19 02:45:59 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Jan 8 18:35:26 2014 +0000
----------------------------------------------------------------------
Allura/allura/lib/app_globals.py | 4 +++
Allura/allura/lib/custom_middleware.py | 6 ++++
Allura/allura/model/timeline.py | 36 ++++++++++++++++++--
Allura/allura/tasks/activity_tasks.py | 25 ++++++++++++++
Allura/setup.py | 6 ++--
.../forgeactivity/tests/functional/test_root.py | 22 ++++++++++++
6 files changed, 94 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dca06665/Allura/allura/lib/app_globals.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 0643460..6c94870 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -280,6 +280,10 @@ class Globals(object):
return False
def create_activity(self, *a, **kw):
pass
+ def create_timeline(self, *a, **kw):
+ pass
+ def create_timelines(self, *a, **kw):
+ pass
def get_timeline(self, *a, **kw):
return []
return NullActivityStreamDirector()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dca06665/Allura/allura/lib/custom_middleware.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/custom_middleware.py b/Allura/allura/lib/custom_middleware.py
index d7cbdc9..940c534 100644
--- a/Allura/allura/lib/custom_middleware.py
+++ b/Allura/allura/lib/custom_middleware.py
@@ -188,8 +188,14 @@ class AlluraTimerMiddleware(TimerMiddleware):
import pymongo
import socket
import urllib2
+ import activitystream
return self.entry_point_timers() + [
+ Timer('activitystream.director.{method_name}', allura.model.timeline.Director,
+ 'create_activity', 'create_timeline', 'get_timeline'),
+ Timer('activitystream.aggregator.{method_name}', allura.model.timeline.Aggregator, '*'),
+ Timer('activitystream.node_manager.{method_name}', activitystream.managers.NodeManager, '*'),
+ Timer('activitystream.activity_manager.{method_name}', activitystream.managers.ActivityManager, '*'),
Timer('jinja', jinja2.Template, 'render', 'stream', 'generate'),
Timer('markdown', markdown.Markdown, 'convert'),
Timer('ming', ming.odm.odmsession.ODMCursor, 'next', # FIXME: this may captures timings ok, but is misleading for counts
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dca06665/Allura/allura/model/timeline.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/timeline.py b/Allura/allura/model/timeline.py
index b45118e..8f8ac41 100644
--- a/Allura/allura/model/timeline.py
+++ b/Allura/allura/model/timeline.py
@@ -16,18 +16,48 @@
# under the License.
import bson
+import logging
from ming.odm import Mapper
-from activitystream import base
+
+from activitystream import ActivityDirector
+from activitystream.base import NodeBase, ActivityObjectBase
+from activitystream.managers import Aggregator as BaseAggregator
+
from allura.lib import security
+from allura.tasks.activity_tasks import create_timelines
+
+log = logging.getLogger(__name__)
+
+
+class Director(ActivityDirector):
+ """Overrides the default ActivityDirector to kick off background
+ timeline aggregations after an activity is created.
+
+ """
+ def create_activity(self, actor, verb, obj, target=None,
+ related_nodes=None):
+ from allura.model.project import Project
+ super(Director, self).create_activity(actor, verb, obj,
+ target=target, related_nodes=related_nodes)
+ # aggregate actor and follower's timelines
+ create_timelines.post(actor.node_id)
+ # aggregate project and follower's timelines
+ for node in [obj, target] + (related_nodes or []):
+ if isinstance(node, Project):
+ create_timelines.post(node.node_id)
+
+
+class Aggregator(BaseAggregator):
+ pass
-class ActivityNode(base.NodeBase):
+class ActivityNode(NodeBase):
@property
def node_id(self):
return "%s:%s" % (self.__class__.__name__, self._id)
-class ActivityObject(base.ActivityObjectBase):
+class ActivityObject(ActivityObjectBase):
@property
def activity_name(self):
"""Override this for each Artifact type."""
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dca06665/Allura/allura/tasks/activity_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/activity_tasks.py b/Allura/allura/tasks/activity_tasks.py
new file mode 100644
index 0000000..26ac02a
--- /dev/null
+++ b/Allura/allura/tasks/activity_tasks.py
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pylons import app_globals as g
+
+from allura.lib.decorators import task
+
+
+@task
+def create_timelines(node_id):
+ g.director.create_timelines(node_id)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dca06665/Allura/setup.py
----------------------------------------------------------------------
diff --git a/Allura/setup.py b/Allura/setup.py
index bc829ce..4f3cf84 100644
--- a/Allura/setup.py
+++ b/Allura/setup.py
@@ -146,7 +146,9 @@ setup(
[easy_widgets.engines]
jinja = allura.config.app_cfg:JinjaEngine
- [activitystream.storage]
- driver = activitystream.storage.mingstorage:MingStorage
+ [activitystream]
+ storage = activitystream.storage.mingstorage:MingStorage
+ director = allura.model.timeline:Director
+ aggregator = allura.model.timeline:Aggregator
""",
)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dca06665/ForgeActivity/forgeactivity/tests/functional/test_root.py
----------------------------------------------------------------------
diff --git a/ForgeActivity/forgeactivity/tests/functional/test_root.py b/ForgeActivity/forgeactivity/tests/functional/test_root.py
index 178bb89..eda71be 100644
--- a/ForgeActivity/forgeactivity/tests/functional/test_root.py
+++ b/ForgeActivity/forgeactivity/tests/functional/test_root.py
@@ -21,7 +21,9 @@ from tg import config
import dateutil.parser
from nose.tools import assert_equal
+from pylons import app_globals as g
+from allura import model as M
from alluratest.controller import TestController
from allura.tests import decorators as td
@@ -148,3 +150,23 @@ class TestActivityController(TestController):
assert director.get_timeline.call_count == 1
assert director.get_timeline.call_args[0][0].shortname == 'test'
assert director.get_timeline.call_args[1]['actor_only'] == False
+
+ @td.with_tracker
+ @td.with_tool('u/test-user-1', 'activity')
+ @td.with_user_project('test-user-1')
+ def test_background_aggregation(self):
+ self.app.get('/u/test-admin/activity/follow?follow=True',
+ extra_environ=dict(username='test-user-1'))
+ # new ticket, creates activity
+ d = {'ticket_form.summary': 'New Ticket'}
+ self.app.post('/bugs/save_ticket', params=d)
+ orig_create_timeline = g.director.aggregator.create_timeline
+ with patch.object(g.director.aggregator, 'create_timeline') as create_timeline:
+ create_timeline.side_effect = orig_create_timeline
+ M.MonQTask.run_ready()
+ # 3 aggregations: 1 actor, 1 follower, 1 project
+ assert_equal(create_timeline.call_count, 3)
+ create_timeline.reset_mock()
+ self.app.get('/u/test-admin/activity/')
+ self.app.get('/u/test-user-1/activity/')
+ assert_equal(create_timeline.call_count, 0)