You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2016/12/22 17:32:07 UTC
[1/2] allura git commit: [#8021] run spam checker on post/ticket/blog
edits, and for admins/devs posts
Repository: allura
Updated Branches:
refs/heads/master da54abd1a -> c780b9717
[#8021] run spam checker on post/ticket/blog edits, and for admins/devs posts
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/c780b971
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/c780b971
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/c780b971
Branch: refs/heads/master
Commit: c780b9717c0ba976b04bf832e5dbbc749da6bd4c
Parents: ca8b596
Author: Dave Brondsema <da...@brondsema.net>
Authored: Mon Dec 19 14:17:50 2016 -0500
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Thu Dec 22 12:31:54 2016 -0500
----------------------------------------------------------------------
Allura/allura/controllers/discuss.py | 1 +
Allura/allura/model/discuss.py | 4 +++-
ForgeBlog/forgeblog/main.py | 3 +++
ForgeTracker/forgetracker/tracker_main.py | 6 ++++--
4 files changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/c780b971/Allura/allura/controllers/discuss.py
----------------------------------------------------------------------
diff --git a/Allura/allura/controllers/discuss.py b/Allura/allura/controllers/discuss.py
index d6ba1ff..509de41 100644
--- a/Allura/allura/controllers/discuss.py
+++ b/Allura/allura/controllers/discuss.py
@@ -298,6 +298,7 @@ class PostController(BaseController):
self.post.edit_count = self.post.edit_count + 1
self.post.last_edit_date = datetime.utcnow()
self.post.last_edit_by_id = c.user._id
+ self.thread.is_spam(self.post) # run spam checker, nothing to do with result yet
self.post.commit()
g.director.create_activity(c.user, 'modified', self.post,
target=self.post.thread.artifact or self.post.thread,
http://git-wip-us.apache.org/repos/asf/allura/blob/c780b971/Allura/allura/model/discuss.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/discuss.py b/Allura/allura/model/discuss.py
index 874a888..f4ddd7d 100644
--- a/Allura/allura/model/discuss.py
+++ b/Allura/allura/model/discuss.py
@@ -286,10 +286,12 @@ class Thread(Artifact, ActivityObject):
def is_spam(self, post):
roles = [r.name for r in c.project.named_roles]
+ spammy = g.spam_checker.check(post.text, artifact=post, user=c.user)
if c.user in c.project.users_with_role(*roles):
+ # always run the check, so it's logged. But don't act on it for admins/developers of their own project
return False
else:
- return g.spam_checker.check(post.text, artifact=post, user=c.user)
+ return spammy
def post(self, text, message_id=None, parent_id=None, notify=True,
notification_text=None, timestamp=None, ignore_security=False,
http://git-wip-us.apache.org/repos/asf/allura/blob/c780b971/ForgeBlog/forgeblog/main.py
----------------------------------------------------------------------
diff --git a/ForgeBlog/forgeblog/main.py b/ForgeBlog/forgeblog/main.py
index 00e1c2a..8cb3cfe 100644
--- a/ForgeBlog/forgeblog/main.py
+++ b/ForgeBlog/forgeblog/main.py
@@ -408,6 +408,9 @@ class PostController(BaseController, FeedController):
self.post.delete()
flash('Post deleted', 'info')
redirect(h.really_unicode(c.app.url).encode('utf-8'))
+ else:
+ g.spam_checker.check(kw['title'] + u'\n' + kw['text'], artifact=self.post,
+ user=c.user, content_type='blog-post')
for k, v in kw.iteritems():
setattr(self.post, k, v)
self.post.commit()
http://git-wip-us.apache.org/repos/asf/allura/blob/c780b971/ForgeTracker/forgetracker/tracker_main.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/tracker_main.py b/ForgeTracker/forgetracker/tracker_main.py
index 3227c4f..44053d9 100644
--- a/ForgeTracker/forgetracker/tracker_main.py
+++ b/ForgeTracker/forgetracker/tracker_main.py
@@ -933,8 +933,8 @@ class RootController(BaseController, FeedController):
require_access(c.app, 'create')
self.rate_limit(redir='.')
ticket = TM.Ticket.new()
- g.spam_checker.check(ticket_form['summary'] + u'\n' + ticket_form.get('description', ''), artifact=ticket,
- user=c.user, content_type='ticket')
+ g.spam_checker.check(ticket_form['summary'] + u'\n' + ticket_form.get('description', ''), artifact=ticket,
+ user=c.user, content_type='ticket')
ticket.update(ticket_form)
c.app.globals.invalidate_bin_counts()
g.director.create_activity(c.user, 'created', ticket,
@@ -1464,6 +1464,8 @@ class TicketController(BaseController, FeedController):
@require_post()
def _update_ticket(self, post_data):
require_access(self.ticket, 'update')
+ g.spam_checker.check(post_data.get('summary', '') + u'\n' + post_data.get('description', ''),
+ artifact=self.ticket, user=c.user, content_type='ticket')
changes = changelog()
comment = post_data.pop('comment', None)
labels = post_data.pop('labels', None) or []
[2/2] allura git commit: [#8021] record spam-check results to database
Posted by br...@apache.org.
[#8021] record spam-check results to database
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/ca8b5962
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/ca8b5962
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/ca8b5962
Branch: refs/heads/master
Commit: ca8b596217a38ceadaae0b0d531420a7a69aca04
Parents: da54abd
Author: Dave Brondsema <da...@brondsema.net>
Authored: Mon Dec 19 13:49:43 2016 -0500
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Thu Dec 22 12:31:54 2016 -0500
----------------------------------------------------------------------
Allura/allura/lib/spam/__init__.py | 11 ++++++++
Allura/allura/lib/spam/akismetfilter.py | 2 +-
Allura/allura/lib/spam/mollomfilter.py | 3 +--
Allura/allura/model/artifact.py | 20 ++++++++++++++
.../allura/tests/unit/spam/test_spam_filter.py | 28 +++++++++++++++++++-
AlluraTest/alluratest/controller.py | 3 ++-
6 files changed, 62 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/ca8b5962/Allura/allura/lib/spam/__init__.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/spam/__init__.py b/Allura/allura/lib/spam/__init__.py
index 3d4fe7f..8e68c5f 100644
--- a/Allura/allura/lib/spam/__init__.py
+++ b/Allura/allura/lib/spam/__init__.py
@@ -18,6 +18,7 @@
import logging
from allura.lib.helpers import exceptionless
+from allura.model.artifact import SpamCheckResult
log = logging.getLogger(__name__)
@@ -40,6 +41,16 @@ class SpamFilter(object):
def submit_ham(self, text, artifact=None, user=None, content_type='comment', **kw):
log.info("No spam checking enabled")
+ def record_result(self, result, artifact, user):
+ filter_name = self.__class__.__name__.replace('SpamFilter', '').lower()
+ log.info("spam=%s (%s): %s" % (str(result), filter_name, artifact.url() if artifact else ''))
+ r = SpamCheckResult(
+ ref=artifact.ref,
+ project_id=artifact.project_id,
+ user=user,
+ result=result,
+ )
+
@classmethod
def get(cls, config, entry_points):
"""
http://git-wip-us.apache.org/repos/asf/allura/blob/ca8b5962/Allura/allura/lib/spam/akismetfilter.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/spam/akismetfilter.py b/Allura/allura/lib/spam/akismetfilter.py
index e5ef1b8..7a87bd9 100644
--- a/Allura/allura/lib/spam/akismetfilter.py
+++ b/Allura/allura/lib/spam/akismetfilter.py
@@ -89,7 +89,7 @@ class AkismetSpamFilter(SpamFilter):
request=request,
),
build_data=False)
- log.info("spam=%s (akismet): %s" % (str(res), artifact.url() if artifact else text))
+ self.record_result(res, artifact, user)
return res
def submit_spam(self, text, artifact=None, user=None, content_type='comment'):
http://git-wip-us.apache.org/repos/asf/allura/blob/ca8b5962/Allura/allura/lib/spam/mollomfilter.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/spam/mollomfilter.py b/Allura/allura/lib/spam/mollomfilter.py
index 88a34c8..533186b 100644
--- a/Allura/allura/lib/spam/mollomfilter.py
+++ b/Allura/allura/lib/spam/mollomfilter.py
@@ -75,7 +75,6 @@ class MollomSpamFilter(SpamFilter):
# Should be able to send url, but can't right now due to a bug in
# the PyMollom lib
# kw['url'] = url
- log_msg = url
user = user or c.user
if user:
kw['authorName'] = user.display_name or user.username
@@ -88,7 +87,7 @@ class MollomSpamFilter(SpamFilter):
cc = self.service.checkContent(**kw)
res = cc['spam'] == 2
artifact.spam_check_id = cc.get('session_id', '')
- log.info("spam=%s (mollom): %s" % (str(res), log_msg))
+ self.record_result(res, artifact, user)
return res
def submit_spam(self, text, artifact=None, user=None, content_type='comment', **kw):
http://git-wip-us.apache.org/repos/asf/allura/blob/ca8b5962/Allura/allura/model/artifact.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/artifact.py b/Allura/allura/model/artifact.py
index 523a7ad..a7fe848 100644
--- a/Allura/allura/model/artifact.py
+++ b/Allura/allura/model/artifact.py
@@ -1071,3 +1071,23 @@ class MovedArtifact(Artifact):
'AppConfig', if_missing=lambda: c.app.config._id)
app_config = RelationProperty('AppConfig')
moved_to_url = FieldProperty(str, required=True, allow_none=False)
+
+
+class SpamCheckResult(MappedClass):
+ class __mongometa__:
+ session = main_orm_session
+ name = 'spam_check_result'
+ indexes = [
+ ('project_id', 'result'),
+ ('user_id', 'result'),
+ ]
+
+ _id = FieldProperty(S.ObjectId)
+ ref_id = ForeignIdProperty('ArtifactReference')
+ ref = RelationProperty('ArtifactReference', via='ref_id')
+ project_id = ForeignIdProperty('Project')
+ project = RelationProperty('Project', via='project_id')
+ user_id = ForeignIdProperty('User')
+ user = RelationProperty('User', via='user_id')
+ timestamp = FieldProperty(datetime, if_missing=datetime.utcnow)
+ result = FieldProperty(bool)
http://git-wip-us.apache.org/repos/asf/allura/blob/ca8b5962/Allura/allura/tests/unit/spam/test_spam_filter.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/unit/spam/test_spam_filter.py b/Allura/allura/tests/unit/spam/test_spam_filter.py
index 1d96c7f..9d603f8 100644
--- a/Allura/allura/tests/unit/spam/test_spam_filter.py
+++ b/Allura/allura/tests/unit/spam/test_spam_filter.py
@@ -20,7 +20,14 @@
import mock
import unittest
+from ming.odm import ThreadLocalORMSession
+from nose.tools import assert_equal
+
from allura.lib.spam import SpamFilter
+from allura import model as M
+from allura.model.artifact import SpamCheckResult
+from alluratest.controller import setup_basic_test
+from forgewiki import model as WM
class MockFilter(SpamFilter):
@@ -53,6 +60,25 @@ class TestSpamFilter(unittest.TestCase):
config = {'spam.method': 'mock'}
entry_points = {'mock': MockFilter}
checker = SpamFilter.get(config, entry_points)
- result = checker.check()
+ result = checker.check('this is our text')
self.assertFalse(result)
self.assertTrue(log.exception.called)
+
+
+class TestSpamFilterFunctional(object):
+
+ def setUp(self):
+ setup_basic_test()
+
+ def test_record_result(self):
+ config = {}
+ artifact = WM.Page.query.get()
+ user = M.User.query.get(username='test-user')
+
+ SpamFilter(config).record_result(True, artifact, user)
+ ThreadLocalORMSession.flush_all()
+
+ results = SpamCheckResult.query.find().all()
+ assert_equal(len(results), 1)
+ assert_equal(results[0].result, True)
+ assert_equal(results[0].user.username, 'test-user')
http://git-wip-us.apache.org/repos/asf/allura/blob/ca8b5962/AlluraTest/alluratest/controller.py
----------------------------------------------------------------------
diff --git a/AlluraTest/alluratest/controller.py b/AlluraTest/alluratest/controller.py
index 6b50f33..eae6fba 100644
--- a/AlluraTest/alluratest/controller.py
+++ b/AlluraTest/alluratest/controller.py
@@ -81,7 +81,8 @@ setup_config_test.__test__ = False
def setup_basic_test(config=None, app_name=DFL_APP_NAME):
'''
- Create clean environment for running tests.
+ Create clean environment for running tests, includes mongodb connection with "mim" (mongo-in-memory) and sample
+ data created.
A lightweight alternative is setup_config_test which doesn't bootstrap app data.
'''