You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@roller.apache.org by ag...@apache.org on 2006/01/13 19:13:52 UTC
svn commit: r368814 - in /incubator/roller/trunk: src/org/roller/business/
src/org/roller/business/referrers/ src/org/roller/business/runnable/
src/org/roller/model/ src/org/roller/presentation/filters/
tests/org/roller/business/ web/WEB-INF/classes/
Author: agilliland
Date: Fri Jan 13 10:13:44 2006
New Revision: 368814
URL: http://svn.apache.org/viewcvs?rev=368814&view=rev
Log:
asynchronous referrer processing.
Added:
incubator/roller/trunk/src/org/roller/business/referrers/
incubator/roller/trunk/src/org/roller/business/referrers/IncomingReferrer.java
incubator/roller/trunk/src/org/roller/business/referrers/QueuedReferrerProcessingJob.java
incubator/roller/trunk/src/org/roller/business/referrers/ReferrerProcessingJob.java
incubator/roller/trunk/src/org/roller/business/referrers/ReferrerQueueManager.java
incubator/roller/trunk/src/org/roller/business/referrers/ReferrerQueueManagerImpl.java
incubator/roller/trunk/src/org/roller/business/runnable/
incubator/roller/trunk/src/org/roller/business/runnable/ContinuousWorkerThread.java
incubator/roller/trunk/src/org/roller/business/runnable/Job.java
incubator/roller/trunk/src/org/roller/business/runnable/WorkerThread.java
Modified:
incubator/roller/trunk/src/org/roller/business/RefererManagerImpl.java
incubator/roller/trunk/src/org/roller/business/RollerImpl.java
incubator/roller/trunk/src/org/roller/model/RefererManager.java
incubator/roller/trunk/src/org/roller/model/Roller.java
incubator/roller/trunk/src/org/roller/presentation/filters/RefererFilter.java
incubator/roller/trunk/tests/org/roller/business/RefererManagerTest.java
incubator/roller/trunk/web/WEB-INF/classes/roller.properties
Modified: incubator/roller/trunk/src/org/roller/business/RefererManagerImpl.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/RefererManagerImpl.java?rev=368814&r1=368813&r2=368814&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/RefererManagerImpl.java (original)
+++ incubator/roller/trunk/src/org/roller/business/RefererManagerImpl.java Fri Jan 13 10:13:44 2006
@@ -8,10 +8,13 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.roller.RollerException;
+import org.roller.config.RollerRuntimeConfig;
import org.roller.model.ParsedRequest;
import org.roller.model.RefererManager;
import org.roller.model.Roller;
import org.roller.model.RollerFactory;
+import org.roller.model.UserManager;
+import org.roller.model.WeblogManager;
import org.roller.pojos.RefererData;
import org.roller.pojos.WeblogEntryData;
import org.roller.pojos.WebsiteData;
@@ -37,6 +40,7 @@
protected PersistenceStrategy mStrategy;
protected Date mRefDate = new Date();
protected SimpleDateFormat mDateFormat = DateUtil.get8charDateFormat();
+ protected boolean doLinkbackExtraction = false;
protected abstract List getReferersWithSameTitle(
WebsiteData website,
@@ -63,6 +67,9 @@
public RefererManagerImpl()
{
+ // do we want to do linkback extractions for referrer processing?
+ this.doLinkbackExtraction =
+ RollerRuntimeConfig.getBooleanProperty("site.linkbacks.enabled");
}
//-----------------------------------------------------------------------
@@ -144,6 +151,153 @@
//------------------------------------------------------------------------
+ public void processReferrer(String requestUrl, String referrerUrl,
+ String weblogHandle, String entryAnchor,
+ String dateString) {
+
+ mLogger.debug("processing referrer ["+referrerUrl+
+ "] accessing ["+requestUrl+"]");
+
+ if(weblogHandle == null)
+ return;
+
+ String selfSiteFragment = "/page/"+weblogHandle;
+ WebsiteData weblog = null;
+ WeblogEntryData entry = null;
+
+ // lookup the weblog now
+ try {
+ UserManager userMgr = RollerFactory.getRoller().getUserManager();
+ weblog = userMgr.getWebsiteByHandle(weblogHandle);
+
+ if(weblog == null)
+ return;
+
+ // now lookup weblog entry if possible
+ if(entryAnchor != null) {
+ WeblogManager weblogMgr = RollerFactory.getRoller().getWeblogManager();
+ entry = weblogMgr.getWeblogEntryByAnchor(weblog, entryAnchor);
+ }
+ } catch(RollerException re) {
+ // problem looking up website, gotta bail
+ mLogger.error("Error looking up website object", re);
+ return;
+ }
+
+ try {
+ List matchRef = null;
+
+ // try to find existing RefererData for referrerUrl
+ if (referrerUrl == null || referrerUrl.trim().length() < 8) {
+ referrerUrl = "direct";
+
+ // Get referer specified by referer URL of direct
+ matchRef = getReferersToWebsite(weblog, referrerUrl);
+ } else {
+ referrerUrl = Utilities.stripJsessionId(referrerUrl);
+
+ // Query for referer with same referer and request URLs
+ matchRef = getMatchingReferers(weblog, requestUrl, referrerUrl);
+
+ // If referer was not found, try adding or leaving off 'www'
+ if ( matchRef.size() == 0 ) {
+ String secondTryUrl = null;
+ if ( referrerUrl.startsWith("http://www") ) {
+ secondTryUrl = "http://"+referrerUrl.substring(11);
+ } else {
+ secondTryUrl = "http://www"+referrerUrl.substring(7);
+ }
+
+ matchRef = getMatchingReferers(weblog, requestUrl, secondTryUrl);
+ if ( matchRef.size() == 1 ) {
+ referrerUrl = secondTryUrl;
+ }
+ }
+ }
+
+ if (matchRef.size() == 1) {
+ // Referer was found in database, so bump up hit count
+ RefererData ref = (RefererData)matchRef.get(0);
+
+ ref.setDayHits(new Integer(ref.getDayHits().intValue() + 1));
+ ref.setTotalHits(new Integer(ref.getTotalHits().intValue() + 1));
+
+ mLogger.debug("Incrementing hit count on existing referer: "+referrerUrl);
+
+ storeReferer(ref);
+ mStrategy.commit();
+
+ } else if (matchRef.size() == 0) {
+ /* TODO: change "" for excerpt column back to null
+ * I changed to an empty string to avoid the bug in Derby found
+ * http://issues.apache.org/jira/browse/DERBY-628
+ *
+ * We need to either wait for the fix to change it back,
+ * or leave it as is if it doesn't affect anything else.
+ *
+ * Elias
+ */
+
+ // Referer was not found in database, so new Referer object
+ Integer one = new Integer(1);
+ RefererData ref =
+ new RefererData(
+ null,
+ weblog,
+ entry,
+ dateString,
+ referrerUrl,
+ null,
+ requestUrl,
+ null,
+ "", // Read comment above regarding Derby bug
+ Boolean.FALSE,
+ Boolean.FALSE,
+ one,
+ one);
+
+ if (mLogger.isDebugEnabled()) {
+ mLogger.debug("newReferer="+ref.getRefererUrl());
+ }
+
+ String refurl = ref.getRefererUrl();
+
+ // If not a direct or search engine then search for linkback
+ if (doLinkbackExtraction
+ && dateString != null
+ && !refurl.equals("direct")
+ && !refurl.startsWith("http://google")
+ && !refurl.startsWith("http://www.google")
+ && !refurl.startsWith("http://search.netscape")
+ && !refurl.startsWith("http://www.blinkpro")
+ && !refurl.startsWith("http://auto.search.msn")
+ && !refurl.startsWith("http://search.yahoo")
+ && !refurl.startsWith("http://uk.search.yahoo")
+ && !refurl.startsWith("http://www.javablogs.com")
+ && !refurl.startsWith("http://www.teoma")
+ ) {
+ // Launch thread to extract referer linkback
+
+ try {
+ Roller mRoller = RollerFactory.getRoller();
+ mRoller.getThreadManager().executeInBackground(
+ new LinkbackExtractorRunnable(ref) );
+ } catch (InterruptedException e) {
+ mLogger.warn("Interrupted during linkback extraction",e);
+ }
+ } else {
+ storeReferer(ref);
+ mStrategy.commit();
+ }
+ }
+ } catch (RollerException pe) {
+ mLogger.error(pe);
+ } catch (NullPointerException npe) {
+ mLogger.error(npe);
+ }
+ }
+
+
/**
* Process incoming request for referer information.
*
Modified: incubator/roller/trunk/src/org/roller/business/RollerImpl.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/RollerImpl.java?rev=368814&r1=368813&r2=368814&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/RollerImpl.java (original)
+++ incubator/roller/trunk/src/org/roller/business/RollerImpl.java Fri Jan 13 10:13:44 2006
@@ -11,6 +11,8 @@
import org.apache.commons.logging.LogFactory;
import org.roller.RollerException;
+import org.roller.business.referrers.ReferrerQueueManager;
+import org.roller.business.referrers.ReferrerQueueManagerImpl;
import org.roller.business.utils.UpgradeDatabase;
@@ -106,6 +108,10 @@
return mThemeManager;
}
+ public ReferrerQueueManager getReferrerQueueManager() {
+ return ReferrerQueueManagerImpl.getInstance();
+ }
+
/**
* @see org.roller.model.Roller#getPluginManager()
*/
@@ -320,6 +326,7 @@
public void shutdown() {
try {
+ if(getReferrerQueueManager() != null) getReferrerQueueManager().shutdown();
if (mIndexManager != null) mIndexManager.shutdown();
if (mThreadManager != null) mThreadManager.shutdown();
} catch(Exception e) {
Added: incubator/roller/trunk/src/org/roller/business/referrers/IncomingReferrer.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/referrers/IncomingReferrer.java?rev=368814&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/referrers/IncomingReferrer.java (added)
+++ incubator/roller/trunk/src/org/roller/business/referrers/IncomingReferrer.java Fri Jan 13 10:13:44 2006
@@ -0,0 +1,65 @@
+/*
+ * IncomingReferrer.java
+ *
+ * Created on December 20, 2005, 3:39 PM
+ */
+
+package org.roller.business.referrers;
+
+/**
+ * Represents an incoming (unprocessed) referrer.
+ *
+ * @author Allen Gilliland
+ */
+public class IncomingReferrer {
+
+ private String referrerUrl = null;
+ private String requestUrl = null;
+ private String weblogHandle = null;
+ private String weblogAnchor = null;
+ private String weblogDateString = null;
+
+
+ public IncomingReferrer() {}
+
+ public String getReferrerUrl() {
+ return referrerUrl;
+ }
+
+ public void setReferrerUrl(String referrerUrl) {
+ this.referrerUrl = referrerUrl;
+ }
+
+ public String getRequestUrl() {
+ return requestUrl;
+ }
+
+ public void setRequestUrl(String requestUrl) {
+ this.requestUrl = requestUrl;
+ }
+
+ public String getWeblogHandle() {
+ return weblogHandle;
+ }
+
+ public void setWeblogHandle(String weblogHandle) {
+ this.weblogHandle = weblogHandle;
+ }
+
+ public String getWeblogAnchor() {
+ return weblogAnchor;
+ }
+
+ public void setWeblogAnchor(String weblogAnchor) {
+ this.weblogAnchor = weblogAnchor;
+ }
+
+ public String getWeblogDateString() {
+ return weblogDateString;
+ }
+
+ public void setWeblogDateString(String weblogDateString) {
+ this.weblogDateString = weblogDateString;
+ }
+
+}
Added: incubator/roller/trunk/src/org/roller/business/referrers/QueuedReferrerProcessingJob.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/referrers/QueuedReferrerProcessingJob.java?rev=368814&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/referrers/QueuedReferrerProcessingJob.java (added)
+++ incubator/roller/trunk/src/org/roller/business/referrers/QueuedReferrerProcessingJob.java Fri Jan 13 10:13:44 2006
@@ -0,0 +1,43 @@
+/*
+ * QueuedReferrerProcessingJob.java
+ *
+ * Created on December 20, 2005, 3:08 PM
+ */
+
+package org.roller.business.referrers;
+
+import org.roller.model.RollerFactory;
+
+
+/**
+ * Same as the ReferrerProcessingJob, except that we add a little logic that
+ * tries to lookup incoming referrers from the ReferrerQueueManager.
+ *
+ * @author Allen Gilliland
+ */
+public class QueuedReferrerProcessingJob extends ReferrerProcessingJob {
+
+ public QueuedReferrerProcessingJob() {
+ super();
+ }
+
+
+ public void execute() {
+
+ ReferrerQueueManager refQueue =
+ RollerFactory.getRoller().getReferrerQueueManager();
+
+ // check the queue for any incoming referrers
+ referrer = refQueue.dequeue();
+
+ // work until the queue is empty
+ while(referrer != null) {
+ super.execute();
+
+ // check if there are more referrers to process
+ referrer = refQueue.dequeue();
+ }
+
+ }
+
+}
Added: incubator/roller/trunk/src/org/roller/business/referrers/ReferrerProcessingJob.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/referrers/ReferrerProcessingJob.java?rev=368814&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/referrers/ReferrerProcessingJob.java (added)
+++ incubator/roller/trunk/src/org/roller/business/referrers/ReferrerProcessingJob.java Fri Jan 13 10:13:44 2006
@@ -0,0 +1,84 @@
+/*
+ * ReferrerProcessingJob.java
+ *
+ * Created on December 16, 2005, 6:26 PM
+ */
+
+package org.roller.business.referrers;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.roller.RollerException;
+import org.roller.business.runnable.Job;
+import org.roller.model.RefererManager;
+import org.roller.model.RollerFactory;
+
+
+/**
+ * A simple Job which processes an IncomingReferrer.
+ *
+ * @author Allen Gilliland
+ */
+public class ReferrerProcessingJob implements Job {
+
+ private static Log mLogger = LogFactory.getLog(ReferrerProcessingJob.class);
+
+ Map inputs = null;
+ IncomingReferrer referrer = null;
+
+ public ReferrerProcessingJob() {}
+
+
+ /**
+ * Execute job.
+ *
+ * We simply pass the referrer into the RefererManager to handle the details.
+ */
+ public void execute() {
+
+ if(this.referrer == null)
+ return;
+
+ mLogger.debug("PROCESSING: "+referrer.getRequestUrl());
+
+ // process a referrer
+ try {
+ RefererManager refMgr = RollerFactory.getRoller().getRefererManager();
+ refMgr.processReferrer(referrer.getRequestUrl(),
+ referrer.getReferrerUrl(),
+ referrer.getWeblogHandle(),
+ referrer.getWeblogAnchor(),
+ referrer.getWeblogDateString());
+ } catch(RollerException re) {
+ // trouble
+ mLogger.warn("Trouble processing referrer", re);
+ }
+ }
+
+
+ /**
+ * Set input.
+ */
+ public void input(Map input) {
+ this.inputs = input;
+
+ // we are looking for the "referrer" key
+ Object ref = input.get("referrer");
+
+ if(ref instanceof IncomingReferrer) {
+ this.referrer = (IncomingReferrer) ref;
+ }
+ }
+
+
+ /**
+ * Get output.
+ */
+ public Map output() {
+
+ return null;
+ }
+
+}
Added: incubator/roller/trunk/src/org/roller/business/referrers/ReferrerQueueManager.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/referrers/ReferrerQueueManager.java?rev=368814&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/referrers/ReferrerQueueManager.java (added)
+++ incubator/roller/trunk/src/org/roller/business/referrers/ReferrerQueueManager.java Fri Jan 13 10:13:44 2006
@@ -0,0 +1,46 @@
+/*
+ * ReferrerQueueManager.java
+ *
+ * Created on December 16, 2005, 5:37 PM
+ */
+
+package org.roller.business.referrers;
+
+/**
+ * A queue for incoming referrers.
+ *
+ * @author Allen Gilliland
+ */
+public interface ReferrerQueueManager {
+
+ /**
+ * Process an incoming referrer.
+ *
+ * This method may contain additional logic on how to deal with referrers.
+ * It may process them immediately or it may store them for later processing.
+ */
+ public void processReferrer(IncomingReferrer ref);
+
+
+ /**
+ * Add a referrer to the queue.
+ *
+ * It is almost always preferable to call processReferrer() instead.
+ */
+ public void enqueue(IncomingReferrer ref);
+
+
+ /**
+ * Get the next item in the queue.
+ *
+ * Returns null if there is nothing in the queue.
+ */
+ public IncomingReferrer dequeue();
+
+
+ /**
+ * Called when the system is being shutdown.
+ */
+ public void shutdown();
+
+}
Added: incubator/roller/trunk/src/org/roller/business/referrers/ReferrerQueueManagerImpl.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/referrers/ReferrerQueueManagerImpl.java?rev=368814&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/referrers/ReferrerQueueManagerImpl.java (added)
+++ incubator/roller/trunk/src/org/roller/business/referrers/ReferrerQueueManagerImpl.java Fri Jan 13 10:13:44 2006
@@ -0,0 +1,187 @@
+/*
+ * ReferrerQueueManagerImpl.java
+ *
+ * Created on December 16, 2005, 5:06 PM
+ */
+
+package org.roller.business.referrers;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.roller.business.runnable.ContinuousWorkerThread;
+import org.roller.business.runnable.WorkerThread;
+import org.roller.config.RollerConfig;
+
+
+/**
+ * The base implementation of the ReferrerQueueManager.
+ *
+ * This class is implemented using the singleton pattern to ensure that only
+ * one instance exists at any given time.
+ *
+ * This implementation can be configured to handle referrers in 2 ways ...
+ * 1. synchronously. referrers are processed immediately.
+ * 2. asynchronously. referrers are queued for later processing.
+ *
+ * Users can control the referrer queue mode via properties in the static
+ * roller.properties configuration file.
+ *
+ * In asynchronous processing mode we start some number of worker threads which
+ * run continously to process any referrers that have been queued. Each worker
+ * processes queued referrers until the queue is empty, then sleeps for a given
+ * amount of time. The number of workers used and their sleep time can be set
+ * via properties of the static roller.properties file.
+ *
+ * @author Allen Gilliland
+ */
+public class ReferrerQueueManagerImpl implements ReferrerQueueManager {
+
+ private static Log mLogger = LogFactory.getLog(ReferrerQueueManagerImpl.class);
+
+ private static ReferrerQueueManager instance = null;
+
+ private boolean asyncMode = false;
+ private int numWorkers = 1;
+ private int sleepTime = 10000;
+ private List workers = null;
+ private List referrerQueue = null;
+
+ static {
+ instance = new ReferrerQueueManagerImpl();
+ }
+
+
+ // private because we are a singleton
+ private ReferrerQueueManagerImpl() {
+ mLogger.info("Initializing Referrer Queue Manager");
+
+ // lookup config options
+ this.asyncMode = RollerConfig.getBooleanProperty("referrers.asyncProcessing.enabled");
+
+ mLogger.info("Asynchronous referrer processing = "+this.asyncMode);
+
+ if(this.asyncMode) {
+
+
+ String num = RollerConfig.getProperty("referrers.queue.numWorkers");
+ String sleep = RollerConfig.getProperty("referrers.queue.sleepTime");
+
+ try {
+ this.numWorkers = Integer.parseInt(num);
+
+ if(numWorkers < 1)
+ this.numWorkers = 1;
+
+ } catch(NumberFormatException nfe) {
+ mLogger.warn("Invalid num workers ["+num+"], using default");
+ }
+
+ try {
+ // multiply by 1000 because we expect input in seconds
+ this.sleepTime = Integer.parseInt(sleep) * 1000;
+ } catch(NumberFormatException nfe) {
+ mLogger.warn("Invalid sleep time ["+sleep+"], using default");
+ }
+
+ // create the processing queue
+ this.referrerQueue = Collections.synchronizedList(new ArrayList());
+
+ // start up workers
+ this.workers = new ArrayList();
+ ContinuousWorkerThread worker = null;
+ QueuedReferrerProcessingJob job = null;
+ for(int i=0; i < this.numWorkers; i++) {
+ job = new QueuedReferrerProcessingJob();
+ worker = new ContinuousWorkerThread("ReferrerWorker"+i, job, this.sleepTime);
+ workers.add(worker);
+ worker.start();
+ }
+ }
+ }
+
+
+ /**
+ * Get access to the singleton instance.
+ */
+ public static ReferrerQueueManager getInstance() {
+ return instance;
+ }
+
+
+ /**
+ * Process an incoming referrer.
+ *
+ * If we are doing asynchronous referrer processing then the referrer will
+ * just go into the queue for later processing. If not then we process it
+ * now.
+ */
+ public void processReferrer(IncomingReferrer referrer) {
+
+ if(this.asyncMode) {
+ mLogger.debug("QUEUING: "+referrer.getRequestUrl());
+
+ // add to queue
+ this.enqueue(referrer);
+ } else {
+ // process now
+ ReferrerProcessingJob job = new ReferrerProcessingJob();
+
+ // setup input
+ HashMap inputs = new HashMap();
+ inputs.put("referrer", referrer);
+ job.input(inputs);
+
+ // execute
+ job.execute();
+ }
+
+ }
+
+
+ /**
+ * Place a referrer in the queue.
+ */
+ public void enqueue(IncomingReferrer referrer) {
+ this.referrerQueue.add(referrer);
+ }
+
+
+ /**
+ * Retrieve the next referrer in the queue.
+ */
+ public synchronized IncomingReferrer dequeue() {
+
+ if(!this.referrerQueue.isEmpty()) {
+ return (IncomingReferrer) this.referrerQueue.remove(0);
+ }
+
+ return null;
+ }
+
+
+ /**
+ * clean up.
+ */
+ public void shutdown() {
+
+ if(this.workers.size() > 0) {
+ mLogger.info("stopping all ReferrerQueue worker threads");
+
+ // kill all of our threads
+ WorkerThread worker = null;
+ Iterator it = this.workers.iterator();
+ while(it.hasNext()) {
+ worker = (WorkerThread) it.next();
+ worker.interrupt();
+ }
+ }
+
+ }
+
+}
Added: incubator/roller/trunk/src/org/roller/business/runnable/ContinuousWorkerThread.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/runnable/ContinuousWorkerThread.java?rev=368814&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/runnable/ContinuousWorkerThread.java (added)
+++ incubator/roller/trunk/src/org/roller/business/runnable/ContinuousWorkerThread.java Fri Jan 13 10:13:44 2006
@@ -0,0 +1,79 @@
+/*
+ * ContinuousWorkerThread.java
+ *
+ * Created on December 20, 2005, 1:57 PM
+ */
+
+package org.roller.business.runnable;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+
+/**
+ * A worker that performs a given job continuously.
+ *
+ * @author Allen Gilliland
+ */
+public class ContinuousWorkerThread extends WorkerThread {
+
+ private static Log mLogger = LogFactory.getLog(ContinuousWorkerThread.class);
+
+ // default sleep time is 10 seconds
+ long sleepTime = 10000;
+
+
+ public ContinuousWorkerThread(String id) {
+ super(id);
+ }
+
+
+ public ContinuousWorkerThread(String id, long sleep) {
+ super(id);
+
+ this.sleepTime = sleep;
+ }
+
+
+ public ContinuousWorkerThread(String id, Job job) {
+ super(id, job);
+ }
+
+
+ public ContinuousWorkerThread(String id, Job job, long sleep) {
+ super(id, job);
+
+ this.sleepTime = sleep;
+ }
+
+
+ /**
+ * Thread execution.
+ *
+ * We run forever. Each time a job completes we sleep for
+ * some amount of time before trying again.
+ *
+ * If we ever get interrupted then we quit.
+ */
+ public void run() {
+
+ mLogger.info(this.id+" Started.");
+
+ // run forever
+ while(true) {
+
+ // execute our job
+ super.run();
+
+ // job is done, lets sleep it off for a bit
+ try {
+ mLogger.debug(this.id+" SLEEPING for "+this.sleepTime+" milliseconds ...");
+ this.sleep(this.sleepTime);
+ } catch (InterruptedException e) {
+ mLogger.info(this.id+" INTERRUPT: "+e.getMessage());
+ break;
+ }
+ }
+ }
+
+}
Added: incubator/roller/trunk/src/org/roller/business/runnable/Job.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/runnable/Job.java?rev=368814&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/runnable/Job.java (added)
+++ incubator/roller/trunk/src/org/roller/business/runnable/Job.java Fri Jan 13 10:13:44 2006
@@ -0,0 +1,35 @@
+/*
+ * Job.java
+ *
+ * Created on December 16, 2005, 6:14 PM
+ */
+
+package org.roller.business.runnable;
+
+import java.util.Map;
+
+/**
+ * A job to be executed.
+ *
+ * @author Allen Gilliland
+ */
+public interface Job {
+
+ /**
+ * Execute the job.
+ */
+ public void execute();
+
+
+ /**
+ * Pass in input to be used for the job.
+ */
+ public void input(Map input);
+
+
+ /**
+ * Get any output from the job.
+ */
+ public Map output();
+
+}
Added: incubator/roller/trunk/src/org/roller/business/runnable/WorkerThread.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/runnable/WorkerThread.java?rev=368814&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/runnable/WorkerThread.java (added)
+++ incubator/roller/trunk/src/org/roller/business/runnable/WorkerThread.java Fri Jan 13 10:13:44 2006
@@ -0,0 +1,76 @@
+/*
+ * WorkerThread.java
+ *
+ * Created on December 16, 2005, 6:12 PM
+ */
+
+package org.roller.business.runnable;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+
+/**
+ * A generic worker thread that knows how execute a Job.
+ *
+ * @author Allen Gilliland
+ */
+public class WorkerThread extends Thread {
+
+ private static Log mLogger = LogFactory.getLog(WorkerThread.class);
+
+ String id = null;
+ Job job = null;
+
+
+ /**
+ * A simple worker.
+ */
+ public WorkerThread(String id) {
+ this.id = id;
+ }
+
+
+ /**
+ * Start off with a job to do.
+ */
+ public WorkerThread(String id, Job job) {
+ this.id = id;
+ this.job = job;
+ }
+
+
+ /**
+ * Thread execution.
+ *
+ * We just execute the job we were given if it's non-null.
+ */
+ public void run() {
+
+ // we only run once
+ if (this.job != null) {
+ // process job
+ try {
+ this.job.execute();
+ } catch(Throwable t) {
+ // oops
+ mLogger.error("Error executing job. "+
+ "Worker = "+this.id+", "+
+ "Job = "+this.job.getClass().getName(), t);
+ }
+ }
+
+ }
+
+
+ /**
+ * Set the job for this worker.
+ */
+ public void setJob(Job newJob) {
+ mLogger.debug("NEW JOB: "+newJob.getClass().getName());
+
+ // set the job
+ this.job = newJob;
+ }
+
+}
Modified: incubator/roller/trunk/src/org/roller/model/RefererManager.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/model/RefererManager.java?rev=368814&r1=368813&r2=368814&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/model/RefererManager.java (original)
+++ incubator/roller/trunk/src/org/roller/model/RefererManager.java Fri Jan 13 10:13:44 2006
@@ -96,6 +96,10 @@
//--------------------------------------------- Referer processing methods
+ public void processReferrer(String requestUrl, String referrerUrl,
+ String weblogHandle, String weblogAnchor,
+ String weblogDateString);
+
/**
* Process request for incoming referers.
* @param request Request to be processed.
Modified: incubator/roller/trunk/src/org/roller/model/Roller.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/model/Roller.java?rev=368814&r1=368813&r2=368814&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/model/Roller.java (original)
+++ incubator/roller/trunk/src/org/roller/model/Roller.java Fri Jan 13 10:13:44 2006
@@ -5,6 +5,7 @@
import java.sql.Connection;
import org.roller.RollerException;
import org.roller.business.PersistenceStrategy;
+import org.roller.business.referrers.ReferrerQueueManager;
import org.roller.pojos.UserData;
@@ -46,6 +47,11 @@
* @throws RollerException If unable to create or return RefererManager.
*/
public RefererManager getRefererManager() throws RollerException;
+
+ /**
+ * Get ReferrerQueueManager.
+ */
+ public ReferrerQueueManager getReferrerQueueManager();
/**
* Get RefererManager associated with this Roller instance.
Modified: incubator/roller/trunk/src/org/roller/presentation/filters/RefererFilter.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/presentation/filters/RefererFilter.java?rev=368814&r1=368813&r2=368814&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/presentation/filters/RefererFilter.java (original)
+++ incubator/roller/trunk/src/org/roller/presentation/filters/RefererFilter.java Fri Jan 13 10:13:44 2006
@@ -2,7 +2,6 @@
import java.io.IOException;
import java.util.regex.Pattern;
-
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
@@ -11,14 +10,18 @@
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.roller.model.RefererManager;
+import org.roller.business.referrers.IncomingReferrer;
+import org.roller.business.referrers.ReferrerQueueManager;
import org.roller.model.RollerFactory;
import org.roller.presentation.RollerContext;
-import org.roller.presentation.RollerRequest;
import org.roller.config.RollerConfig;
+import org.roller.model.UserManager;
+import org.roller.pojos.WebsiteData;
+import org.roller.presentation.WeblogPageRequest;
+import org.roller.util.SpamChecker;
+
/**
* Keep track of referers.
@@ -27,88 +30,145 @@
* @web.filter name="RefererFilter"
*/
public class RefererFilter implements Filter {
+
+ private static Log mLogger = LogFactory.getLog(RefererFilter.class);
+ private static final String ROBOT_PATTERN_PROP_NAME = "referrer.robotCheck.userAgentPattern";
+
private FilterConfig mFilterConfig = null;
- private static Log mLogger = LogFactory.getFactory().getInstance(RefererFilter.class);
private static Pattern robotPattern = null;
- private static final String ROBOT_PATTERN_PROP_NAME = "referrer.robotCheck.userAgentPattern";
-
- /**
- * destroy
- */
- public void destroy() {
- }
-
+
+
/**
* doFilter
*/
- public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) throws IOException, ServletException {
+ public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain)
+ throws IOException, ServletException {
+
HttpServletRequest request = (HttpServletRequest) req;
+ HttpServletResponse response = (HttpServletResponse) res;
+ boolean ignoreReferrer = false;
boolean isRefSpammer = false;
boolean isRobot = false;
-
+ String referrerUrl = request.getHeader("Referer");
+ String requestUrl = request.getRequestURL().toString();
+
+ // parse the incoming request and make sure it's a valid page request
+ WeblogPageRequest pageRequest = null;
try {
- if (robotPattern != null) {
- // If the pattern is present, we check for whether the User-Agent matches,
- // and set isRobot if so. Currently, all referral processing, including
- // spam check, is skipped for robots identified in this way.
- String userAgent = request.getHeader("User-Agent");
- isRobot = (userAgent != null && userAgent.length() > 0 && robotPattern.matcher(userAgent).matches());
+ pageRequest = new WeblogPageRequest(request);
+ } catch(Exception e) {
+ // illegal page request
+ response.sendError(HttpServletResponse.SC_NOT_FOUND);
+ mLogger.warn("Illegal page request: "+request.getRequestURL());
+ return;
+ }
+
+ // determine if this request came from a robot
+ if (robotPattern != null) {
+ // If the pattern is present, we check for whether the User-Agent matches,
+ // and set isRobot if so. Currently, all referral processing, including
+ // spam check, is skipped for robots identified in this way.
+ String userAgent = request.getHeader("User-Agent");
+ isRobot = (userAgent != null && userAgent.length() > 0 && robotPattern.matcher(userAgent).matches());
+ }
+
+ // validate the referrer
+ if (pageRequest != null && pageRequest.getWeblogHandle() != null && !isRobot) {
+ String handle = pageRequest.getWeblogHandle();
+
+ RollerContext rctx =
+ RollerContext.getRollerContext(mFilterConfig.getServletContext());
+
+ // Base page URLs, with and without www.
+ String basePageUrlWWW =
+ rctx.getAbsoluteContextUrl(request)+"/page/"+handle;
+ String basePageUrl = basePageUrlWWW;
+ if ( basePageUrlWWW.startsWith("http://www.") ) {
+ // chop off the http://www.
+ basePageUrl = "http://"+basePageUrlWWW.substring(11);
}
-
- if (!isRobot) {
- RollerRequest rreq = RollerRequest.getRollerRequest(request);
- RollerContext rctx = RollerContext.getRollerContext(mFilterConfig.getServletContext());
-
- if (rreq != null && rreq.getWebsite() != null) {
- String handle = rreq.getWebsite().getHandle();
-
- // Base page URLs, with and without www.
- String basePageUrlWWW = rctx.getAbsoluteContextUrl(request) + "/page/" + handle;
- String basePageUrl = basePageUrlWWW;
- if (basePageUrlWWW.startsWith("http://www.")) {
- // chop off the http://www.
- basePageUrl = "http://" + basePageUrlWWW.substring(11);
- }
-
- // Base comment URLs, with and without www.
- String baseCommentsUrlWWW = rctx.getAbsoluteContextUrl(request) + "/comments/" + handle;
- String baseCommentsUrl = baseCommentsUrlWWW;
- if (baseCommentsUrlWWW.startsWith("http://www.")) {
- // chop off the http://www.
- baseCommentsUrl = "http://" + baseCommentsUrlWWW.substring(11);
- }
-
- // Don't process hits from same user's blogs as referers by
- // ignoring Don't process referer from pages that start with base URLs.
- String referer = request.getHeader("Referer");
- if (referer == null || (!referer.startsWith(basePageUrl) && !referer.startsWith(basePageUrlWWW) && !referer.startsWith(baseCommentsUrl) && !referer.startsWith(baseCommentsUrlWWW)))
- {
- RefererManager refMgr = RollerFactory.getRoller().getRefererManager();
- isRefSpammer = refMgr.processRequest(rreq);
+
+ // ignore referres coming from users own blog
+ if (referrerUrl == null ||
+ (!referrerUrl.startsWith(basePageUrl) &&
+ !referrerUrl.startsWith(basePageUrlWWW))) {
+
+ String selfSiteFragment = "/page/"+handle;
+ WebsiteData weblog = null;
+
+ // lookup the weblog now
+ try {
+ UserManager userMgr = RollerFactory.getRoller().getUserManager();
+ weblog = userMgr.getWebsiteByHandle(handle);
+ } catch(Exception e) {
+ // if we can't get the WebsiteData object we can't continue
+ response.sendError(HttpServletResponse.SC_NOT_FOUND);
+ mLogger.error("Error retrieving weblog: "+handle, e);
+ return;
+ }
+
+ // validate the referrer
+ if ( referrerUrl != null ) {
+ // ignore a Referrer from the persons own blog
+ if (referrerUrl.indexOf(selfSiteFragment) != -1) {
+ referrerUrl = null;
+ ignoreReferrer = true;
} else {
- if (mLogger.isDebugEnabled()) {
- mLogger.debug("Ignoring referer=" + referer);
+ // treat editor referral as direct
+ int lastSlash = requestUrl.indexOf("/", 8);
+ if (lastSlash == -1) lastSlash = requestUrl.length();
+ String requestSite = requestUrl.substring(0, lastSlash);
+
+ if (referrerUrl.matches(requestSite + ".*\\.do.*")) {
+ referrerUrl = null;
+ } else {
+ // If referer URL is blacklisted, throw it out
+ isRefSpammer = SpamChecker.checkReferrer(weblog, referrerUrl);
}
}
}
+
+ } else {
+ mLogger.debug("Ignoring referer = "+referrerUrl);
+ ignoreReferrer = true;
}
- } catch (Exception e) {
- mLogger.error("Processing referer", e);
}
-
+
+ // pre-processing complete, let's finish the job
if (isRefSpammer) {
- HttpServletResponse response = (HttpServletResponse) res;
+ // spammers get a 403 Access Denied
response.sendError(HttpServletResponse.SC_FORBIDDEN);
- } else {
- chain.doFilter(req, res);
+ return;
+
+ } else if(!isRobot && !ignoreReferrer) {
+ // referrer is valid, lets record it
+ try {
+ IncomingReferrer referrer = new IncomingReferrer();
+ referrer.setReferrerUrl(referrerUrl);
+ referrer.setRequestUrl(requestUrl);
+ referrer.setWeblogHandle(pageRequest.getWeblogHandle());
+ referrer.setWeblogAnchor(pageRequest.getWeblogAnchor());
+ referrer.setWeblogDateString(pageRequest.getWeblogDate());
+
+ ReferrerQueueManager refQueue =
+ RollerFactory.getRoller().getReferrerQueueManager();
+ refQueue.processReferrer(referrer);
+ } catch(Exception e) {
+ mLogger.error("Error processing referrer", e);
+ }
}
+
+ // referrer processed, continue with request
+ chain.doFilter(req, res);
}
-
+
+
/**
* init
*/
public void init(FilterConfig filterConfig) throws ServletException {
mFilterConfig = filterConfig;
+
String robotPatternStr = RollerConfig.getProperty(ROBOT_PATTERN_PROP_NAME);
if (robotPatternStr != null && robotPatternStr.length() >0) {
// Parse the pattern, and store the compiled form.
@@ -121,4 +181,11 @@
}
}
}
+
+
+ /**
+ * destroy
+ */
+ public void destroy() {}
+
}
Modified: incubator/roller/trunk/tests/org/roller/business/RefererManagerTest.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/tests/org/roller/business/RefererManagerTest.java?rev=368814&r1=368813&r2=368814&view=diff
==============================================================================
--- incubator/roller/trunk/tests/org/roller/business/RefererManagerTest.java (original)
+++ incubator/roller/trunk/tests/org/roller/business/RefererManagerTest.java Fri Jan 13 10:13:44 2006
@@ -1 +1 @@
-package org.roller.business;
import java.sql.Timestamp;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import junit.framework.Test;
import junit.framework.TestSuite;
import org.roller.RollerException;
import org.roller.model.PropertiesManager;
import org.roller.model.RefererManager;
import org.roller.model.Roller;
import org.roller.pojos.RefererData;
import org.roller.pojos.RollerPropertyData;
import org.roller.pojos.UserData;
import org.roller.pojos.WeblogEntryData;
import org.roller.pojos.WebsiteData;
import org.roller.util.DateUtil;
import org.roller.RollerTestBase;
/**
* Test Roller Referer Management.
*/
public class RefererManagerTest extends RollerTestBase
{
RefererManager rmgr;
//List refs;
int count = 20;
String testDay;
String origSpamWords;
//------------------------------------------------------------------------
public RefererManagerTest(String name)
{
super(name);
}
//--------
----------------------------------------------------------------
public static void main(String args[])
{
junit.textui.TestRunner.run(RefererManagerTest.class);
}
//------------------------------------------------------------------------
public static Test suite()
{
return new TestSuite(RefererManagerTest.class);
}
public void setUp() throws Exception
{
super.setUp();
// add "spamtest" to refererSpamWords
Roller mRoller = getRoller();
PropertiesManager pmgr = mRoller.getPropertiesManager();
RollerPropertyData spamprop = pmgr.getProperty("spam.blacklist");
this.origSpamWords = spamprop.getValue();
spamprop.setValue(spamprop.getValue() + ", spamtest");
pmgr.store(spamprop);
// Process count unique referers
rmgr = getRoller().getRefererManager();
Calendar lCalendar = Calendar.getInstance();
lCalendar.setTime(new Date()
);
for (int i = 0; i < count; i++)
{
lCalendar.add(Calendar.DATE, -1);
Timestamp day = new Timestamp(lCalendar.getTime().getTime());
getRoller().begin(UserData.SYSTEM_USER);
MockRequest mock = new MockRequest(
DateUtil.format8chars(day),
"http://test"+i,
"http://test"+i,
null,
mWebsite
);
rmgr.processRequest(mock);
getRoller().commit();
testDay = mock.getDateString();
}
}
public void tearDown() throws Exception
{
List refs = rmgr.getReferers(mWebsite);
// Remove all referers processes
for (int i = 0; i < refs.size(); i++)
{
rmgr.removeReferer(((RefererData)refs.get(i)).getId());
}
// Make sure al
l were removed
refs = rmgr.getReferers(mWebsite);
assertEquals(0,refs.size());
// reset refererSpamWords to original value
Roller mRoller = getRoller();
PropertiesManager pmgr = mRoller.getPropertiesManager();
RollerPropertyData spamprop = pmgr.getProperty("spam.blacklist");
spamprop.setValue(this.origSpamWords);
pmgr.store(spamprop);
super.tearDown();
}
//------------------------------------------------------------------------
public void testGetReferersToDate() throws Exception
{
List referers = rmgr.getReferersToDate(mWebsite, testDay);
assertEquals("Should be one Referer.", referers.size(), 1);
}
//------------------------------------------------------------------------
public void testRefererProcessing() throws RollerException
{
List refs = rmgr.getReferers(mWebsite);
assertEquals("number of r
eferers should equal count", count, refs.size());
int hits = rmgr.getDayHits(mWebsite);
assertEquals("There should be one fewer hits than referers", count, hits);
}
public void testSelfRefererDenial() throws RollerException
{
// test against "self referrals"
getRoller().begin(UserData.SYSTEM_USER);
// create "direct" referer
boolean isSpam = rmgr.processRequest(
new MockRequest(
"20020101",
"direct",
"http://test.com",
null, mWebsite
)
);
getRoller().commit();
assertFalse("is not spam", isSpam);
int newRefCount = rmgr.getReferers(mWebsite).size();
// now create self-referer
getRoller().begin(UserData.SYSTEM_USER);
isSpam = rmgr.processRequest(
new MockRequest(
"20020202",
"http://test.com/pa
ge/" + mWebsite.getHandle(),
"http://test.com",
null, mWebsite
)
);
getRoller().commit();
assertFalse("is not spam", isSpam);
// number of referrers should not have changed
List refs = rmgr.getReferers(mWebsite);
assertEquals("self referal not ignored", newRefCount, refs.size());
// now create self-referer from editor page
isSpam = rmgr.processRequest(
new MockRequest(
"20020202",
"http://test.com/weblog.do",
"http://test.com",
null, mWebsite
)
);
getRoller().commit();
assertFalse("is not spam", isSpam);
// number of referrers should not have changed
refs = rmgr.getReferers(mWebsite);
assertEquals("editor referal not ignored", newRefCount, refs.size());
}
/**
* Test
to see if Referer Spam detection works.
*/
public void testSpamBlocking()
{
boolean isSpam = rmgr.processRequest(
new MockRequest(
"20040101",
"http://www.spamtest.com",
"http://test.com",
null, mWebsite
)
);
//assertTrue("failed to detect referer spam", isSpam);
}
public void testApplyRefererFilters() throws Exception
{
List refs = rmgr.getReferers(mWebsite);
assertEquals(count, refs.size());
String origWords = null;
getRoller().begin(UserData.SYSTEM_USER);
Roller mRoller = getRoller();
PropertiesManager pmgr = mRoller.getPropertiesManager();
RollerPropertyData spamprop = pmgr.getProperty("spam.blacklist");
origWords = spamprop.getValue();
spamprop.setValue(spamprop.getValue() + ", test");
pmgr.store(spamprop);
getRoller().commit();
getRoller().begin(UserData.SYSTEM_USER);
getRoller().getRefererManager().applyRefererFilters();
getRoller().commit();
refs = rmgr.getReferers(mWebsite);
assertEquals(0, refs.size());
getRoller().begin(UserData.SYSTEM_USER);
spamprop = pmgr.getProperty("spam.blacklist");
spamprop.setValue(origWords);
pmgr.store(spamprop);
getRoller().commit();
}
public void testApplyRefererFiltersWebsite() throws Exception
{
List refs = rmgr.getReferers(mWebsite);
assertEquals(count, refs.size());
String origWords = null;
getRoller().begin(UserData.SYSTEM_USER);
mWebsite = getRoller().getUserManager().retrieveWebsite(mWebsite.getId());
origWords = mWebsite.getBlacklist();
mWebsite.setBlacklist("test");
mWebsite.save();
getRoller().commit();
getRoller().begin(UserData.SYSTEM_USER);
getRoller().getRefererManager().applyRefererFilters();
getRoller().commit();
refs = rmgr.getReferers(mWebsite);
assertEquals(0, refs.size());
}
}
class MockRequest implements org.roller.model.ParsedRequest
{
private String mDateStr = null;
private String mRefUrl = null;
private String mReqUrl = null;
private WeblogEntryData mEntry = null;
private WebsiteData mWebsite = null;
private boolean mIsDateSpecified = false;
public MockRequest(
String dateStr, String refUrl, String reqUrl,
WeblogEntryData entry, WebsiteData website)
{
mDateStr = dateStr;
mRefUrl = refUrl;
mReqUrl = reqUrl;
mEntry = entry;
mWebsite = website;
if (mDateStr != null) mIsDateSpecified = true;
}
/**
* @see org.roller.pojos.ParsedRequest#getDateString()
*/
public String getDateString()
{
return mDateStr;
}
/
**
* @see org.roller.pojos.ParsedRequest#getRefererURL()
*/
public String getRefererURL()
{
return mRefUrl;
}
/**
* @see org.roller.pojos.ParsedRequest#getRequestURL()
*/
public String getRequestURL()
{
return mReqUrl;
}
/**
* @see org.roller.pojos.ParsedRequest#getWeblogEntry()
*/
public WeblogEntryData getWeblogEntry()
{
return mEntry;
}
/**
* @see org.roller.pojos.ParsedRequest#getWebsite()
*/
public WebsiteData getWebsite()
{
return mWebsite;
}
/**
* Returns the isDateSpecified.
* @return boolean
*/
public boolean isDateSpecified()
{
return mIsDateSpecified;
}
/**
* Sets the isDateSpecified.
* @param isDateSpecified The isDateSpecified to set
*/
public void setDateSpecified(boolean isDateSpecified)
{
mIsDateSpecified = isDateSpecified;
}
/**
* @see
org.roller.pojos.ParsedRequest#isEnableLinkback()
*/
public boolean isEnableLinkback()
{
return false;
}
}
\ No newline at end of file
+package org.roller.business;
import java.sql.Timestamp;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import junit.framework.Test;
import junit.framework.TestSuite;
import org.roller.RollerException;
import org.roller.model.PropertiesManager;
import org.roller.model.RefererManager;
import org.roller.model.Roller;
import org.roller.pojos.RefererData;
import org.roller.pojos.RollerPropertyData;
import org.roller.pojos.UserData;
import org.roller.pojos.WeblogEntryData;
import org.roller.pojos.WebsiteData;
import org.roller.util.DateUtil;
import org.roller.RollerTestBase;
/**
* Test Roller Referer Management.
*/
public class RefererManagerTest extends RollerTestBase
{
RefererManager rmgr;
//List refs;
int count = 20;
String testDay;
String origSpamWords;
//------------------------------------------------------------------------
public RefererManagerTest(String name)
{
super(name);
}
//--------
----------------------------------------------------------------
public static void main(String args[])
{
junit.textui.TestRunner.run(RefererManagerTest.class);
}
//------------------------------------------------------------------------
public static Test suite()
{
return new TestSuite(RefererManagerTest.class);
}
public void setUp() throws Exception
{
super.setUp();
// add "spamtest" to refererSpamWords
Roller mRoller = getRoller();
PropertiesManager pmgr = mRoller.getPropertiesManager();
RollerPropertyData spamprop = pmgr.getProperty("spam.blacklist");
this.origSpamWords = spamprop.getValue();
spamprop.setValue(spamprop.getValue() + ", spamtest");
pmgr.store(spamprop);
// Process count unique referers
rmgr = getRoller().getRefererManager();
Calendar lCalendar = Calendar.getInstance();
lCalendar.setTime(new Date()
);
for (int i = 0; i < count; i++)
{
lCalendar.add(Calendar.DATE, -1);
Timestamp day = new Timestamp(lCalendar.getTime().getTime());
testDay = DateUtil.format8chars(day);
getRoller().begin(UserData.SYSTEM_USER);
/*
MockRequest mock = new MockRequest(
DateUtil.format8chars(day),
"http://test"+i,
"http://test"+i,
null,
mWebsite
);
rmgr.processRequest(mock);
*/
rmgr.processReferrer("http://test"+i, "http://test"+i,
mWebsite.getHandle(), null, testDay);
getRoller().commit();
//testDay = mock.getDateString();
}
}
public void tearDown() throws Exception
{
List refs = rmgr.getReferers(mW
ebsite);
// Remove all referers processes
for (int i = 0; i < refs.size(); i++)
{
rmgr.removeReferer(((RefererData)refs.get(i)).getId());
}
// Make sure all were removed
refs = rmgr.getReferers(mWebsite);
assertEquals(0,refs.size());
// reset refererSpamWords to original value
Roller mRoller = getRoller();
PropertiesManager pmgr = mRoller.getPropertiesManager();
RollerPropertyData spamprop = pmgr.getProperty("spam.blacklist");
spamprop.setValue(this.origSpamWords);
pmgr.store(spamprop);
super.tearDown();
}
//------------------------------------------------------------------------
public void testGetReferersToDate() throws Exception
{
List referers = rmgr.getReferersToDate(mWebsite, testDay);
assertEquals("Should be one Referer.", referers.size(), 1);
}
//---------------
---------------------------------------------------------
public void testRefererProcessing() throws RollerException
{
List refs = rmgr.getReferers(mWebsite);
assertEquals("number of referers should equal count", count, refs.size());
int hits = rmgr.getDayHits(mWebsite);
assertEquals("There should be one fewer hits than referers", count, hits);
}
/* Self-Referrer Denial is now caught by the RefererFilter -- Allen G
public void testSelfRefererDenial() throws RollerException
{
// test against "self referrals"
getRoller().begin(UserData.SYSTEM_USER);
// create "direct" referer
boolean isSpam = rmgr.processRequest(
new MockRequest(
"20020101",
"direct",
"http://test.com",
null, mWebsite
)
);
getRoller().commit();
assertFalse("is not spam", isSpam);
int newRefCount = rmgr.getReferers(mWebsite).size();
// now create self-referer
getRoller().begin(UserData.SYSTEM_USER);
isSpam = rmgr.processRequest(
new MockRequest(
"20020202",
"http://test.com/page/" + mWebsite.getHandle(),
"http://test.com",
null, mWebsite
)
);
getRoller().commit();
assertFalse("is not spam", isSpam);
// number of referrers should not have changed
List refs = rmgr.getReferers(mWebsite);
assertEquals("self referal not ignored", newRefCount, refs.size());
// now create self-referer from editor page
isSpam = rmgr.processRequest(
new MockRequest(
"20020202",
"http://test.com/weblog.do",
"http://test.com",
null, mWebsite
)
);
getRoller().commit();
assertFalse("is not spam", isSpam);
// number of referrers should not have changed
refs = rmgr.getReferers(mWebsite);
assertEquals("editor referal not ignored", newRefCount, refs.size());
}
*/
/**
* Test to see if Referer Spam detection works.
*/
/* Referrer spam detection now happens in the RefererFilter -- Allen G
public void testSpamBlocking()
{
boolean isSpam = rmgr.processRequest(
new MockRequest(
"20040101",
"http://www.spamtest.com",
"http://test.com",
null, mWebsite
)
);
//assertTrue("failed to detect referer spam", isSpam);
}
*/
public void testApplyRefererFilters() throws Exception
{
List refs = rmgr.getReferers(mWebsite);
assertEquals(count, refs.size());
String origWords = null;
getRoller().begin(UserData.SYSTEM_USER);
Roller mRoller = getRoller();
PropertiesManager pmgr = mRoller.getPropertiesManager();
RollerPropertyData spamprop = pmgr.getProperty("spam.blacklist");
origWords = spamprop.getValue();
spamprop.setValue(spamprop.getValue() + ", test");
pmgr.store(spamprop);
getRoller().commit();
getRoller().begin(UserData.SYSTEM_USER);
getRoller().getRefererManager().applyRefererFilters();
getRoller().commit();
refs = rmgr.getReferers(mWebsite);
assertEquals(0, refs.size());
getRoller().begin(UserData.SYSTEM_USER);
spamprop = pmgr.getProperty("spam.blacklist");
spamprop.setValue(origWords);
pmgr.store(spamprop);
getRoller().commit();
}
public void testApplyRefererFiltersWebsite() throws Exception
{
List refs = rmgr.getReferers(mWebsite);
assertEquals(count, refs.siz
e());
String origWords = null;
getRoller().begin(UserData.SYSTEM_USER);
mWebsite = getRoller().getUserManager().retrieveWebsite(mWebsite.getId());
origWords = mWebsite.getBlacklist();
mWebsite.setBlacklist("test");
mWebsite.save();
getRoller().commit();
getRoller().begin(UserData.SYSTEM_USER);
getRoller().getRefererManager().applyRefererFilters();
getRoller().commit();
refs = rmgr.getReferers(mWebsite);
assertEquals(0, refs.size());
}
}
class MockRequest implements org.roller.model.ParsedRequest
{
private String mDateStr = null;
private String mRefUrl = null;
private String mReqUrl = null;
private WeblogEntryData mEntry = null;
private WebsiteData mWebsite = null;
private boolean mIsDateSpecified = false;
public MockRequest(
String dateStr, String refUrl, String reqUrl,
WeblogEntryData entry, Webs
iteData website)
{
mDateStr = dateStr;
mRefUrl = refUrl;
mReqUrl = reqUrl;
mEntry = entry;
mWebsite = website;
if (mDateStr != null) mIsDateSpecified = true;
}
/**
* @see org.roller.pojos.ParsedRequest#getDateString()
*/
public String getDateString()
{
return mDateStr;
}
/**
* @see org.roller.pojos.ParsedRequest#getRefererURL()
*/
public String getRefererURL()
{
return mRefUrl;
}
/**
* @see org.roller.pojos.ParsedRequest#getRequestURL()
*/
public String getRequestURL()
{
return mReqUrl;
}
/**
* @see org.roller.pojos.ParsedRequest#getWeblogEntry()
*/
public WeblogEntryData getWeblogEntry()
{
return mEntry;
}
/**
* @see org.roller.pojos.ParsedRequest#getWebsite()
*/
public WebsiteData getWebsite()
{
return mWebsite;
}
/**
* Re
turns the isDateSpecified.
* @return boolean
*/
public boolean isDateSpecified()
{
return mIsDateSpecified;
}
/**
* Sets the isDateSpecified.
* @param isDateSpecified The isDateSpecified to set
*/
public void setDateSpecified(boolean isDateSpecified)
{
mIsDateSpecified = isDateSpecified;
}
/**
* @see org.roller.pojos.ParsedRequest#isEnableLinkback()
*/
public boolean isEnableLinkback()
{
return false;
}
}
\ No newline at end of file
Modified: incubator/roller/trunk/web/WEB-INF/classes/roller.properties
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/web/WEB-INF/classes/roller.properties?rev=368814&r1=368813&r2=368814&view=diff
==============================================================================
--- incubator/roller/trunk/web/WEB-INF/classes/roller.properties (original)
+++ incubator/roller/trunk/web/WEB-INF/classes/roller.properties Fri Jan 13 10:13:44 2006
@@ -181,6 +181,7 @@
#----------------------------------
# comment, referrer and trackback settings
+
comment.authenticator.classname=org.roller.presentation.velocity.MathCommentAuthenticator
comment.notification.separateOwnerMessage=false
comment.notification.hideCommenterAddresses=false
@@ -206,6 +207,12 @@
#that get a lot of legitimate crawler bot traffic. The pattern here is a suggestion that
#has been reported to work well.
#referrer.robotCheck.userAgentPattern=.*(slurp|bot|java).*
+
+# Change to true if you want to process referrers asynchronously.
+# You can choose how many threads to use and sleep time (in seconds) between work.
+referrers.asyncProcessing.enabled=false
+referrers.queue.numWorkers=3
+referrers.queue.sleepTime=10
#----------------------------------
# ping settings