You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@roller.apache.org by sn...@apache.org on 2006/05/02 00:23:34 UTC
svn commit: r398712 [5/32] - in /incubator/roller/trunk/src/org/apache: ./
roller/ roller/business/ roller/business/hibernate/
roller/business/referrers/ roller/business/runnable/
roller/business/search/ roller/business/search/operations/ roller/busine...
Added: incubator/roller/trunk/src/org/apache/roller/business/referrers/ReferrerQueueManagerImpl.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/referrers/ReferrerQueueManagerImpl.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/referrers/ReferrerQueueManagerImpl.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/referrers/ReferrerQueueManagerImpl.java Mon May 1 15:23:02 2006
@@ -0,0 +1,216 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * ReferrerQueueManagerImpl.java
+ *
+ * Created on December 16, 2005, 5:06 PM
+ */
+
+package org.apache.roller.business.referrers;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.roller.RollerException;
+import org.apache.roller.business.runnable.ContinuousWorkerThread;
+import org.apache.roller.business.runnable.WorkerThread;
+import org.apache.roller.config.RollerConfig;
+import org.apache.roller.model.RollerFactory;
+
+
+/**
+ * The base implementation of the ReferrerQueueManager.
+ *
+ * This class is implemented using the singleton pattern to ensure that only
+ * one instance exists at any given time.
+ *
+ * This implementation can be configured to handle referrers in 2 ways ...
+ * 1. synchronously. referrers are processed immediately.
+ * 2. asynchronously. referrers are queued for later processing.
+ *
+ * Users can control the referrer queue mode via properties in the static
+ * roller.properties configuration file.
+ *
+ * In asynchronous processing mode we start some number of worker threads which
+ * run continously to process any referrers that have been queued. Each worker
+ * processes queued referrers until the queue is empty, then sleeps for a given
+ * amount of time. The number of workers used and their sleep time can be set
+ * via properties of the static roller.properties file.
+ *
+ * @author Allen Gilliland
+ */
+public class ReferrerQueueManagerImpl implements ReferrerQueueManager {
+
+ private static Log mLogger = LogFactory.getLog(ReferrerQueueManagerImpl.class);
+
+ private static ReferrerQueueManager instance = null;
+
+ private boolean asyncMode = false;
+ private int numWorkers = 1;
+ private int sleepTime = 10000;
+ private List workers = null;
+ private List referrerQueue = null;
+
+ static {
+ instance = new ReferrerQueueManagerImpl();
+ }
+
+
+ // private because we are a singleton
+ private ReferrerQueueManagerImpl() {
+ mLogger.info("Initializing Referrer Queue Manager");
+
+ // lookup config options
+ this.asyncMode = RollerConfig.getBooleanProperty("referrers.asyncProcessing.enabled");
+
+ mLogger.info("Asynchronous referrer processing = "+this.asyncMode);
+
+ if(this.asyncMode) {
+
+
+ String num = RollerConfig.getProperty("referrers.queue.numWorkers");
+ String sleep = RollerConfig.getProperty("referrers.queue.sleepTime");
+
+ try {
+ this.numWorkers = Integer.parseInt(num);
+
+ if(numWorkers < 1)
+ this.numWorkers = 1;
+
+ } catch(NumberFormatException nfe) {
+ mLogger.warn("Invalid num workers ["+num+"], using default");
+ }
+
+ try {
+ // multiply by 1000 because we expect input in seconds
+ this.sleepTime = Integer.parseInt(sleep) * 1000;
+ } catch(NumberFormatException nfe) {
+ mLogger.warn("Invalid sleep time ["+sleep+"], using default");
+ }
+
+ // create the processing queue
+ this.referrerQueue = Collections.synchronizedList(new ArrayList());
+
+ // start up workers
+ this.workers = new ArrayList();
+ ContinuousWorkerThread worker = null;
+ QueuedReferrerProcessingJob job = null;
+ for(int i=0; i < this.numWorkers; i++) {
+ job = new QueuedReferrerProcessingJob();
+ worker = new ContinuousWorkerThread("ReferrerWorker"+i, job, this.sleepTime);
+ workers.add(worker);
+ worker.start();
+ }
+ }
+ }
+
+
+ /**
+ * Get access to the singleton instance.
+ */
+ public static ReferrerQueueManager getInstance() {
+ return instance;
+ }
+
+
+ /**
+ * Process an incoming referrer.
+ *
+ * If we are doing asynchronous referrer processing then the referrer will
+ * just go into the queue for later processing. If not then we process it
+ * now.
+ */
+ public void processReferrer(IncomingReferrer referrer) {
+
+ if(this.asyncMode) {
+ mLogger.debug("QUEUING: "+referrer.getRequestUrl());
+
+ // add to queue
+ this.enqueue(referrer);
+ } else {
+ // process now
+ ReferrerProcessingJob job = new ReferrerProcessingJob();
+
+ // setup input
+ HashMap inputs = new HashMap();
+ inputs.put("referrer", referrer);
+ job.input(inputs);
+
+ // execute
+ job.execute();
+
+ try {
+ // flush changes
+ RollerFactory.getRoller().flush();
+ } catch (RollerException ex) {
+ mLogger.error("ERROR commiting referrer", ex);
+ }
+ }
+
+ }
+
+
+ /**
+ * Place a referrer in the queue.
+ */
+ public void enqueue(IncomingReferrer referrer) {
+ this.referrerQueue.add(referrer);
+
+ if(this.referrerQueue.size() > 250) {
+ mLogger.warn("Referrer queue is rather full. queued="+this.referrerQueue.size());
+ }
+ }
+
+
+ /**
+ * Retrieve the next referrer in the queue.
+ */
+ public synchronized IncomingReferrer dequeue() {
+
+ if(!this.referrerQueue.isEmpty()) {
+ return (IncomingReferrer) this.referrerQueue.remove(0);
+ }
+
+ return null;
+ }
+
+
+ /**
+ * clean up.
+ */
+ public void shutdown() {
+
+ if(this.workers != null && this.workers.size() > 0) {
+ mLogger.info("stopping all ReferrerQueue worker threads");
+
+ // kill all of our threads
+ WorkerThread worker = null;
+ Iterator it = this.workers.iterator();
+ while(it.hasNext()) {
+ worker = (WorkerThread) it.next();
+ worker.interrupt();
+ }
+ }
+
+ }
+
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/roller-persistence.png
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/roller-persistence.png?rev=398712&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/roller/trunk/src/org/apache/roller/business/roller-persistence.png
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/roller/trunk/src/org/apache/roller/business/roller-services-impl.png
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/roller-services-impl.png?rev=398712&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/roller/trunk/src/org/apache/roller/business/roller-services-impl.png
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/roller/trunk/src/org/apache/roller/business/runnable/ContinuousWorkerThread.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/runnable/ContinuousWorkerThread.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/runnable/ContinuousWorkerThread.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/runnable/ContinuousWorkerThread.java Mon May 1 15:23:02 2006
@@ -0,0 +1,96 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * ContinuousWorkerThread.java
+ *
+ * Created on December 20, 2005, 1:57 PM
+ */
+
+package org.apache.roller.business.runnable;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+
+/**
+ * A worker that performs a given job continuously.
+ *
+ * @author Allen Gilliland
+ */
+public class ContinuousWorkerThread extends WorkerThread {
+
+ private static Log mLogger = LogFactory.getLog(ContinuousWorkerThread.class);
+
+ // default sleep time is 10 seconds
+ long sleepTime = 10000;
+
+
+ public ContinuousWorkerThread(String id) {
+ super(id);
+ }
+
+
+ public ContinuousWorkerThread(String id, long sleep) {
+ super(id);
+
+ this.sleepTime = sleep;
+ }
+
+
+ public ContinuousWorkerThread(String id, Job job) {
+ super(id, job);
+ }
+
+
+ public ContinuousWorkerThread(String id, Job job, long sleep) {
+ super(id, job);
+
+ this.sleepTime = sleep;
+ }
+
+
+ /**
+ * Thread execution.
+ *
+ * We run forever. Each time a job completes we sleep for
+ * some amount of time before trying again.
+ *
+ * If we ever get interrupted then we quit.
+ */
+ public void run() {
+
+ mLogger.info(this.id+" Started.");
+
+ // run forever
+ while(true) {
+
+ // execute our job
+ super.run();
+
+ // job is done, lets sleep it off for a bit
+ try {
+ mLogger.debug(this.id+" SLEEPING for "+this.sleepTime+" milliseconds ...");
+ this.sleep(this.sleepTime);
+ } catch (InterruptedException e) {
+ mLogger.info(this.id+" INTERRUPT: "+e.getMessage());
+ break;
+ }
+ }
+ }
+
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/runnable/Job.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/runnable/Job.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/runnable/Job.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/runnable/Job.java Mon May 1 15:23:02 2006
@@ -0,0 +1,52 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Job.java
+ *
+ * Created on December 16, 2005, 6:14 PM
+ */
+
+package org.apache.roller.business.runnable;
+
+import java.util.Map;
+
+/**
+ * A job to be executed.
+ *
+ * @author Allen Gilliland
+ */
+public interface Job {
+
+ /**
+ * Execute the job.
+ */
+ public void execute();
+
+
+ /**
+ * Pass in input to be used for the job.
+ */
+ public void input(Map input);
+
+
+ /**
+ * Get any output from the job.
+ */
+ public Map output();
+
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/runnable/WorkerThread.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/runnable/WorkerThread.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/runnable/WorkerThread.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/runnable/WorkerThread.java Mon May 1 15:23:02 2006
@@ -0,0 +1,99 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * WorkerThread.java
+ *
+ * Created on December 16, 2005, 6:12 PM
+ */
+
+package org.apache.roller.business.runnable;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.roller.model.Roller;
+import org.apache.roller.model.RollerFactory;
+
+
+/**
+ * A generic worker thread that knows how execute a Job.
+ *
+ * @author Allen Gilliland
+ */
+public class WorkerThread extends Thread {
+
+ private static Log mLogger = LogFactory.getLog(WorkerThread.class);
+
+ String id = null;
+ Job job = null;
+
+
+ /**
+ * A simple worker.
+ */
+ public WorkerThread(String id) {
+ this.id = id;
+ }
+
+
+ /**
+ * Start off with a job to do.
+ */
+ public WorkerThread(String id, Job job) {
+ this.id = id;
+ this.job = job;
+ }
+
+
+ /**
+ * Thread execution.
+ *
+ * We just execute the job we were given if it's non-null.
+ */
+ public void run() {
+
+ // we only run once
+ if (this.job != null) {
+ // process job
+ try {
+ this.job.execute();
+ } catch(Throwable t) {
+ // oops
+ mLogger.error("Error executing job. "+
+ "Worker = "+this.id+", "+
+ "Job = "+this.job.getClass().getName(), t);
+ }
+
+ // since this is a thread we have to make sure that we tidy up ourselves
+ Roller roller = RollerFactory.getRoller();
+ roller.release();
+ }
+
+ }
+
+
+ /**
+ * Set the job for this worker.
+ */
+ public void setJob(Job newJob) {
+ mLogger.debug("NEW JOB: "+newJob.getClass().getName());
+
+ // set the job
+ this.job = newJob;
+ }
+
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/search/FieldConstants.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/FieldConstants.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/FieldConstants.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/FieldConstants.java Mon May 1 15:23:02 2006
@@ -0,0 +1,51 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 19, 2003
+ *
+ * Authored by: Mindaugas Idzelis (min@idzelis.com)
+ */
+package org.apache.roller.business.search;
+
+
+/**
+ * @author aim4min
+ *
+ * To change the template for this generated type comment go to
+ * Window>Preferences>Java>Code Generation>Code and Comments
+ */
+public final class FieldConstants
+{
+ //~ Static fields/initializers =============================================
+
+ public static final String ANCHOR = "anchor";
+ public static final String UPDATED = "updated";
+ public static final String ID = "id";
+ public static final String USERNAME = "username";
+ public static final String CATEGORY = "cat";
+ public static final String TITLE = "title";
+ public static final String PUBLISHED = "published";
+ public static final String CONTENT = "content";
+ public static final String CONTENT_STORED = "content_stored";
+ public static final String C_CONTENT = "comment";
+ public static final String C_EMAIL = "email";
+ public static final String C_NAME = "name";
+ public static final String CONSTANT = "constant";
+ public static final String CONSTANT_V = "v"; //must be lowercase, or match the transform rules of the analyzer
+ public static final String WEBSITE_HANDLE = "handle";
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/search/IndexUtil.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/IndexUtil.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/IndexUtil.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/IndexUtil.java Mon May 1 15:23:02 2006
@@ -0,0 +1,66 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 20, 2003
+ *
+ * Authored by: Mindaugas Idzelis (min@idzelis.com)
+ */
+package org.apache.roller.business.search;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.Term;
+import org.apache.roller.business.IndexManagerImpl;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+/**
+ * @author aim4min
+ *
+ * Class containing helper methods.
+ */
+public class IndexUtil {
+
+ /**
+ * Create a lucene term from the first token of the input string.
+ *
+ * @param field The lucene document field to create a term with
+ * @param input The input you wish to convert into a term
+ * @return Lucene search term
+ */
+ public static final Term getTerm(String field, String input) {
+ if (input==null || field==null) return null;
+ Analyzer analyer = IndexManagerImpl.getAnalyzer();
+ TokenStream tokens = analyer.tokenStream(field,
+ new StringReader(input));
+
+ Token token = null;
+ Term term = null;
+ try {
+ token = tokens.next();
+ } catch (IOException e) {}
+ if (token!=null) {
+ String termt = token.termText();
+ term = new Term(field,termt);
+ }
+ return term;
+ }
+
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/AddEntryOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/AddEntryOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/AddEntryOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/AddEntryOperation.java Mon May 1 15:23:02 2006
@@ -0,0 +1,90 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 16, 2003
+ *
+ * Authored by: Mindaugas Idzelis (min@idzelis.com)
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.model.Roller;
+import org.apache.roller.model.RollerFactory;
+import org.apache.roller.pojos.WeblogEntryData;
+
+
+/**
+ * @author aim4min
+ *
+ * An operation that adds a new log entry into the index.
+ */
+public class AddEntryOperation extends WriteToIndexOperation
+{
+ //~ Static fields/initializers =============================================
+
+ private static Log mLogger =
+ LogFactory.getFactory().getInstance(AddEntryOperation.class);
+
+ //~ Instance fields ========================================================
+
+ private WeblogEntryData data;
+
+ //~ Constructors ===========================================================
+
+ /**
+ * Adds a web log entry into the index.
+ */
+ public AddEntryOperation(IndexManagerImpl mgr, WeblogEntryData data)
+ {
+ super(mgr);
+ this.data = data;
+ }
+
+ //~ Methods ================================================================
+
+ /* (non-Javadoc)
+ * @see java.lang.Runnable#run()
+ */
+ public void doRun()
+ {
+ IndexWriter writer = beginWriting();
+ Roller roller = RollerFactory.getRoller();
+ try
+ {
+ if (writer != null)
+ {
+ writer.addDocument(getDocument(data));
+ }
+ }
+ catch (IOException e)
+ {
+ mLogger.error("Problems adding doc to index", e);
+ }
+ finally
+ {
+ if (roller != null) roller.release();
+ endWriting();
+ }
+ }
+
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/IndexOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/IndexOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/IndexOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/IndexOperation.java Mon May 1 15:23:02 2006
@@ -0,0 +1,190 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 16, 2003
+ * Authored by: Mindaugas Idzelis (min@idzelis.com)
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.business.search.FieldConstants;
+import org.apache.roller.pojos.CommentData;
+import org.apache.roller.pojos.WeblogCategoryData;
+import org.apache.roller.pojos.WeblogEntryData;
+import org.apache.roller.util.Utilities;
+import org.apache.roller.config.RollerConfig;
+
+/**
+ * @author aim4min
+ * <p/>
+ * This is the base class for all index operation. These operations include:
+ * <p/>
+ * SearchOperation AddWeblogOperation RemoveWeblogOperation
+ * RebuildUserIndexOperation
+ */
+public abstract class IndexOperation implements Runnable {
+ private static Log mLogger = LogFactory.getFactory().getInstance(IndexOperation.class);
+
+ //~ Instance fields
+ // ========================================================
+ protected IndexManagerImpl manager;
+
+ private IndexReader reader;
+
+ private IndexWriter writer;
+
+ //~ Constructors
+ // ===========================================================
+
+ public IndexOperation(IndexManagerImpl manager) {
+ this.manager = manager;
+ }
+
+ //~ Methods
+ // ================================================================
+
+ protected Document getDocument(WeblogEntryData data) {
+
+ // Actual comment content is indexed only if search.index.comments
+ // is true or absent from the (static) configuration properties.
+ // If false in the configuration, comments are treated as if empty.
+ boolean indexComments = RollerConfig.getBooleanProperty("search.index.comments", true);
+
+ String commentContent = "";
+ String commentEmail = "";
+ String commentName = "";
+ if (indexComments) {
+ List comments = data.getComments();
+ if (comments != null) {
+ StringBuffer commentEmailBuf = new StringBuffer();
+ StringBuffer commentContentBuf = new StringBuffer();
+ StringBuffer commentNameBuf = new StringBuffer();
+ for (Iterator cItr = comments.iterator(); cItr.hasNext();) {
+ CommentData comment = (CommentData) cItr.next();
+ if (comment.getSpam() == null || !comment.getSpam().booleanValue()) {
+ if (comment.getContent() != null) {
+ commentContentBuf.append(comment.getContent());
+ commentContentBuf.append(",");
+ }
+ if (comment.getEmail() != null) {
+ commentEmailBuf.append(comment.getEmail());
+ commentEmailBuf.append(",");
+ }
+ if (comment.getName() != null) {
+ commentNameBuf.append(comment.getName());
+ commentNameBuf.append(",");
+ }
+ }
+ }
+ commentEmail = commentEmailBuf.toString();
+ commentContent = commentContentBuf.toString();
+ commentName = commentNameBuf.toString();
+ }
+ }
+
+ Document doc = new Document();
+
+ doc.add(Field.Keyword(FieldConstants.ID, data.getId()));
+
+ doc.add(Field.Keyword(FieldConstants.WEBSITE_HANDLE, data.getWebsite().getHandle()));
+
+ doc.add(Field.UnIndexed(FieldConstants.ANCHOR, data.getAnchor()));
+ doc.add(Field.Text(FieldConstants.USERNAME, data.getCreator().getUserName()));
+ doc.add(Field.Text(FieldConstants.TITLE, data.getTitle()));
+
+ // index the entry text, but don't store it - moved to end of block
+ doc.add(Field.UnStored(FieldConstants.CONTENT, data.getText()));
+
+ // store an abbreviated version of the entry text, but don't index
+ doc.add(Field.UnIndexed(FieldConstants.CONTENT_STORED, Utilities
+ .truncateNicely(Utilities.removeHTML(data.getText()), 240, 260, "...")));
+
+ doc.add(Field.Keyword(FieldConstants.UPDATED, data.getUpdateTime()
+ .toString()));
+ doc.add(Field.Keyword(FieldConstants.PUBLISHED, data.getPubTime()
+ .toString()));
+
+ // index Comments
+ doc.add(Field.UnStored(FieldConstants.C_CONTENT, commentContent));
+ doc.add(Field.UnStored(FieldConstants.C_EMAIL, commentEmail));
+ doc.add(Field.UnStored(FieldConstants.C_NAME, commentName));
+
+ doc.add(Field.UnStored(FieldConstants.CONSTANT, FieldConstants.CONSTANT_V));
+
+ // index Category
+ WeblogCategoryData categorydata = data.getCategory();
+ Field category = (categorydata == null) ? Field.UnStored(FieldConstants.CATEGORY, "") : Field.Text(FieldConstants.CATEGORY, categorydata.getName());
+ doc.add(category);
+
+ return doc;
+ }
+
+ protected IndexReader beginDeleting() {
+ try {
+ reader = IndexReader.open(manager.getIndexDirectory());
+ } catch (IOException e) {
+ }
+
+ return reader;
+ }
+
+ protected void endDeleting() {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ }
+ }
+ }
+
+ protected IndexWriter beginWriting() {
+ try {
+ writer = new IndexWriter(manager.getIndexDirectory(), IndexManagerImpl.getAnalyzer(), false);
+ } catch (IOException e) {
+ mLogger.error("ERROR creating writer");
+ }
+
+ return writer;
+ }
+
+ protected void endWriting() {
+ if (writer != null) {
+ try {
+ writer.close();
+ } catch (IOException e) {
+ mLogger.error("ERROR closing writer", e);
+ }
+ }
+ }
+
+ public void run() {
+ doRun();
+ }
+
+ protected abstract void doRun();
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReIndexEntryOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReIndexEntryOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReIndexEntryOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReIndexEntryOperation.java Mon May 1 15:23:02 2006
@@ -0,0 +1,109 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 16, 2003
+ *
+ * Authored by: Mindaugas Idzelis (min@idzelis.com)
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.business.search.FieldConstants;
+import org.apache.roller.model.Roller;
+import org.apache.roller.model.RollerFactory;
+import org.apache.roller.pojos.WeblogEntryData;
+
+/**
+ * @author aim4min
+ *
+ * An operation that adds a new log entry into the index.
+ */
+public class ReIndexEntryOperation extends WriteToIndexOperation
+{
+ //~ Static fields/initializers =============================================
+
+ private static Log mLogger =
+ LogFactory.getFactory().getInstance(AddEntryOperation.class);
+
+ //~ Instance fields ========================================================
+
+ private WeblogEntryData data;
+
+ //~ Constructors ===========================================================
+
+ /**
+ * Adds a web log entry into the index.
+ */
+ public ReIndexEntryOperation(IndexManagerImpl mgr, WeblogEntryData data)
+ {
+ super(mgr);
+ this.data = data;
+ }
+
+ //~ Methods ================================================================
+
+ /* (non-Javadoc)
+ * @see java.lang.Runnable#run()
+ */
+ public void doRun()
+ {
+ IndexReader reader = beginDeleting();
+ try
+ {
+ if (reader != null)
+ {
+ Term term = new Term(FieldConstants.ID, data.getId());
+ reader.delete(term);
+ }
+ }
+ catch (IOException e)
+ {
+ mLogger.error("Error deleting doc from index", e);
+ }
+ finally
+ {
+ endDeleting();
+ }
+
+ IndexWriter writer = beginWriting();
+ Roller roller = RollerFactory.getRoller();
+ try
+ {
+ if (writer != null)
+ {
+ writer.addDocument(getDocument(data));
+ }
+ }
+ catch (IOException e)
+ {
+ mLogger.error("Problems adding doc to index", e);
+ }
+ finally
+ {
+ if (roller != null) roller.release();
+ endWriting();
+ }
+ }
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReadFromIndexOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReadFromIndexOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReadFromIndexOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/ReadFromIndexOperation.java Mon May 1 15:23:02 2006
@@ -0,0 +1,64 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+
+package org.apache.roller.business.search.operations;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.roller.business.IndexManagerImpl;
+
+/**
+ * @author aim4min
+ */
+public abstract class ReadFromIndexOperation extends IndexOperation
+{
+
+ /**
+ * @param manager
+ */
+ public ReadFromIndexOperation(IndexManagerImpl mgr)
+ {
+ super(mgr);
+ }
+
+ private static Log mLogger = LogFactory.getFactory().getInstance(
+ ReadFromIndexOperation.class);
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Runnable#run()
+ */
+ public final void run()
+ {
+ try
+ {
+ manager.getReadWriteLock().readLock().acquire();
+ doRun();
+ }
+ catch (InterruptedException e)
+ {
+ mLogger.info("Error acquiring read lock on index", e);
+ }
+ finally
+ {
+ manager.getReadWriteLock().readLock().release();
+ }
+ }
+
+}
\ No newline at end of file
Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/RebuildWebsiteIndexOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/RebuildWebsiteIndexOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/RebuildWebsiteIndexOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/RebuildWebsiteIndexOperation.java Mon May 1 15:23:02 2006
@@ -0,0 +1,173 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 16, 2003
+ *
+ * Authored by: Mindaugas Idzelis (min@idzelis.com)
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.roller.RollerException;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.business.search.FieldConstants;
+import org.apache.roller.business.search.IndexUtil;
+import org.apache.roller.model.Roller;
+import org.apache.roller.model.RollerFactory;
+import org.apache.roller.model.WeblogManager;
+import org.apache.roller.pojos.WeblogEntryData;
+import org.apache.roller.pojos.WebsiteData;
+
+
+/**
+ * @author aim4min
+ *
+ * An index operation that rebuilds a given users index (or all indexes.)
+ */
+public class RebuildWebsiteIndexOperation extends WriteToIndexOperation
+{
+ //~ Static fields/initializers =============================================
+
+ private static Log mLogger =
+ LogFactory.getFactory().getInstance(RebuildWebsiteIndexOperation.class);
+
+ //~ Instance fields ========================================================
+
+ private WebsiteData website;
+
+ //~ Constructors ===========================================================
+
+ /**
+ * Create a new operation that will recreate an index.
+ *
+ * @param website The website to rebuild the index for, or null for all users.
+ */
+ public RebuildWebsiteIndexOperation(IndexManagerImpl mgr, WebsiteData website)
+ {
+ super(mgr);
+ this.website = website;
+ }
+
+ //~ Methods ================================================================
+
+ /* (non-Javadoc)
+ * @see java.lang.Runnable#run()
+ */
+ public void doRun()
+ {
+ Date start = new Date();
+
+ IndexReader reader = beginDeleting();
+
+ try
+ {
+ if (reader != null)
+ {
+ Term tWebsite = null;
+ if (website != null)
+ {
+ tWebsite = IndexUtil.getTerm(
+ FieldConstants.WEBSITE_HANDLE, website.getHandle());
+ }
+ if (tWebsite != null)
+ {
+ reader.delete(tWebsite);
+ }
+ else
+ {
+ Term all =
+ IndexUtil.getTerm(FieldConstants.CONSTANT,
+ FieldConstants.CONSTANT_V);
+ reader.delete(all);
+ }
+ }
+ }
+ catch (IOException e)
+ {
+ mLogger.info("Problems deleting doc from index", e);
+ }
+ finally
+ {
+ endDeleting();
+ }
+
+ IndexWriter writer = beginWriting();
+
+ Roller roller = RollerFactory.getRoller();
+ try
+ {
+ if (writer != null)
+ {
+ WeblogManager weblogManager = roller.getWeblogManager();
+
+ List entries = weblogManager .getWeblogEntries(
+ website, // userName
+ null, // startDate
+ new Date(), // endDate (don't index 'future' entries)
+ null, // catName
+ WeblogEntryData.PUBLISHED, // status
+ null, // sortby (null mean pubTime)
+ null); // maxEntries (null to get 'em all)
+
+ for (Iterator wbItr = entries.iterator(); wbItr.hasNext();)
+ {
+ WeblogEntryData entry = (WeblogEntryData) wbItr.next();
+ writer.addDocument(getDocument(entry));
+ mLogger.debug(
+ MessageFormat.format("Indexed entry {0}: {1}",
+ new Object[] {entry.getPubTime(), entry.getAnchor()}));
+ }
+ // release the database connection
+ roller.release();
+ }
+ }
+ catch (Exception e)
+ {
+ mLogger.error("ERROR adding doc to index", e);
+ }
+ finally
+ {
+ endWriting();
+ if (roller != null) roller.release();
+ }
+
+ Date end = new Date();
+ double length = (end.getTime() - start.getTime()) / (double) 1000;
+
+ if (website == null)
+ {
+ mLogger.info(
+ "Completed rebuilding index for all users in '" + length + "' secs");
+ }
+ else
+ {
+ mLogger.info("Completed rebuilding index for website handle: '" +
+ website.getHandle() + "' in '" + length + "' seconds");
+ }
+ }
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveEntryOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveEntryOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveEntryOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveEntryOperation.java Mon May 1 15:23:02 2006
@@ -0,0 +1,90 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 16, 2003
+ * Authored by: Mindaugas Idzelis (min@idzelis.com)
+ *
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.business.search.FieldConstants;
+import org.apache.roller.pojos.WeblogEntryData;
+
+
+/**
+ * @author aim4min
+ *
+ * An operation that removes the weblog from the index.
+ */
+public class RemoveEntryOperation extends WriteToIndexOperation
+{
+ //~ Static fields/initializers =============================================
+
+ private static Log mLogger =
+ LogFactory.getFactory().getInstance(RemoveEntryOperation.class);
+
+ //~ Instance fields ========================================================
+
+ private WeblogEntryData data;
+
+ //~ Constructors ===========================================================
+
+ /**
+ *
+ */
+ public RemoveEntryOperation(IndexManagerImpl mgr, WeblogEntryData data)
+ {
+ super(mgr);
+ this.data = data;
+ }
+
+ //~ Methods ================================================================
+
+ /* (non-Javadoc)
+ * @see java.lang.Runnable#run()
+ */
+ public void doRun()
+ {
+ IndexReader reader = beginDeleting();
+ try
+ {
+ if (reader != null)
+ {
+ Term term = new Term(FieldConstants.ID, data.getId());
+ reader.delete(term);
+ }
+ }
+ catch (IOException e)
+ {
+ mLogger.error("Error deleting doc from index", e);
+ }
+ finally
+ {
+ endDeleting();
+ }
+ }
+
+
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveWebsiteIndexOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveWebsiteIndexOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveWebsiteIndexOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/RemoveWebsiteIndexOperation.java Mon May 1 15:23:02 2006
@@ -0,0 +1,112 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 16, 2003
+ *
+ * Authored by: Mindaugas Idzelis (min@idzelis.com)
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.business.search.FieldConstants;
+import org.apache.roller.business.search.IndexUtil;
+import org.apache.roller.pojos.WebsiteData;
+
+
+/**
+ * @author aim4min
+ *
+ * An index operation that rebuilds a given users index (or all indexes.)
+ */
+public class RemoveWebsiteIndexOperation extends WriteToIndexOperation
+{
+ //~ Static fields/initializers =============================================
+
+ private static Log mLogger =
+ LogFactory.getFactory().getInstance(RemoveWebsiteIndexOperation.class);
+
+ //~ Instance fields ========================================================
+
+ private WebsiteData website;
+
+ //~ Constructors ===========================================================
+
+ /**
+ * Create a new operation that will recreate an index.
+ *
+ * @param website The website to rebuild the index for, or null for all sites.
+ */
+ public RemoveWebsiteIndexOperation(IndexManagerImpl mgr, WebsiteData website)
+ {
+ super(mgr);
+ this.website = website;
+ }
+
+ //~ Methods ================================================================
+
+ /* (non-Javadoc)
+ * @see java.lang.Runnable#run()
+ */
+ public void doRun()
+ {
+ Date start = new Date();
+ IndexReader reader = beginDeleting();
+ try
+ {
+ if (reader != null)
+ {
+ String handle = null;
+ if (website != null)
+ {
+ handle = website.getHandle();
+ }
+ Term tHandle =
+ IndexUtil.getTerm(FieldConstants.WEBSITE_HANDLE, handle);
+
+ if (tHandle != null)
+ {
+ reader.delete(tHandle);
+ }
+ }
+ }
+ catch (IOException e)
+ {
+ mLogger.info("Problems deleting doc from index", e);
+ }
+ finally
+ {
+ endDeleting();
+ }
+
+ Date end = new Date();
+ double length = (end.getTime() - start.getTime()) / (double) 1000;
+
+ if (website != null)
+ {
+ mLogger.info("Completed deleting indices for website '" +
+ website.getName() + "' in '" + length + "' seconds");
+ }
+ }
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/SearchOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/SearchOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/SearchOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/SearchOperation.java Mon May 1 15:23:02 2006
@@ -0,0 +1,179 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Jul 18, 2003
+ * Authored by: Mindaugas Idzelis (min@idzelis.com)
+ */
+package org.apache.roller.business.search.operations;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queryParser.MultiFieldQueryParser;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.roller.business.IndexManagerImpl;
+import org.apache.roller.business.search.FieldConstants;
+import org.apache.roller.business.search.IndexUtil;
+import org.apache.roller.model.IndexManager;
+
+
+/**
+ * @author aim4min
+ *
+ * An operation that searches the index.
+ */
+public class SearchOperation extends ReadFromIndexOperation
+{
+ //~ Static fields/initializers =============================================
+
+ private static Log mLogger =
+ LogFactory.getFactory().getInstance(SearchOperation.class);
+
+ private static String[] SEARCH_FIELDS = new String[]{
+ FieldConstants.CONTENT, FieldConstants.TITLE,
+ FieldConstants.C_CONTENT, FieldConstants.CATEGORY
+ };
+
+ private static Sort SORTER = new Sort( new SortField(
+ FieldConstants.PUBLISHED, SortField.STRING, true) );
+
+ //~ Instance fields ========================================================
+
+ private String term;
+ private String websiteHandle;
+ private String category;
+ private Hits searchresults;
+ private String parseError;
+
+ //~ Constructors ===========================================================
+
+ /**
+ * Create a new operation that searches the index.
+ */
+ public SearchOperation(IndexManager mgr)
+ {
+ // TODO: finish moving IndexManager to backend, so this cast is not needed
+ super((IndexManagerImpl)mgr);
+ }
+
+ //~ Methods ================================================================
+
+ public void setTerm(String term)
+ {
+ this.term = term;
+ }
+
+ /* (non-Javadoc)
+ * @see java.lang.Runnable#run()
+ */
+ public void doRun()
+ {
+ searchresults = null;
+
+ IndexSearcher searcher = null;
+
+ try
+ {
+ IndexReader reader = manager.getSharedIndexReader();
+ searcher = new IndexSearcher(reader);
+
+ Query query =
+ MultiFieldQueryParser.parse(
+ term, SEARCH_FIELDS, new StandardAnalyzer());
+
+ Term tUsername =
+ IndexUtil.getTerm(FieldConstants.WEBSITE_HANDLE, websiteHandle);
+
+ if (tUsername != null)
+ {
+ BooleanQuery bQuery = new BooleanQuery();
+ bQuery.add(query, true, false);
+ bQuery.add(new TermQuery(tUsername), true, false);
+ query = bQuery;
+ }
+
+ Term tCategory =
+ IndexUtil.getTerm(FieldConstants.CATEGORY, category);
+
+ if (tCategory != null)
+ {
+ BooleanQuery bQuery = new BooleanQuery();
+ bQuery.add(query, true, false);
+ bQuery.add(new TermQuery(tCategory), true, false);
+ query = bQuery;
+ }
+ searchresults = searcher.search(query, null/*Filter*/, SORTER);
+ }
+ catch (IOException e)
+ {
+ mLogger.error("Error searching index", e);
+ parseError = e.getMessage();
+ }
+ catch (ParseException e)
+ {
+ // who cares?
+ parseError = e.getMessage();
+ }
+ // don't need to close the reader, since we didn't do any writing!
+ }
+
+ public Hits getResults()
+ {
+ return searchresults;
+ }
+
+ public int getResultsCount()
+ {
+ if (searchresults == null) return -1;
+
+ return searchresults.length();
+ }
+
+ public String getParseError()
+ {
+ return parseError;
+ }
+
+ /**
+ * @param string
+ */
+ public void setWebsiteHandle(String websiteHandle)
+ {
+ this.websiteHandle = websiteHandle;
+ }
+
+ /**
+ * @param parameter
+ */
+ public void setCategory(String category)
+ {
+ this.category = category;
+ }
+
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/WriteToIndexOperation.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/WriteToIndexOperation.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/WriteToIndexOperation.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/WriteToIndexOperation.java Mon May 1 15:23:02 2006
@@ -0,0 +1,71 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+/*
+ * Created on Aug 12, 2003
+ *
+ * To change the template for this generated file go to
+ * Window>Preferences>Java>Code Generation>Code and Comments
+ */
+package org.apache.roller.business.search.operations;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.roller.business.IndexManagerImpl;
+
+/**
+ * @author aim4min
+ *
+ * To change the template for this generated type comment go to
+ * Window>Preferences>Java>Code Generation>Code and Comments
+ */
+public abstract class WriteToIndexOperation extends IndexOperation {
+
+ /**
+ * @param manager
+ */
+ public WriteToIndexOperation(IndexManagerImpl mgr)
+ {
+ super(mgr);
+ }
+
+ private static Log mLogger =
+ LogFactory.getFactory().getInstance(WriteToIndexOperation.class);
+
+ /* (non-Javadoc)
+ * @see java.lang.Runnable#run()
+ */
+ public void run()
+ {
+ try
+ {
+ manager.getReadWriteLock().writeLock().acquire();
+ mLogger.info("Starting search index operation");
+ doRun();
+ mLogger.info("Search index operation complete");
+ }
+ catch (InterruptedException e)
+ {
+ mLogger.error("Error acquiring write lock on index", e);
+ }
+ finally
+ {
+ manager.getReadWriteLock().writeLock().release();
+ }
+ manager.resetSharedReader();
+ }
+}
Added: incubator/roller/trunk/src/org/apache/roller/business/search/operations/package.html
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/operations/package.html?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/operations/package.html (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/operations/package.html Mon May 1 15:23:02 2006
@@ -0,0 +1,26 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. The ASF licenses this file to You
+ under the Apache License, Version 2.0 (the "License"); you may not
+ use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. For additional information regarding
+ copyright in this work, please see the NOTICE file in the top level
+ directory of this distribution.
+-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+ <title></title>
+</head>
+<body>
+Lucene-based search operations to be executed by ThreadManager.<br>
+</body>
+</html>
Added: incubator/roller/trunk/src/org/apache/roller/business/search/package.html
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/search/package.html?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/search/package.html (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/search/package.html Mon May 1 15:23:02 2006
@@ -0,0 +1,26 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. The ASF licenses this file to You
+ under the Apache License, Version 2.0 (the "License"); you may not
+ use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. For additional information regarding
+ copyright in this work, please see the NOTICE file in the top level
+ directory of this distribution.
+-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+ <title></title>
+</head>
+<body>
+Utilities needed by implementation of search interface.
+</body>
+</html>
Added: incubator/roller/trunk/src/org/apache/roller/business/utils/ConsistencyCheck.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/apache/roller/business/utils/ConsistencyCheck.java?rev=398712&view=auto
==============================================================================
--- incubator/roller/trunk/src/org/apache/roller/business/utils/ConsistencyCheck.java (added)
+++ incubator/roller/trunk/src/org/apache/roller/business/utils/ConsistencyCheck.java Mon May 1 15:23:02 2006
@@ -0,0 +1,332 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+
+package org.apache.roller.business.utils;
+
+import java.io.FileInputStream;
+import java.sql.*;
+import java.util.*;
+
+/**
+ * Roller database consistency checker.<br />
+ * Don't run this unless you know what you are doing!</br >
+ *
+ * <p>Configuration:<br />
+ *
+ * Program looks in current directory for db.properties file with database
+ * connection properties driverClassName and connectionUrl.
+ *
+ * Program expects JDBC driver jar to be on classpath.</p>
+ *
+ * <p>Usage:<br />
+ *
+ * java -cp ./WEB-INF/lib/rollerbeans.jar org.apache.roller.business.utils.ConsistencyCheck<br />
+ *
+ * <br />Options:<br />
+ * -v Verbose<br />
+ * -purge Delete orphans</p>
+ */
+public class ConsistencyCheck
+{
+ /**
+ * Consistency checker, find and optionally delete orphans.
+ */
+ public static void main(String[] args) throws Exception
+ {
+ Properties props = new Properties();
+ props.load(new FileInputStream("rollerdb.properties"));
+ Connection con = createConnection(props,"");
+
+ boolean delete = false;
+ boolean verbose = false;
+ if (args.length > 0)
+ {
+ if ("-purge".equals(args[0]))
+ {
+ delete = true;
+ }
+ else if ("-v".equals(args[0]))
+ {
+ verbose = true;
+ }
+ }
+
+ findAndDeleteOrphans(con, delete, verbose);
+ }
+
+ /**
+ * Create connection based on properties:<br/>
+ * - driverClassName<br/>
+ * - connectionUrl<br/>
+ * - userName<br/>
+ * - password<br/>
+ */
+ public static Connection createConnection(Properties props, String prefix)
+ throws Exception
+ {
+ Connection con = null;
+ if (prefix == null)
+ {
+ prefix = "";
+ }
+ String driverClassName = props.getProperty(prefix+"driverClassName");
+ String connectionUrl = props.getProperty(prefix+"connectionUrl");
+ String userName = props.getProperty(prefix+"userName");
+ String password = props.getProperty(prefix+"password");
+
+ Class.forName(driverClassName);
+ if (userName != null && password != null)
+ {
+ con = DriverManager.getConnection(connectionUrl, userName, password);
+ }
+ else
+ {
+ con = DriverManager.getConnection(connectionUrl);
+ }
+ return con;
+ }
+
+ /** Find and optionally delete all safely deletable orphans. */
+ public static void findAndDeleteOrphans(Connection con, boolean delete, boolean verbose)
+ throws SQLException
+ {
+ // websites with bad user?
+ findOrphans(con, "website", "userid", "rolleruser", delete, verbose);
+
+ // userroles with bad user?
+ findOrphans(con, "userrole", "userid", "rolleruser", delete, verbose);
+
+ // folders with bad website?
+ findOrphans(con, "folder", "websiteid", "website", delete, verbose);
+
+ // bookmarks with bad folder?
+ findOrphans(con, "bookmark", "folderid", "folder", delete, verbose);
+
+ // weblogcategories with bad website?
+ findOrphans(con, "weblogcategory", "websiteid", "website", delete, verbose);
+
+ // weblogcategoryassocs with bad category?
+ findOrphans(con, "weblogcategoryassoc", "categoryid", "weblogcategory", delete, verbose);
+
+ // weblog entries with bad website?
+ findOrphans(con, "weblogentry", "websiteid", "website", delete, verbose);
+
+ // comments with bad weblogentry?
+ findOrphans(con, "comment", "entryid", "weblogentry", delete, verbose);
+
+ // Referers with bad website?
+ findOrphans(con, "referer", "websiteid", "website", delete, verbose);
+
+ // Referers with bad website?
+ findOrphans(con, "referer", "entryid", "weblogentry", delete, verbose);
+
+ if (delete)
+ {
+ correctWeblogEntries(con);
+ correctWebsites(con);
+ correctFolderTrees(con, delete);
+ }
+ }
+
+ /**
+ * @param con
+ * @param delete
+ */
+ private static void correctFolderTrees(Connection con, boolean delete) throws SQLException
+ {
+ PreparedStatement rootStatement = con.prepareStatement(
+ "select a.id from folder as f, folderassoc as a where "+
+ "f.websiteid=? and f.id=a.folderid and "+
+ "a.relation='PARENT' and a.ancestorid is null");
+ PreparedStatement childrenStatement = con.prepareStatement(
+ "select id from folderassoc where ancestorid=?");
+
+ // loop through all websites
+ Statement websitesStatement = con.createStatement();
+ ResultSet websitesResultSet =
+ websitesStatement.executeQuery("select id from website");
+ while (websitesResultSet.next())
+ {
+ String websiteId = websitesResultSet.getString(1);
+ //debug("Website "+websiteId);
+
+ // find root folder(s)
+ List rootIds = new LinkedList();
+ rootStatement.clearParameters();
+ rootStatement.setString(1, websiteId);
+ ResultSet rootResultSet = rootStatement.executeQuery();
+ while (rootResultSet.next())
+ {
+ rootIds.add(rootResultSet.getString(1));
+ }
+ if (rootIds.size() > 1)
+ {
+ // too many roots, need to figure out which are bogus
+ Iterator rootIter = rootIds.iterator();
+ while (rootIter.hasNext())
+ {
+ String rootId = (String)rootIter.next();
+ childrenStatement.clearParameters();
+ childrenStatement.setString(1, rootId);
+ ResultSet childrenResultSet = childrenStatement.executeQuery();
+ List childIds = new LinkedList();
+ while (childrenResultSet.next())
+ {
+ childIds.add(childrenResultSet.getString(1));
+ }
+ if (childIds.size() == 0)
+ {
+ debug("Folder "+rootId+" in website "+websiteId+"is a bogus root folder!");
+ }
+ }
+ }
+ else if (rootIds.size() == 0)
+ {
+ debug("Website "+websiteId+" has no root folder!");
+ }
+ }
+ }
+
+ private static void correctWeblogEntries(Connection con) throws SQLException
+ {
+ List entries = findOrphans(con, "weblogentry", "categoryid", "weblogcategory", false, false);
+ Iterator entryIter = entries.iterator();
+ while (entryIter.hasNext())
+ {
+ String entryid = (String) entryIter.next();
+ Statement websiteSt = con.createStatement();
+ ResultSet websiteRs = websiteSt.executeQuery(
+ "select websiteid from weblogentry where id="+entryid);
+ websiteRs.first();
+ String websiteid = websiteRs.getString(0);
+
+ String rootid = getRootCategoryId(con, websiteid);
+ Statement st = con.createStatement();
+ st.executeUpdate("update weblogentry set categoryid='"+rootid+"' "
+ +" where id='"+entryid+"'");
+ }
+ }
+
+ public static void correctWebsites(Connection con) throws SQLException
+ {
+ List websites = findOrphans(con, "website", "defaultcatid", "weblogcategory", false, false);
+ Iterator websiteIter = websites.iterator();
+ while (websiteIter.hasNext())
+ {
+ String websiteid = (String) websiteIter.next();
+ String rootid = getRootCategoryId(con, websiteid);
+ Statement st = con.createStatement();
+ st.executeUpdate("update website set defaultcatid='"+rootid+"' "
+ +" where id='"+websiteid+"'");
+ }
+
+ websites = findOrphans(con, "website", "bloggercatid", "weblogcategory", false, false);
+ websiteIter = websites.iterator();
+ while (websiteIter.hasNext())
+ {
+ String websiteid = (String) websiteIter.next();
+ String rootid = getRootCategoryId(con, websiteid);
+ Statement st = con.createStatement();
+ st.executeUpdate("update website set bloggercatid='"+rootid+"' "
+ +"where id='"+websiteid+"'");
+ }
+ }
+
+ public static String getRootCategoryId(Connection con, String websiteid)
+ throws SQLException
+ {
+ Statement st = con.createStatement();
+ String query =
+ "select c.id from weblogcategory as c, weblogcategoryassoc as a "
+ +"where a.categoryid=c.id and a.ancestorid is null "
+ +"and c.websiteid ='"+websiteid+"'";
+ //System.out.println(query);
+ ResultSet rs = st.executeQuery(query);
+ rs.next();
+ return rs.getString(1);
+ }
+
+ /** Find orphans, records in a manytable that refer to a onetable that
+ * no longer exists.
+ * @param con Database connection to be used.
+ * @param manytable Name of the manytable.
+ * @param fkname Name of the foreign key field in the manytable.
+ * @param onetable Name of the onetable.
+ * @param delete True if orphans in manytable are to be deleted.
+ * @return List of orphans found (will be empty if delete is true.
+ * @throws SQLException
+ */
+ public static List findOrphans(
+ Connection con, String manytable, String fkname, String onetable, boolean delete, boolean verbose)
+ throws SQLException
+ {
+ List orphans = new LinkedList();
+
+ Statement stall = con.createStatement();
+ ResultSet rsall = stall.executeQuery(
+ "select id,"+fkname+" as fk from "+manytable);
+ while (rsall.next())
+ {
+ String id = rsall.getString("id");
+ String fk = rsall.getString("fk");
+ if (fk != null)
+ {
+ Statement stone = con.createStatement();
+ ResultSet rsone = stone.executeQuery(
+ "select id from "+onetable+" where id='"+fk+"' limit 1");
+ if (!rsone.next())
+ {
+ orphans.add(id);
+ System.out.println(" Found orphan in "+manytable+" id="+id);
+ }
+ }
+ }
+
+ if (!delete)
+ {
+ debug("Orphans found in "+manytable+" = "+orphans.size());
+ if (verbose)
+ {
+ Iterator iter = orphans.iterator();
+ while (iter.hasNext())
+ {
+ String id = (String) iter.next();
+ debug(" "+manytable+" id="+id);
+ }
+ }
+ }
+ else
+ {
+ debug("Deleting orphans found in "+manytable+" count = "+orphans.size());
+ Iterator iter = orphans.iterator();
+ while (iter.hasNext())
+ {
+ String id = (String) iter.next();
+ Statement stdel = con.createStatement();
+ stdel.executeUpdate("delete from "+manytable+" where id='"+id+"'");
+ }
+ orphans = new LinkedList();
+ }
+ return orphans;
+ }
+
+ private static void debug(String msg)
+ {
+ System.out.println("DEBUG: "+msg);
+ }
+}