You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@roller.apache.org by ag...@apache.org on 2007/07/03 01:41:13 UTC
svn commit: r552628 - in /roller/trunk/apps/planet:
src/java/org/apache/roller/planet/business/
src/java/org/apache/roller/planet/business/fetcher/
src/java/org/apache/roller/planet/business/hibernate/
src/java/org/apache/roller/planet/business/jpa/ sr...
Author: agilliland
Date: Mon Jul 2 16:41:11 2007
New Revision: 552628
URL: http://svn.apache.org/viewvc?view=rev&rev=552628
Log:
updating the Planet fetching and updating process for subscriptions to make it more layered and pluggable. major updates here include ...
1. Modified FeedFetcher interface to remove old monolithic refreshEntries() method and replace it with just a fetchSubscription(url) method.
2. Added a new FeedUpdater interface which acts as the feed update processor and utilizes the FeedFetcher in conjunction with the PlanetManager to update all the feed data. Making this a separate interface allows us to neatly partition logic and provide pluggable implementations based on different updating strategies such as single threaded vs. multi-threaded, etc.
Added:
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/FeedFetcher.java
- copied, changed from r552533, roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/FeedFetcher.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/FetcherException.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcher.java
- copied, changed from r552533, roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/RomeFeedFetcher.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/FeedUpdater.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/SingleThreadedFeedUpdater.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/UpdaterException.java
roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/SingleThreadedFeedUpdaterTest.java
Removed:
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/FeedFetcher.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/RomeFeedFetcher.java
Modified:
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/Planet.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetImpl.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetModule.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/jpa/JPAPlanetImpl.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/jpa/JPAPlanetModule.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/Subscription.hbm.xml
roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/Subscription.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/SubscriptionEntry.hbm.xml
roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/SubscriptionEntry.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/PlanetTask.java
roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/RefreshPlanetTask.java
roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetTestSuite.java
roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/RomeFeedFetcherTest.java
Modified: roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/Planet.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/Planet.java?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/Planet.java (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/Planet.java Mon Jul 2 16:41:11 2007
@@ -20,6 +20,7 @@
import org.apache.roller.planet.PlanetException;
import org.apache.roller.planet.business.PropertiesManager;
+import org.apache.roller.planet.business.fetcher.FeedFetcher;
/**
Copied: roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/FeedFetcher.java (from r552533, roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/FeedFetcher.java)
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/FeedFetcher.java?view=diff&rev=552628&p1=roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/FeedFetcher.java&r1=552533&p2=roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/FeedFetcher.java&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/FeedFetcher.java (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/FeedFetcher.java Mon Jul 2 16:41:11 2007
@@ -16,20 +16,35 @@
* directory of this distribution.
*/
-package org.apache.roller.planet.business;
+package org.apache.roller.planet.business.fetcher;
-import org.apache.roller.planet.PlanetException;
+import org.apache.roller.planet.pojos.Subscription;
/**
- * A FeedFetcher is what is responsible for actually fetching subscriptions from
- * their urls and persisting them to the db for the application to make use of.
+ * A FeedFetcher is what is responsible for actually pulling subscriptions from
+ * their source and transforming them into Roller Planet Subscriptions and Entries.
+ *
+ * It does not perform any persistence of feeds.
*/
public interface FeedFetcher {
- /**
- * Refresh all subscriptions by fetching and parsing feeds.
+ /**
+ * Fetch a single subscription.
+ *
+ * This method takes in a feed url and is expected to fetch that feed and
+ * return a transient instance of a Subscription representing the
+ * given feed.
+ *
+ * It is important to understand that this method will *NOT* return a
+ * persistent version of an existing Subscription if it happens to
+ * exist. This method is only here to pull feeds from their source
+ * so that they may be used in any way desired by the rest of the system.
+ *
+ * @param feedURL The feed url to use when fetching the subscription.
+ * @return Subscription The fetched subscription.
+ * @throws FetcherException If there is an error fetching the subscription.
*/
- public void refreshEntries(String cacheDirPath) throws PlanetException;
+ public Subscription fetchSubscription(String feedURL) throws FetcherException;
}
Added: roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/FetcherException.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/FetcherException.java?view=auto&rev=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/FetcherException.java (added)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/FetcherException.java Mon Jul 2 16:41:11 2007
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+
+package org.apache.roller.planet.business.fetcher;
+
+import org.apache.roller.planet.PlanetException;
+
+
+/**
+ * Exception generated from FeedFetcher.
+ */
+public class FetcherException extends PlanetException {
+
+ public FetcherException(String msg) {
+ super(msg);
+ }
+
+ public FetcherException(String msg, Throwable t) {
+ super(msg, t);
+ }
+
+}
Copied: roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcher.java (from r552533, roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/RomeFeedFetcher.java)
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcher.java?view=diff&rev=552628&p1=roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/RomeFeedFetcher.java&r1=552533&p2=roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcher.java&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/RomeFeedFetcher.java (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcher.java Mon Jul 2 16:41:11 2007
@@ -16,27 +16,30 @@
* directory of this distribution.
*/
-package org.apache.roller.planet.business;
+package org.apache.roller.planet.business.fetcher;
+import com.sun.syndication.feed.module.DCModule;
+import com.sun.syndication.feed.synd.SyndCategory;
+import com.sun.syndication.feed.synd.SyndContent;
import com.sun.syndication.feed.synd.SyndEntry;
import com.sun.syndication.feed.synd.SyndFeed;
+import com.sun.syndication.fetcher.FeedFetcher;
import com.sun.syndication.fetcher.impl.FeedFetcherCache;
import com.sun.syndication.fetcher.impl.HttpURLFeedFetcher;
import com.sun.syndication.fetcher.impl.SyndFeedInfo;
import java.io.File;
+import java.net.MalformedURLException;
import java.net.URL;
import java.sql.Timestamp;
-import java.text.MessageFormat;
+import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
-import java.util.Set;
-import java.util.TreeSet;
+import java.util.List;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.roller.planet.PlanetException;
-import org.apache.roller.planet.business.hibernate.HibernatePlanetManagerImpl;
-import org.apache.roller.planet.config.PlanetRuntimeConfig;
+import org.apache.roller.planet.config.PlanetConfig;
import org.apache.roller.planet.pojos.SubscriptionEntry;
import org.apache.roller.planet.pojos.Subscription;
import org.apache.roller.planet.util.rome.DiskFeedInfoCache;
@@ -45,26 +48,190 @@
/**
* A FeedFetcher based on the ROME RSS/Atom feed parser (http://rome.dev.java.net).
*/
-public class RomeFeedFetcher implements FeedFetcher {
+public class RomeFeedFetcher implements org.apache.roller.planet.business.fetcher.FeedFetcher {
private static Log log = LogFactory.getLog(RomeFeedFetcher.class);
- public RomeFeedFetcher() {}
+ public RomeFeedFetcher() {
+ // no-op
+ }
+
+
+ /**
+ * @inheritDoc
+ */
+ public Subscription fetchSubscription(String feedURL)
+ throws FetcherException {
+
+ if(feedURL == null) {
+ throw new IllegalArgumentException("feed url cannot be null");
+ }
+
+ // setup Rome feed fetcher
+ FeedFetcher feedFetcher = getRomeFetcher();
+
+ // fetch the feed
+ log.debug("Fetching feed: "+feedURL);
+ SyndFeed feed;
+ try {
+ feed = feedFetcher.retrieveFeed(new URL(feedURL));
+ } catch (Exception ex) {
+ throw new FetcherException("Error fetching subscription - "+feedURL, ex);
+ }
+
+ log.debug("Feed pulled, extracting data into Subscription");
+
+ // build planet subscription from fetched feed
+ Subscription newSub = new Subscription();
+ newSub.setFeedURL(feedURL);
+ newSub.setSiteURL(feed.getLink());
+ newSub.setTitle(feed.getTitle());
+ newSub.setAuthor(feed.getAuthor());
+ newSub.setLastUpdated(feed.getPublishedDate());
+
+
+ // normalize any data that couldn't be properly extracted
+ if(newSub.getSiteURL() == null) {
+ // set the site url to the feed url then
+ newSub.setSiteURL(newSub.getFeedURL());
+ }
+ if(newSub.getAuthor() == null) {
+ // set the author to the title
+ newSub.setAuthor(newSub.getTitle());
+ }
+ if(newSub.getLastUpdated() == null) {
+ // no update time specified in feed, so try consulting feed info cache
+ FeedFetcherCache feedCache = getRomeFetcherCache();
+ try {
+ SyndFeedInfo feedInfo = feedCache.getFeedInfo(new URL(newSub.getFeedURL()));
+ if(feedInfo.getLastModified() != null) {
+ long lastUpdatedLong = ((Long)feedInfo.getLastModified()).longValue();
+ if (lastUpdatedLong != 0) {
+ newSub.setLastUpdated(new Date(lastUpdatedLong));
+ }
+ }
+ } catch (MalformedURLException ex) {
+ // should never happen since we check this above
+ }
+ }
+
+ if(log.isDebugEnabled()) {
+ log.debug("Subscription is: "+newSub.toString());
+ }
+
+
+ // some kludge to deal with feeds w/ no entry dates
+ // we assign arbitrary dates chronologically by entry starting either
+ // from the current time or the last update time of the subscription
+ Calendar cal = Calendar.getInstance();
+ if (newSub.getLastUpdated() != null) {
+ cal.setTime(newSub.getLastUpdated());
+ } else {
+ cal.setTime(new Date());
+ cal.add(Calendar.DATE, -1);
+ }
+
+ // add entries
+ List<SyndEntry> feedEntries = feed.getEntries();
+ for( SyndEntry feedEntry : feedEntries ) {
+ SubscriptionEntry newEntry = buildEntry(feedEntry);
+
+ // some kludge to handle feeds with no entry dates
+ if (newEntry.getPubTime() == null) {
+ log.debug("No published date, assigning fake date for "+feedURL);
+ newEntry.setPubTime(new Timestamp(cal.getTimeInMillis()));
+ cal.add(Calendar.DATE, -1);
+ }
+
+ if(newEntry != null) {
+ newSub.addEntry(newEntry);
+ }
+ }
+
+ log.debug(feedEntries.size()+" entries included");
+
+ return newSub;
+ }
- // refresh Entries for all Subscriptions
- public void refreshEntries(String cacheDirPath) throws PlanetException {
+ // build a SubscriptionEntry from Rome SyndEntry and SyndFeed
+ private SubscriptionEntry buildEntry(SyndEntry romeEntry) {
+
+ // if we don't have a permalink then we can't continue
+ if(romeEntry.getLink() == null) {
+ return null;
+ }
- PlanetManager mgr = PlanetFactory.getPlanet().getPlanetManager();
+ SubscriptionEntry newEntry = new SubscriptionEntry();
+
+ newEntry.setTitle(romeEntry.getTitle());
+ newEntry.setPermalink(romeEntry.getLink());
+
+ // Play some games to get the author
+ DCModule entrydc = (DCModule)romeEntry.getModule(DCModule.URI);
+ if (romeEntry.getAuthor() != null) {
+ newEntry.setAuthor(romeEntry.getAuthor());
+ } else {
+ newEntry.setAuthor(entrydc.getCreator()); // use <dc:creator>
+ }
+
+ // Play some games to get the updated date
+ if (romeEntry.getUpdatedDate() != null) {
+ newEntry.setUpdateTime(new Timestamp(romeEntry.getUpdatedDate().getTime()));
+ }
+ // TODO: should we set a default update time here?
- Date now = new Date();
- long startTime = System.currentTimeMillis();
+ // And more games getting publish date
+ if (romeEntry.getPublishedDate() != null) {
+ newEntry.setPubTime(new Timestamp(romeEntry.getPublishedDate().getTime())); // use <pubDate>
+ } else if (entrydc != null && entrydc.getDate() != null) {
+ newEntry.setPubTime(new Timestamp(entrydc.getDate().getTime())); // use <dc:date>
+ } else {
+ newEntry.setPubTime(newEntry.getUpdateTime());
+ }
+
+ // get content and unescape if it is 'text/plain'
+ if (romeEntry.getContents().size() > 0) {
+ SyndContent content= (SyndContent)romeEntry.getContents().get(0);
+ if (content != null && content.getType().equals("text/plain")) {
+ newEntry.setText(StringEscapeUtils.unescapeHtml(content.getValue()));
+ } else if (content != null) {
+ newEntry.setText(content.getValue());
+ }
+ }
+
+ // no content, try summary
+ if (newEntry.getText() == null || newEntry.getText().trim().length() == 0) {
+ if (romeEntry.getDescription() != null) {
+ newEntry.setText(romeEntry.getDescription().getValue());
+ }
+ }
+
+ // copy categories
+ if (romeEntry.getCategories().size() > 0) {
+ List list = new ArrayList();
+ Iterator cats = romeEntry.getCategories().iterator();
+ while (cats.hasNext()) {
+ SyndCategory cat = (SyndCategory)cats.next();
+ list.add(cat.getName());
+ }
+ newEntry.setCategoriesString(list);
+ }
+
+ return newEntry;
+ }
+
+
+ // get a feed fetcher cache, if possible
+ private FeedFetcherCache getRomeFetcherCache() {
+
+ String cacheDirPath = PlanetConfig.getProperty("cache.dir");
// can't continue without cache dir
if (cacheDirPath == null) {
- log.warn("Planet cache directory not set, aborting refresh");
- return;
+ log.warn("Planet cache directory not set, feeds cannot be cached.");
+ return null;
}
// allow ${user.home} in cache dir property
@@ -74,7 +241,7 @@
// allow ${catalina.home} in cache dir property
if (System.getProperty("catalina.home") != null) {
cacheDirName = cacheDirName.replaceFirst(
- "\\$\\{catalina.home}",System.getProperty("catalina.home"));
+ "\\$\\{catalina.home}",System.getProperty("catalina.home"));
}
// create cache dir if it does not exist
@@ -84,155 +251,36 @@
if (!cacheDir.exists()) cacheDir.mkdirs();
} catch (Exception e) {
log.error("Unable to create planet cache directory: " + cacheDir.getPath(), e);
- return;
+ return null;
}
// abort if cache dir is not writable
if (!cacheDir.canWrite()) {
log.error("Planet cache directory is not writable: " + cacheDir.getPath());
- return;
+ return null;
}
- FeedFetcherCache feedInfoCache =
- new DiskFeedInfoCache(cacheDirName);
-
- String proxyHost = PlanetRuntimeConfig.getProperty("site.proxyhost");
- int proxyPort = PlanetRuntimeConfig.getIntProperty("site.proxyport");
- if (proxyHost != null && proxyPort > 0) {
- System.setProperty("proxySet", "true");
- System.setProperty("http.proxyHost", proxyHost);
- System.setProperty("http.proxyPort", Integer.toString(proxyPort));
- }
- /** a hack to set 15 sec timeouts for java.net.HttpURLConnection */
- System.setProperty("sun.net.client.defaultConnectTimeout", "15000");
- System.setProperty("sun.net.client.defaultReadTimeout", "15000");
-
- com.sun.syndication.fetcher.FeedFetcher feedFetcher =
- new HttpURLFeedFetcher(feedInfoCache);
- //FeedFetcher feedFetcher = new HttpClientFeedFetcher(feedInfoCache);
- feedFetcher.setUsingDeltaEncoding(false);
- feedFetcher.setUserAgent("RollerPlanetAggregator");
-
- // Loop through all subscriptions in the system
- Iterator subs = mgr.getSubscriptions().iterator();
- while (subs.hasNext()) {
-
- long subStartTime = System.currentTimeMillis();
-
- Subscription sub = (Subscription)subs.next();
-
- // reattach sub. sub gets detached as we iterate
- sub = mgr.getSubscriptionById(sub.getId());
-
- Set newEntries = this.getNewEntries(sub, feedFetcher, feedInfoCache);
- int count = newEntries.size();
-
- log.debug(" Entry count: " + count);
- if (count > 0) {
- mgr.deleteEntries(sub);
- sub.addEntries(newEntries);
- mgr.saveSubscription(sub);
- }
-
- long subEndTime = System.currentTimeMillis();
- log.debug(" " + count + " - "
- + ((subEndTime-subStartTime)/1000.0)
- + " seconds to process (" + count + ") entries of "
- + sub.getFeedURL());
- }
-
- long endTime = System.currentTimeMillis();
- log.info("--- DONE --- Refreshed entries in "
- + ((endTime-startTime)/1000.0) + " seconds");
+ return new DiskFeedInfoCache(cacheDirName);
}
-
-
- // get new Entries for a specific Subscription
- protected Set getNewEntries(Subscription sub,
- com.sun.syndication.fetcher.FeedFetcher feedFetcher,
- FeedFetcherCache feedInfoCache)
- throws PlanetException {
-
- Set newEntries = new TreeSet();
- SyndFeed feed = null;
- URL feedURL = null;
- Date lastUpdated = new Date();
- try {
- feedURL = new URL(sub.getFeedURL());
- log.debug("Get feed from cache "+sub.getFeedURL());
- feed = feedFetcher.retrieveFeed(feedURL);
- SyndFeedInfo feedInfo = feedInfoCache.getFeedInfo(feedURL);
- if (feedInfo.getLastModified() != null) {
- long lastUpdatedLong =
- ((Long)feedInfo.getLastModified()).longValue();
- if (lastUpdatedLong != 0) {
- lastUpdated = new Date(lastUpdatedLong);
- }
- }
- Thread.sleep(100); // be nice
- } catch (Exception e) {
- log.warn("ERROR parsing " + sub.getFeedURL()
- + " : " + e.getClass().getName() + " : " + e.getMessage());
- log.debug(e);
- return newEntries; // bail out
- }
- if (lastUpdated!=null && sub.getLastUpdated()!=null) {
- Calendar feedCal = Calendar.getInstance();
- feedCal.setTime(lastUpdated);
-
- Calendar subCal = Calendar.getInstance();
- subCal.setTime(sub.getLastUpdated());
-
- if (!feedCal.after(subCal)) {
- if (log.isDebugEnabled()) {
- String msg = MessageFormat.format(
- " Skipping ({0} / {1})",
- new Object[] {
- lastUpdated, sub.getLastUpdated()});
- log.debug(msg);
- }
- return newEntries; // bail out
- }
- }
- if (feed.getPublishedDate() != null) {
- sub.setLastUpdated(feed.getPublishedDate());
- // saving sub here causes detachment issues, so we save it later
- }
+
+
+ // get a feed fetcher
+ private FeedFetcher getRomeFetcher() {
- // Horrible kludge for Feeds without entry dates: most recent entry is
- // given feed's last publish date (or yesterday if none exists) and
- // earler entries are placed at once day intervals before that.
- Calendar cal = Calendar.getInstance();
- if (sub.getLastUpdated() != null) {
- cal.setTime(sub.getLastUpdated());
+ FeedFetcherCache feedCache = getRomeFetcherCache();
+
+ FeedFetcher feedFetcher = null;
+ if(feedCache != null) {
+ feedFetcher = new HttpURLFeedFetcher(feedCache);
} else {
- cal.setTime(new Date());
- cal.add(Calendar.DATE, -1);
+ feedFetcher = new HttpURLFeedFetcher();
}
- // Populate subscription object with new entries
- Iterator entries = feed.getEntries().iterator();
- while (entries.hasNext()) {
- try {
- SyndEntry romeEntry = (SyndEntry) entries.next();
- SubscriptionEntry entry =
- new SubscriptionEntry(feed, romeEntry, sub);
- log.debug("Entry title=" + entry.getTitle() + " content size=" + entry.getContent().length());
- if (entry.getPubTime() == null) {
- log.debug("No published date, assigning fake date for "+feedURL);
- entry.setPubTime(new Timestamp(cal.getTimeInMillis()));
- }
- if (entry.getPermalink() == null) {
- log.warn("No permalink, rejecting entry from "+feedURL);
- } else {
- newEntries.add(entry);
- }
- cal.add(Calendar.DATE, -1);
- } catch (Exception e) {
- log.error("ERROR processing subscription entry", e);
- }
- }
- return newEntries;
+ // set options
+ feedFetcher.setUsingDeltaEncoding(false);
+ feedFetcher.setUserAgent("RollerPlanetAggregator");
+
+ return feedFetcher;
}
}
Modified: roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetImpl.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetImpl.java?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetImpl.java (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetImpl.java Mon Jul 2 16:41:11 2007
@@ -22,7 +22,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.roller.planet.PlanetException;
import org.apache.roller.planet.business.AbstractManagerImpl;
-import org.apache.roller.planet.business.FeedFetcher;
+import org.apache.roller.planet.business.fetcher.FeedFetcher;
import org.apache.roller.planet.business.InitializationException;
import org.apache.roller.planet.business.Planet;
import org.apache.roller.planet.business.PlanetManager;
Modified: roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetModule.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetModule.java?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetModule.java (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetModule.java Mon Jul 2 16:41:11 2007
@@ -20,12 +20,12 @@
import com.google.inject.Binder;
import com.google.inject.Module;
-import org.apache.roller.planet.business.FeedFetcher;
+import org.apache.roller.planet.business.fetcher.FeedFetcher;
import org.apache.roller.planet.business.MultiPlanetURLStrategy;
import org.apache.roller.planet.business.Planet;
import org.apache.roller.planet.business.PlanetManager;
import org.apache.roller.planet.business.PropertiesManager;
-import org.apache.roller.planet.business.RomeFeedFetcher;
+import org.apache.roller.planet.business.fetcher.RomeFeedFetcher;
import org.apache.roller.planet.business.URLStrategy;
Modified: roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/jpa/JPAPlanetImpl.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/jpa/JPAPlanetImpl.java?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/jpa/JPAPlanetImpl.java (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/jpa/JPAPlanetImpl.java Mon Jul 2 16:41:11 2007
@@ -25,7 +25,7 @@
import org.apache.roller.planet.business.URLStrategy;
import org.apache.roller.planet.business.Planet;
import org.apache.roller.planet.business.PlanetManager;
-import org.apache.roller.planet.business.FeedFetcher;
+import org.apache.roller.planet.business.fetcher.FeedFetcher;
import org.apache.roller.planet.business.InitializationException;
import org.apache.roller.planet.business.PropertiesManager;
Modified: roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/jpa/JPAPlanetModule.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/jpa/JPAPlanetModule.java?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/jpa/JPAPlanetModule.java (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/jpa/JPAPlanetModule.java Mon Jul 2 16:41:11 2007
@@ -20,12 +20,12 @@
import com.google.inject.Binder;
import com.google.inject.Module;
-import org.apache.roller.planet.business.FeedFetcher;
+import org.apache.roller.planet.business.fetcher.FeedFetcher;
import org.apache.roller.planet.business.MultiPlanetURLStrategy;
import org.apache.roller.planet.business.Planet;
import org.apache.roller.planet.business.PlanetManager;
import org.apache.roller.planet.business.PropertiesManager;
-import org.apache.roller.planet.business.RomeFeedFetcher;
+import org.apache.roller.planet.business.fetcher.RomeFeedFetcher;
import org.apache.roller.planet.business.URLStrategy;
Added: roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/FeedUpdater.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/FeedUpdater.java?view=auto&rev=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/FeedUpdater.java (added)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/FeedUpdater.java Mon Jul 2 16:41:11 2007
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+
+package org.apache.roller.planet.business.updater;
+
+import org.apache.roller.planet.pojos.PlanetGroup;
+import org.apache.roller.planet.pojos.Subscription;
+
+
+/**
+ * A FeedUpdater is responsible for processing the updates of all Subscriptions
+ * and their entries. It is intended to combine the use of the FeedFetcher for
+ * pulling fresh feed data with the PlanetManager for updating and persisting
+ * the updated data.
+ *
+ * NOTE: it must be explicitly stated that the operations of the FeedUpdater are
+ * *not* considered atomic and they are *not* guaranteed to happen synchronously.
+ * So callers of these methods should bear that in mind when using this class.
+ */
+public interface FeedUpdater {
+
+ /**
+ * Update a single Subscription.
+ *
+ * This method takes in an existing Subscription and updates it with
+ * the data from the subscriptions source after fetching an updated version
+ * of the subscription.
+ *
+ * @param subscription The PlanetSubscription to be updated.
+ * @throws FetcherException If there is an error updating the subscription.
+ */
+ public void updateSubscription(Subscription sub) throws UpdaterException;
+
+
+ /**
+ * Update all Subscriptions in the system.
+ *
+ * @throws UpdaterException If there is an error during the update and the operation cannot continue.
+ */
+ public void updateSubscriptions() throws UpdaterException;
+
+
+ /**
+ * Update all Subscriptions that are part of the specified group.
+ *
+ * @throws UpdaterException If there is an error during the update and the operation cannot continue.
+ */
+ public void updateSubscriptions(PlanetGroup group) throws UpdaterException;
+
+}
Added: roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/SingleThreadedFeedUpdater.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/SingleThreadedFeedUpdater.java?view=auto&rev=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/SingleThreadedFeedUpdater.java (added)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/SingleThreadedFeedUpdater.java Mon Jul 2 16:41:11 2007
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+
+package org.apache.roller.planet.business.updater;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Set;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.roller.planet.PlanetException;
+import org.apache.roller.planet.business.PlanetFactory;
+import org.apache.roller.planet.business.PlanetManager;
+import org.apache.roller.planet.business.fetcher.FeedFetcher;
+import org.apache.roller.planet.business.fetcher.FetcherException;
+import org.apache.roller.planet.config.PlanetRuntimeConfig;
+import org.apache.roller.planet.pojos.PlanetGroup;
+import org.apache.roller.planet.pojos.Subscription;
+import org.apache.roller.planet.pojos.SubscriptionEntry;
+
+
+/**
+ * A single threaded implementation of a FeedUpdater.
+ */
+public class SingleThreadedFeedUpdater implements FeedUpdater {
+
+ private static Log log = LogFactory.getLog(SingleThreadedFeedUpdater.class);
+
+
+ public SingleThreadedFeedUpdater() {
+ // no-op
+ }
+
+
+ /**
+ * @inheritDoc
+ */
+ public void updateSubscription(Subscription sub) throws UpdaterException {
+
+ if(sub == null) {
+ throw new IllegalArgumentException("cannot update null subscription");
+ }
+
+ updateProxySettings();
+
+ log.debug("updating feed: "+sub.getFeedURL());
+
+ long subStartTime = System.currentTimeMillis();
+
+ Subscription updatedSub;
+ try {
+ // fetch the latest version of the subscription
+ FeedFetcher fetcher = PlanetFactory.getPlanet().getFeedFetcher();
+ updatedSub = fetcher.fetchSubscription(sub.getFeedURL());
+ } catch (FetcherException ex) {
+ throw new UpdaterException("Error fetching updated subscription", ex);
+ }
+
+ // if this subscription hasn't changed since last update then we're done
+ if(sub.getLastUpdated() != null &&
+ !updatedSub.getLastUpdated().after(sub.getLastUpdated())) {
+ log.debug("Skipping update, feed hasn't changed - "+sub.getFeedURL());
+ }
+
+ // update subscription attributes
+ sub.setSiteURL(updatedSub.getSiteURL());
+ sub.setTitle(updatedSub.getTitle());
+ sub.setAuthor(updatedSub.getAuthor());
+ sub.setLastUpdated(updatedSub.getLastUpdated());
+
+ // update subscription entries
+ int entries = 0;
+ Set<SubscriptionEntry> newEntries = updatedSub.getEntries();
+ if(newEntries.size() > 0) try {
+ PlanetManager pmgr = PlanetFactory.getPlanet().getPlanetManager();
+
+ // clear out old entries
+ pmgr.deleteEntries(sub);
+
+ // add fresh entries
+ sub.getEntries().clear();
+ sub.addEntries(newEntries);
+
+ // save and flush
+ pmgr.saveSubscription(sub);
+ PlanetFactory.getPlanet().flush();
+
+ } catch(PlanetException ex) {
+ throw new UpdaterException("Error persisting updated subscription", ex);
+ }
+
+ long subEndTime = System.currentTimeMillis();
+ log.debug("updated feed -- "+sub.getFeedURL()+" -- in "+
+ ((subEndTime-subStartTime)/1000.0)+" seconds. "+entries+
+ " entries updated.");
+ }
+
+
+ /**
+ * @inheritDoc
+ */
+ public void updateSubscriptions() throws UpdaterException {
+
+ updateProxySettings();
+
+ log.debug("--- BEGIN --- Updating all subscriptions");
+
+ long startTime = System.currentTimeMillis();
+
+ try {
+ // update all subscriptions in the system
+ PlanetManager pmgr = PlanetFactory.getPlanet().getPlanetManager();
+ updateSubscriptions(pmgr.getSubscriptions());
+ } catch (PlanetException ex) {
+ throw new UpdaterException("Error getting subscriptions list", ex);
+ }
+
+ long endTime = System.currentTimeMillis();
+ log.info("--- DONE --- Updated subscriptions in "
+ + ((endTime-startTime)/1000.0) + " seconds");
+ }
+
+
+ /**
+ * @inheritDoc
+ */
+ public void updateSubscriptions(PlanetGroup group) throws UpdaterException {
+
+ if(group == null) {
+ throw new IllegalArgumentException("cannot update null group");
+ }
+
+ updateProxySettings();
+
+ log.debug("--- BEGIN --- Updating subscriptions in group = "+group.getHandle());
+
+ long startTime = System.currentTimeMillis();
+
+ updateSubscriptions(group.getSubscriptions());
+
+ long endTime = System.currentTimeMillis();
+ log.info("--- DONE --- Updated subscriptions in "
+ + ((endTime-startTime)/1000.0) + " seconds");
+ }
+
+
+ // convenience method which handles updating any arbitrary collection of subs
+ private void updateSubscriptions(Collection<Subscription> subscriptions) {
+
+ PlanetManager pmgr = PlanetFactory.getPlanet().getPlanetManager();
+
+ Iterator subs = subscriptions.iterator();
+ while (subs.hasNext()) {
+ Subscription sub = (Subscription)subs.next();
+
+ try {
+ // reattach sub. sub gets detached as we iterate
+ sub = pmgr.getSubscriptionById(sub.getId());
+ } catch (PlanetException ex) {
+ log.warn("Subscription went missing while doing update: "+ex.getMessage());
+ }
+
+ // this updates and saves
+ try {
+ updateSubscription(sub);
+ } catch(Exception ex) {
+ log.warn("Error updating subscription - "+sub.getFeedURL(), ex);
+ }
+ }
+ }
+
+
+ // upate proxy settings for jvm based on planet configuration
+ private void updateProxySettings() {
+ String proxyHost = PlanetRuntimeConfig.getProperty("site.proxyhost");
+ int proxyPort = PlanetRuntimeConfig.getIntProperty("site.proxyport");
+ if (proxyHost != null && proxyPort > 0) {
+ System.setProperty("proxySet", "true");
+ System.setProperty("http.proxyHost", proxyHost);
+ System.setProperty("http.proxyPort", Integer.toString(proxyPort));
+ }
+ /** a hack to set 15 sec timeouts for java.net.HttpURLConnection */
+ System.setProperty("sun.net.client.defaultConnectTimeout", "15000");
+ System.setProperty("sun.net.client.defaultReadTimeout", "15000");
+ }
+
+}
Added: roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/UpdaterException.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/UpdaterException.java?view=auto&rev=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/UpdaterException.java (added)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/updater/UpdaterException.java Mon Jul 2 16:41:11 2007
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+
+package org.apache.roller.planet.business.updater;
+
+import org.apache.roller.planet.PlanetException;
+
+
+/**
+ * Exception generated from FeedUpdater.
+ */
+public class UpdaterException extends PlanetException {
+
+ public UpdaterException(String msg) {
+ super(msg);
+ }
+
+ public UpdaterException(String msg, Throwable t) {
+ super(msg, t);
+ }
+
+}
Modified: roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/Subscription.hbm.xml
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/Subscription.hbm.xml?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/Subscription.hbm.xml (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/Subscription.hbm.xml Mon Jul 2 16:41:11 2007
@@ -76,20 +76,16 @@
unique="false"
/>
- <set name="groups" table="rag_group_subscription"
- lazy="true" cascade="none" sort="unsorted">
-
+ <set name="groups" table="rag_group_subscription" lazy="true" inverse="true" cascade="none" sort="unsorted">
<key column="subscription_id" />
- <many-to-many class="org.apache.roller.planet.pojos.PlanetGroup"
- column="group_id" outer-join="auto" />
-
+ <many-to-many class="org.apache.roller.planet.pojos.PlanetGroup" column="group_id" outer-join="auto" />
</set>
-
- <bag name="entries" lazy="true" inverse="true" cascade="all">
+
+ <set name="entries" table="rag_entry" lazy="true" inverse="true" cascade="all">
<key column="subscription_id" />
<one-to-many class="org.apache.roller.planet.pojos.SubscriptionEntry" />
- </bag>
-
+ </set>
+
</class>
</hibernate-mapping>
Modified: roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/Subscription.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/Subscription.java?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/Subscription.java (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/Subscription.java Mon Jul 2 16:41:11 2007
@@ -17,20 +17,16 @@
package org.apache.roller.planet.pojos;
import java.io.Serializable;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.List;
import java.util.Set;
import org.apache.roller.util.UUIDGenerator;
/**
* Planet Subscription.
- *
- * @hibernate.class lazy="true" table="rag_subscription"
*/
public class Subscription implements Serializable, Comparable {
@@ -46,7 +42,7 @@
// associations
private Set groups = new HashSet();
- private List entries = new ArrayList();
+ private Set entries = new HashSet();
public Subscription() {}
@@ -70,10 +66,22 @@
return this.feedUrl.hashCode();
}
+ public String toString() {
+ StringBuffer buf = new StringBuffer();
+
+ buf.append("{");
+ buf.append(feedUrl).append(", ");
+ buf.append(siteUrl).append(", ");
+ buf.append(title).append(", ");
+ buf.append(author).append(", ");
+ buf.append(lastUpdated);
+ buf.append("}");
+
+ return buf.toString();
+
+ }
+
- /**
- * @hibernate.id column="id" generator-class="assigned"
- */
public String getId() {
return id;
}
@@ -83,9 +91,6 @@
}
- /**
- * @hibernate.property column="title" non-null="false" unique="false"
- */
public String getTitle() {
return title;
}
@@ -95,9 +100,6 @@
}
- /**
- * @hibernate.property column="author" non-null="false" unique="false"
- */
public String getAuthor() {
return author;
}
@@ -107,9 +109,6 @@
}
- /**
- * @hibernate.property column="feed_url" non-null="true" unique="false"
- */
public String getFeedURL() {
return feedUrl;
}
@@ -119,9 +118,6 @@
}
- /**
- * @hibernate.property column="site_url" non-null="false" unique="false"
- */
public String getSiteURL() {
return siteUrl;
}
@@ -131,9 +127,6 @@
}
- /**
- * @hibernate.property column="last_updated" non-null="false" unique="false"
- */
public Date getLastUpdated() {
return lastUpdated;
}
@@ -143,9 +136,6 @@
}
- /**
- * @hibernate.property column="inbound_links" non-null="false" unique="false"
- */
public int getInboundlinks() {
return inboundlinks;
}
@@ -155,9 +145,6 @@
}
- /**
- * @hibernate.property column="inbound_blogs" non-null="false" unique="false"
- */
public int getInboundblogs() {
return inboundblogs;
}
@@ -167,11 +154,6 @@
}
- /**
- * @hibernate.set table="rag_group_subscription" lazy="true" cascade="none"
- * @hibernate.collection-key column="subscription_id"
- * @hibernate.collection-many-to-many column="group_id" class="org.apache.roller.planet.pojos.PlanetGroup"
- */
public Set getGroups() {
return groups;
}
@@ -182,27 +164,28 @@
}
- /**
- * @hibernate.bag lazy="true" inverse="true" cascade="all"
- * @hibernate.collection-key column="subscription_id"
- * @hibernate.collection-one-to-many class="org.apache.roller.planet.pojos.SubscriptionEntry"
- */
- public List getEntries() {
+ public Set getEntries() {
return entries;
}
// private because there is no need for people to do this
- private void setEntries(List entries) {
+ private void setEntries(Set entries) {
this.entries = entries;
}
+ /**
+ * Add a SubscriptionEntry to this Subscription.
+ */
public void addEntry(SubscriptionEntry entry) {
// bi-directional one-to-many
entry.setSubscription(this);
this.getEntries().add(entry);
}
+ /**
+ * Add a collection of SubscriptionEntry to this Subscription.
+ */
public void addEntries(Collection newEntries) {
// bi-directional one-to-many
for (Iterator it = newEntries.iterator(); it.hasNext();) {
Modified: roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/SubscriptionEntry.hbm.xml
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/SubscriptionEntry.hbm.xml?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/SubscriptionEntry.hbm.xml (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/SubscriptionEntry.hbm.xml Mon Jul 2 16:41:11 2007
@@ -97,7 +97,7 @@
<many-to-one
name="subscription"
class="org.apache.roller.planet.pojos.Subscription"
- cascade="save-update"
+ cascade="none"
outer-join="auto"
update="true"
insert="true"
Modified: roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/SubscriptionEntry.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/SubscriptionEntry.java?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/SubscriptionEntry.java (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/pojos/SubscriptionEntry.java Mon Jul 2 16:41:11 2007
@@ -23,14 +23,7 @@
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-
import org.apache.roller.planet.util.Utilities;
-import com.sun.syndication.feed.module.DCModule;
-import com.sun.syndication.feed.synd.SyndCategory;
-import com.sun.syndication.feed.synd.SyndContent;
-import com.sun.syndication.feed.synd.SyndEntry;
-import com.sun.syndication.feed.synd.SyndFeed;
-import org.apache.commons.lang.StringEscapeUtils;
import org.apache.roller.util.UUIDGenerator;
@@ -41,8 +34,6 @@
* The model coded in this class simple, perhaps too simple, and in the future
* it should be replaced by more complete model that can fully represent all
* forms of RSS and Atom.
- *
- * @hibernate.class lazy="true" table="rag_entry"
*/
public class SubscriptionEntry implements Serializable, Comparable {
@@ -64,73 +55,6 @@
public SubscriptionEntry() {}
- /**
- * Create entry from Rome entry.
- */
- public SubscriptionEntry(
- SyndFeed romeFeed, SyndEntry romeEntry, Subscription sub) {
- setSubscription(sub);
- initFromRomeEntry(romeFeed, romeEntry);
- }
-
-
- /**
- * Init entry from Rome entry
- */
- private void initFromRomeEntry(SyndFeed romeFeed, SyndEntry romeEntry) {
- setTitle(romeEntry.getTitle());
- setPermalink(romeEntry.getLink());
-
- // Play some games to get the author
- DCModule entrydc = (DCModule)romeEntry.getModule(DCModule.URI);
- DCModule feeddc = (DCModule)romeFeed.getModule(DCModule.URI);
- if (romeEntry.getAuthor() != null) {
- setAuthor(romeEntry.getAuthor());
- } else {
- setAuthor(entrydc.getCreator()); // use <dc:creator>
- }
-
- // Play some games to get the published date too
- if (romeEntry.getUpdatedDate() != null) {
- setUpdateTime(new Timestamp(romeEntry.getUpdatedDate().getTime()));
- }
- if (romeEntry.getPublishedDate() != null) {
- setPubTime(new Timestamp(romeEntry.getPublishedDate().getTime())); // use <pubDate>
- } else if (entrydc != null && entrydc.getDate() != null) {
- setPubTime(new Timestamp(entrydc.getDate().getTime())); // use <dc:date>
- } else {
- setPubTime(getUpdateTime());
- }
-
- // get content and unescape if it is 'text/plain'
- if (romeEntry.getContents().size() > 0) {
- SyndContent content= (SyndContent)romeEntry.getContents().get(0);
- if (content != null && content.getType().equals("text/plain")) {
- setText(StringEscapeUtils.unescapeHtml(content.getValue()));
- } else if (content != null) {
- setText(content.getValue());
- }
- }
-
- // no content, try summary
- if (getText() == null || getText().trim().length() == 0) {
- if (romeEntry.getDescription() != null) {
- setText(romeEntry.getDescription().getValue());
- }
- }
-
- // copy categories
- if (romeEntry.getCategories().size() > 0) {
- List list = new ArrayList();
- Iterator cats = romeEntry.getCategories().iterator();
- while (cats.hasNext()) {
- SyndCategory cat = (SyndCategory)cats.next();
- list.add(cat.getName());
- }
- setCategoriesString(list);
- }
- }
-
/**
* Compare planet entries by comparing permalinks.
@@ -158,9 +82,6 @@
}
- /**
- * @hibernate.id column="id" generator-class="assigned"
- */
public String getId() {
return id;
}
@@ -170,9 +91,6 @@
}
- /**
- * @hibernate.property column="handle" non-null="false" unique="false"
- */
public String getHandle() {
return handle;
}
@@ -182,9 +100,6 @@
}
- /**
- * @hibernate.property column="title" non-null="false" unique="false"
- */
public String getTitle() {
return title;
}
@@ -194,9 +109,6 @@
}
- /**
- * @hibernate.property column="guid" non-null="false" unique="true"
- */
public String getGuid() {
return guid;
}
@@ -206,9 +118,6 @@
}
- /**
- * @hibernate.property column="permalink" non-null="true" unique="false"
- */
public String getPermalink() {
return permalink;
}
@@ -218,9 +127,6 @@
}
- /**
- * @hibernate.property column="author" non-null="false" unique="false"
- */
public String getAuthor() {
return author;
}
@@ -230,9 +136,6 @@
}
- /**
- * @hibernate.property column="content" non-null="false" unique="false"
- */
public String getText() {
return text;
}
@@ -242,9 +145,6 @@
}
- /**
- * @hibernate.property column="published" non-null="true" unique="false"
- */
public Timestamp getPubTime() {
return published;
}
@@ -254,9 +154,6 @@
}
- /**
- * @hibernate.property column="updated" non-null="false" unique="false"
- */
public Timestamp getUpdateTime() {
return updated;
}
@@ -266,9 +163,6 @@
}
- /**
- * @hibernate.property column="categories" non-null="false" unique="false"
- */
public String getCategoriesString() {
return categoriesString;
}
@@ -278,9 +172,6 @@
}
- /**
- * @hibernate.many-to-one column="subscription_id" cascade="save-update" not-null="true"
- */
public Subscription getSubscription() {
return subscription;
}
@@ -339,7 +230,7 @@
return cat;
}
- private void setCategoriesString(List categories) {
+ public void setCategoriesString(List categories) {
StringBuffer sb = new StringBuffer();
Iterator cats = categories.iterator();
while (cats.hasNext()) {
Modified: roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/PlanetTask.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/PlanetTask.java?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/PlanetTask.java (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/PlanetTask.java Mon Jul 2 16:41:11 2007
@@ -18,7 +18,7 @@
package org.apache.roller.planet.tasks;
-import org.apache.roller.planet.business.FeedFetcher;
+import org.apache.roller.planet.business.fetcher.FeedFetcher;
import org.apache.roller.planet.business.PlanetFactory;
import org.apache.roller.planet.business.URLStrategy;
import org.apache.roller.planet.config.PlanetConfig;
Modified: roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/RefreshPlanetTask.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/RefreshPlanetTask.java?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/RefreshPlanetTask.java (original)
+++ roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/RefreshPlanetTask.java Mon Jul 2 16:41:11 2007
@@ -20,10 +20,10 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.roller.planet.PlanetException;
-import org.apache.roller.planet.config.PlanetConfig;
-import org.apache.roller.planet.business.Planet;
import org.apache.roller.planet.business.PlanetFactory;
+import org.apache.roller.planet.business.startup.PlanetStartup;
+import org.apache.roller.planet.business.updater.FeedUpdater;
+import org.apache.roller.planet.business.updater.SingleThreadedFeedUpdater;
/**
@@ -42,21 +42,25 @@
try {
// Update all feeds in planet
log.info("Refreshing Planet entries");
- Planet planet = PlanetFactory.getPlanet();
- planet.getFeedFetcher().refreshEntries(
- PlanetConfig.getProperty("cache.dir"));
- planet.flush();
- planet.release();
+ FeedUpdater updater = new SingleThreadedFeedUpdater();
+ updater.updateSubscriptions();
+ PlanetFactory.getPlanet().release();
- } catch (PlanetException e) {
+ } catch (Exception e) {
log.error("ERROR refreshing planet", e);
}
}
- public static void main(String[] args) throws Exception{
+ public static void main(String[] args) throws Exception {
+
RefreshPlanetTask task = new RefreshPlanetTask();
task.initialize();
+
+ // need to prepare and bootstrap Planet as well
+ PlanetStartup.prepare();
+ PlanetFactory.bootstrap();
+
task.run();
}
Modified: roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetTestSuite.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetTestSuite.java?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetTestSuite.java (original)
+++ roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetTestSuite.java Mon Jul 2 16:41:11 2007
@@ -59,8 +59,12 @@
suite.addTestSuite(EntryBasicTests.class);
suite.addTestSuite(EntryFunctionalTests.class);
+ // fetching
suite.addTestSuite(RomeFeedFetcherTest.class);
-
+
+ // updating
+ suite.addTestSuite(SingleThreadedFeedUpdaterTest.class);
+
return suite;
}
Modified: roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/RomeFeedFetcherTest.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/RomeFeedFetcherTest.java?view=diff&rev=552628&r1=552627&r2=552628
==============================================================================
--- roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/RomeFeedFetcherTest.java (original)
+++ roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/RomeFeedFetcherTest.java Mon Jul 2 16:41:11 2007
@@ -21,6 +21,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.roller.planet.TestUtils;
+import org.apache.roller.planet.business.fetcher.FeedFetcher;
import org.apache.roller.planet.pojos.Planet;
import org.apache.roller.planet.pojos.PlanetGroup;
import org.apache.roller.planet.pojos.Subscription;
@@ -31,50 +32,32 @@
*/
public class RomeFeedFetcherTest extends TestCase {
- public static Log log = LogFactory.getLog(RomeFeedFetcherTest.class);
+ public static Log log = LogFactory.getLog(RomeFeedFetcherTest.class);
- private Planet testPlanet = null;
- private PlanetGroup testGroup = null;
- private Subscription testSub = null;
-
- String feed_url = "http://rollerweblogger.org/roller/feed/entries/rss";
+ String feed_url = "http://rollerweblogger.org/roller/feed/entries/atom";
protected void setUp() throws Exception {
// setup planet
TestUtils.setupPlanet();
-
- testPlanet = TestUtils.setupPlanet("fetcherTestPlanet");
- testGroup = TestUtils.setupGroup(testPlanet, "fetcherTestGroup");
-
- // add test subscription
- PlanetManager mgr = PlanetFactory.getPlanet().getPlanetManager();
- testSub = new Subscription();
- testSub.setTitle(feed_url);
- testSub.setFeedURL(feed_url);
- mgr.saveSubscription(testSub);
- PlanetFactory.getPlanet().flush();
}
protected void tearDown() throws Exception {
- TestUtils.teardownSubscription(testSub.getId());
- TestUtils.teardownGroup(testGroup.getId());
- TestUtils.teardownPlanet(testPlanet.getId());
}
- public void testRefreshEntries() throws Exception {
+ public void testFetchFeed() throws Exception {
- PlanetManager mgr = PlanetFactory.getPlanet().getPlanetManager();
FeedFetcher feedFetcher = PlanetFactory.getPlanet().getFeedFetcher();
- // refresh entries
- feedFetcher.refreshEntries("." + File.separator + "planet-cache");
- TestUtils.endSession(true);
-
- Subscription sub = mgr.getSubscription(feed_url);
+ // fetch feed
+ Subscription sub = feedFetcher.fetchSubscription(feed_url);
assertNotNull(sub);
+ assertEquals(feed_url, sub.getFeedURL());
+ assertEquals("http://rollerweblogger.org/roller/", sub.getSiteURL());
+ assertEquals("Blogging Roller", sub.getTitle());
+ assertNotNull(sub.getLastUpdated());
assertTrue(sub.getEntries().size() > 0);
}
Added: roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/SingleThreadedFeedUpdaterTest.java
URL: http://svn.apache.org/viewvc/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/SingleThreadedFeedUpdaterTest.java?view=auto&rev=552628
==============================================================================
--- roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/SingleThreadedFeedUpdaterTest.java (added)
+++ roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/SingleThreadedFeedUpdaterTest.java Mon Jul 2 16:41:11 2007
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2005 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.roller.planet.business;
+
+import junit.framework.TestCase;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.roller.planet.TestUtils;
+import org.apache.roller.planet.business.fetcher.FeedFetcher;
+import org.apache.roller.planet.business.updater.FeedUpdater;
+import org.apache.roller.planet.business.updater.SingleThreadedFeedUpdater;
+import org.apache.roller.planet.pojos.Subscription;
+
+
+/**
+ * Test feed updater.
+ */
+public class SingleThreadedFeedUpdaterTest extends TestCase {
+
+ public static Log log = LogFactory.getLog(SingleThreadedFeedUpdaterTest.class);
+
+ private Subscription testSub = null;
+
+ private String feed_url = "http://rollerweblogger.org/roller/feed/entries/atom";
+
+
+ protected void setUp() throws Exception {
+ // setup planet
+ TestUtils.setupPlanet();
+
+ // add test subscription
+ PlanetManager mgr = PlanetFactory.getPlanet().getPlanetManager();
+ testSub = new Subscription();
+ testSub.setTitle(feed_url);
+ testSub.setFeedURL(feed_url);
+ mgr.saveSubscription(testSub);
+ PlanetFactory.getPlanet().flush();
+ }
+
+
+ protected void tearDown() throws Exception {
+ TestUtils.teardownSubscription(testSub.getId());
+ }
+
+
+ public void testUpdateSubscription() throws Exception {
+
+ PlanetManager mgr = PlanetFactory.getPlanet().getPlanetManager();
+ Subscription sub = mgr.getSubscriptionById(testSub.getId());
+
+ // update the subscription
+ FeedUpdater updater = new SingleThreadedFeedUpdater();
+ updater.updateSubscription(sub);
+ TestUtils.endSession(true);
+
+ // verify the results
+ sub = mgr.getSubscription(feed_url);
+ assertNotNull(sub);
+ assertEquals(feed_url, sub.getFeedURL());
+ assertEquals("http://rollerweblogger.org/roller/", sub.getSiteURL());
+ assertEquals("Blogging Roller", sub.getTitle());
+ assertNotNull(sub.getLastUpdated());
+ assertTrue(sub.getEntries().size() > 0);
+ }
+
+}