You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@roller.apache.org by ag...@apache.org on 2006/08/22 03:38:16 UTC
svn commit: r433455 - in
/incubator/roller/branches/roller_3.0/src/org/apache/roller:
business/hibernate/ ui/core/tasks/
Author: agilliland
Date: Mon Aug 21 18:38:15 2006
New Revision: 433455
URL: http://svn.apache.org/viewvc?rev=433455&view=rev
Log:
cleanup handling of local feeds in planet for roller 3.0
- separated out sync websites task and rank subscriptions task.
- sync websites now uses roller 3.0 urls
- getEntries() method for local roller planet impl now works with 3.0 urls
Added:
incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/TechnoratiRankingsTask.java
Modified:
incubator/roller/branches/roller_3.0/src/org/apache/roller/business/hibernate/HibernateRollerPlanetManagerImpl.java
incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/RefreshEntriesTask.java
incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/SyncWebsitesTask.java
Modified: incubator/roller/branches/roller_3.0/src/org/apache/roller/business/hibernate/HibernateRollerPlanetManagerImpl.java
URL: http://svn.apache.org/viewvc/incubator/roller/branches/roller_3.0/src/org/apache/roller/business/hibernate/HibernateRollerPlanetManagerImpl.java?rev=433455&r1=433454&r2=433455&view=diff
==============================================================================
--- incubator/roller/branches/roller_3.0/src/org/apache/roller/business/hibernate/HibernateRollerPlanetManagerImpl.java (original)
+++ incubator/roller/branches/roller_3.0/src/org/apache/roller/business/hibernate/HibernateRollerPlanetManagerImpl.java Mon Aug 21 18:38:15 2006
@@ -1,24 +1,19 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. The ASF licenses this file to You
-* under the Apache License, Version 2.0 (the "License"); you may not
-* use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License. For additional information regarding
-* copyright in this work, please see the NOTICE file in the top level
-* directory of this distribution.
-*/
-/*
- * HibernateRollerPlanetManagerImpl.java
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * Created on April 17, 2006, 1:53 PM
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
*/
package org.apache.roller.business.hibernate;
@@ -52,7 +47,7 @@
/**
* An extended version of the base PlanetManager implementation.
- *
+ *
* This is meant for use by Roller installations that are running the planet
* aggregator in the same application instance and want to fetch feeds from
* their local Roller blogs in a more efficient manner.
@@ -85,18 +80,20 @@
return super.getNewEntries(sub, feedFetcher, feedInfoCache);
}
- // url must be local, lets deal with it
- Set newEntries = new TreeSet();
try {
// for local feeds, sub.author = website.handle
- if (sub.getAuthor()!=null && sub.getFeedURL().endsWith(sub.getAuthor())) {
+ // feed is from our domain and we have a handle, lets deal with it
+ if(sub.getAuthor() != null) {
log.debug("Getting LOCAL feed "+sub.getFeedURL());
+ Set newEntries = new TreeSet();
+
// get corresponding website object
UserManager usermgr = RollerFactory.getRoller().getUserManager();
WebsiteData website = usermgr.getWebsiteByHandle(sub.getAuthor());
- if (website == null) return newEntries;
+ if (website == null)
+ return newEntries;
// figure website last update time
WeblogManager blogmgr = RollerFactory.getRoller().getWeblogManager();
@@ -115,14 +112,15 @@
"site.newsfeeds.defaultEntries");
entries = blogmgr.getWeblogEntries(
website,
- null,
+ null,
null, // startDate
new Date(), // endDate
null, // catName
WeblogEntryData.PUBLISHED, // status
null, // sortby (null means pubTime)
-null, 0, // offset
- entryCount); // locale
+ null, // locale
+ 0, // offset
+ entryCount);
sub.setLastUpdated(siteUpdated);
saveSubscription(sub);
@@ -152,6 +150,7 @@
log.error("ERROR processing subscription entry", e);
}
}
+
return newEntries;
}
} catch (Exception e) {
Modified: incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/RefreshEntriesTask.java
URL: http://svn.apache.org/viewvc/incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/RefreshEntriesTask.java?rev=433455&r1=433454&r2=433455&view=diff
==============================================================================
--- incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/RefreshEntriesTask.java (original)
+++ incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/RefreshEntriesTask.java Mon Aug 21 18:38:15 2006
@@ -34,10 +34,11 @@
public class RefreshEntriesTask extends TimerTask implements ScheduledTask {
private static Log logger = LogFactory.getLog(RefreshEntriesTask.class);
- private Roller roller = null;
- /** Task may be run from the command line */
+ /**
+ * Task may be run from the command line
+ */
public static void main(String[] args) {
try {
RollerFactory.setRoller(
@@ -54,12 +55,13 @@
public void init(Roller roller, String realPath) throws RollerException {
- this.roller = (Roller)roller;
+ // no-op
}
public void run() {
try {
+ Roller roller = RollerFactory.getRoller();
roller.getPlanetManager().refreshEntries();
roller.flush();
roller.release();
Modified: incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/SyncWebsitesTask.java
URL: http://svn.apache.org/viewvc/incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/SyncWebsitesTask.java?rev=433455&r1=433454&r2=433455&view=diff
==============================================================================
--- incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/SyncWebsitesTask.java (original)
+++ incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/SyncWebsitesTask.java Mon Aug 21 18:38:15 2006
@@ -18,43 +18,36 @@
package org.apache.roller.ui.core.tasks;
-import java.io.IOException;
import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.TimerTask;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.roller.RollerException;
-import org.apache.roller.config.RollerConfig;
import org.apache.roller.config.RollerRuntimeConfig;
import org.apache.roller.model.PlanetManager;
import org.apache.roller.model.Roller;
import org.apache.roller.model.RollerFactory;
import org.apache.roller.model.ScheduledTask;
import org.apache.roller.model.UserManager;
-import org.apache.roller.pojos.PlanetConfigData;
import org.apache.roller.pojos.PlanetGroupData;
import org.apache.roller.pojos.PlanetSubscriptionData;
-import org.apache.roller.pojos.UserData;
import org.apache.roller.pojos.WebsiteData;
-import org.apache.roller.util.Technorati;
+import org.apache.roller.util.URLUtilities;
/**
- * Ensure that every user is represented by a subscription in Planet Roller
- * database. Also "ranks" each subscription by populating Technorati inbound
- * blogs and links counts.
+ * Ensure that every weblog has a subscription in Planet Roller database.
*/
public class SyncWebsitesTask extends TimerTask implements ScheduledTask {
- private static Log logger = LogFactory.getLog(SyncWebsitesTask.class);
- private Roller roller = null;
+ private static Log log = LogFactory.getLog(SyncWebsitesTask.class);
- /** Task may be run from the command line */
+ /**
+ * Task may be run from the command line
+ */
public static void main(String[] args) {
try {
RollerFactory.setRoller(
@@ -71,186 +64,96 @@
public void init(Roller roller, String realPath) throws RollerException {
- this.roller = roller;
+ // no-op
}
public void run() {
syncWebsites();
- rankSubscriptions();
}
/**
- * Ensure there's a subscription in the "all" group for every Roller user.
+ * Ensure there's a subscription in the "all" group for every Roller weblog.
*/
private void syncWebsites() {
- try {
- List liveUserFeeds = new ArrayList();
- String baseURL = RollerRuntimeConfig.getProperty("site.absoluteurl");
- if (baseURL == null || baseURL.trim().length()==0) {
- logger.error("ERROR: cannot sync websites with Planet Roller - "
- +"absolute URL not specified in Roller Config");
- } else {
- PlanetManager planet = roller.getPlanetManager();
- UserManager userManager = roller.getUserManager();
- PlanetGroupData group = planet.getGroup("all");
- if (group == null) {
- group = new PlanetGroupData();
- group.setHandle("all");
- group.setTitle("all");
- planet.saveGroup(group);
- roller.flush();
- }
- try {
- String baseFeedURL = baseURL + "/rss/";
- String baseSiteURL = baseURL + "/page/";
- // get list of all enabled and active weblogs
- Iterator websites =
- roller.getUserManager().getWebsites(null, Boolean.TRUE, Boolean.TRUE, null, null, 0, -1).iterator();
- while (websites.hasNext()) {
- WebsiteData website = (WebsiteData)websites.next();
-
- StringBuffer sitesb = new StringBuffer();
- sitesb.append(baseSiteURL);
- sitesb.append(website.getHandle());
- String siteUrl = sitesb.toString();
-
- StringBuffer feedsb = new StringBuffer();
- feedsb.append(baseFeedURL);
- feedsb.append(website.getHandle());
- String feedUrl = feedsb.toString();
-
- liveUserFeeds.add(feedUrl);
-
- PlanetSubscriptionData sub =
- planet.getSubscription(feedUrl);
- if (sub == null) {
- logger.info("ADDING feed: "+feedUrl);
- sub = new PlanetSubscriptionData();
- sub.setTitle(website.getName());
- sub.setFeedURL(feedUrl);
- sub.setSiteURL(siteUrl);
- sub.setAuthor(website.getHandle());
- planet.saveSubscription(sub);
- group.addSubscription(sub);
- } else {
- sub.setTitle(website.getName());
- sub.setAuthor(website.getHandle());
- planet.saveSubscription(sub);
- }
- }
- planet.saveGroup(group);
- roller.flush();
- roller.release();
-
- // TODO: new planet manager method deleteSubs(list)
- group = group = planet.getGroup("all");
- Iterator subs = group.getSubscriptions().iterator();
- while (subs.hasNext()) {
- PlanetSubscriptionData sub =
- (PlanetSubscriptionData)subs.next();
- if (!liveUserFeeds.contains(sub.getFeedURL())) {
- logger.info("DELETING feed: "+sub.getFeedURL());
- planet.deleteSubscription(sub);
- }
- }
- roller.flush();
- } finally {
- roller.release();
- }
- }
- } catch (RollerException e) {
- logger.error("ERROR refreshing entries", e);
+
+ // make sure we have an absolute url value
+ String absUrl = RollerRuntimeConfig.getProperty("site.absoluteurl");
+ if(absUrl == null || absUrl.trim().length() == 0) {
+ log.error("ERROR: cannot sync websites with Planet Roller - "
+ +"absolute URL not specified in Roller Config");
+ return;
}
- }
-
-
- /**
- * Loop through all subscriptions get get Technorati rankings for each
- */
- private void rankSubscriptions() {
- int count = 0;
- int errorCount = 0;
+
try {
- PlanetManager planet = roller.getPlanetManager();
- PlanetConfigData config = planet.getConfiguration();
- Technorati technorati = null;
- try {
- if (config.getProxyHost()!=null && config.getProxyPort() != -1) {
- technorati = new Technorati(
- config.getProxyHost(), config.getProxyPort());
- } else {
- technorati = new Technorati();
- }
- } catch (IOException e) {
- logger.error("Aborting collection of Technorati rankings.\n"
- +"technorati.license not found at root of classpath.\n"
- +"Get license at http://technorati.com/developers/apikey.html\n"
- +"Put the license string in a file called technorati.license.\n"
- +"And place that file at the root of Roller's classpath.\n"
- +"For example, in the /WEB-INF/classes directory.");
- return;
+ PlanetManager planet = RollerFactory.getRoller().getPlanetManager();
+ UserManager userManager = RollerFactory.getRoller().getUserManager();
+
+ // first, make sure there is an "all" planet group
+ PlanetGroupData group = planet.getGroup("all");
+ if(group == null) {
+ group = new PlanetGroupData();
+ group.setHandle("all");
+ group.setTitle("all");
+ planet.saveGroup(group);
}
- UserManager userManager = roller.getUserManager();
- try {
- int limit = RollerConfig.getIntProperty(
- "planet.aggregator.technorati.limit", 500);
- int userCount = planet.getSubscriptionCount();
- int mod = (userCount / limit) + 1;
+
+ // walk through all enable weblogs and add/update subs as needed
+ List liveUserFeeds = new ArrayList();
+ Iterator websites =
+ userManager.getWebsites(null, Boolean.TRUE, Boolean.TRUE, null, null, 0, -1).iterator();
+ while(websites.hasNext()) {
+ WebsiteData weblog = (WebsiteData) websites.next();
- Calendar cal = Calendar.getInstance();
- cal.setTime(new Date());
- int day = cal.get(Calendar.DAY_OF_YEAR);
+ String siteUrl = URLUtilities.getWeblogURL(weblog, null, true);
+ String feedUrl = URLUtilities.getWeblogFeedURL(weblog, null, "entries", "rss", null, false, true);
- int start = (day % mod) * limit;
- int end = start + limit;
- end = end > userCount ? userCount : end;
- logger.info("Updating subscriptions ["+start+":"+end+"]");
+ // add feed url to the "live" list
+ liveUserFeeds.add(feedUrl);
- Iterator subs = planet.getAllSubscriptions();
- while (subs.hasNext()) {
- PlanetSubscriptionData sub =
- (PlanetSubscriptionData)subs.next();
- if (count >= start && count < end) {
- try {
- Technorati.Result result =
- technorati.getBloginfo(sub.getSiteURL());
- if (result != null && result.getWeblog() != null) {
- sub.setInboundblogs(
- result.getWeblog().getInboundblogs());
- sub.setInboundlinks(
- result.getWeblog().getInboundlinks());
- logger.debug("Adding rank for "
- +sub.getFeedURL()+" ["+count+"|"
- +sub.getInboundblogs()+"|"
- +sub.getInboundlinks()+"]");
- } else {
- logger.debug(
- "No ranking available for "
- +sub.getFeedURL()+" ["+count+"]");
- sub.setInboundlinks(0);
- sub.setInboundblogs(0);
- }
- planet.saveSubscription(sub);
- } catch (Exception e) {
- logger.warn("WARN ranking subscription ["
- + count + "]: " + e.getMessage());
- if (errorCount++ > 5) {
- logger.warn(
- " Stopping ranking, too many errors");
- break;
- }
- }
- }
- count++;
+ // if sub already exists then update it, otherwise add it
+ PlanetSubscriptionData sub = planet.getSubscription(feedUrl);
+ if (sub == null) {
+ log.info("ADDING feed: "+feedUrl);
+
+ sub = new PlanetSubscriptionData();
+ sub.setTitle(weblog.getName());
+ sub.setFeedURL(feedUrl);
+ sub.setSiteURL(siteUrl);
+ sub.setAuthor(weblog.getHandle());
+
+ planet.saveSubscription(sub);
+ group.addSubscription(sub);
+ } else {
+ sub.setTitle(weblog.getName());
+ sub.setAuthor(weblog.getHandle());
+
+ planet.saveSubscription(sub);
}
- roller.flush();
- } finally {
- roller.release();
}
- } catch (Exception e) {
- logger.error("ERROR ranking subscriptions", e);
+
+ // new subs added, existing subs updated, now delete old subs
+ Iterator subs = group.getSubscriptions().iterator();
+ while(subs.hasNext()) {
+ PlanetSubscriptionData sub =
+ (PlanetSubscriptionData) subs.next();
+ if (!liveUserFeeds.contains(sub.getFeedURL())) {
+ log.info("DELETING feed: "+sub.getFeedURL());
+ planet.deleteSubscription(sub);
+ group.removeSubscription(sub);
+ }
+ }
+
+ // all done, lets save
+ planet.saveGroup(group);
+ RollerFactory.getRoller().flush();
+
+ } catch (RollerException e) {
+ log.error("ERROR refreshing entries", e);
+ } finally {
+ // don't forget to release
+ RollerFactory.getRoller().release();
}
}
Added: incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/TechnoratiRankingsTask.java
URL: http://svn.apache.org/viewvc/incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/TechnoratiRankingsTask.java?rev=433455&view=auto
==============================================================================
--- incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/TechnoratiRankingsTask.java (added)
+++ incubator/roller/branches/roller_3.0/src/org/apache/roller/ui/core/tasks/TechnoratiRankingsTask.java Mon Aug 21 18:38:15 2006
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+
+package org.apache.roller.ui.core.tasks;
+
+import java.io.IOException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.TimerTask;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.roller.RollerException;
+import org.apache.roller.config.RollerConfig;
+import org.apache.roller.model.PlanetManager;
+import org.apache.roller.model.Roller;
+import org.apache.roller.model.RollerFactory;
+import org.apache.roller.model.ScheduledTask;
+import org.apache.roller.model.UserManager;
+import org.apache.roller.pojos.PlanetConfigData;
+import org.apache.roller.pojos.PlanetSubscriptionData;
+import org.apache.roller.util.Technorati;
+
+
+/**
+ * Rank each subscription by populating Technorati inbound blog and link counts.
+ */
+public class TechnoratiRankingsTask extends TimerTask implements ScheduledTask {
+
+ private static Log log = LogFactory.getLog(TechnoratiRankingsTask.class);
+
+
+ /**
+ * Task may be run from the command line
+ */
+ public static void main(String[] args) {
+ try {
+ RollerFactory.setRoller(
+ "org.apache.roller.business.hibernate.HibernateRollerImpl");
+ TechnoratiRankingsTask task = new TechnoratiRankingsTask();
+ task.init(RollerFactory.getRoller(), "dummy");
+ task.run();
+ System.exit(0);
+ } catch (Throwable t) {
+ t.printStackTrace();
+ System.exit(-1);
+ }
+ }
+
+
+ public void init(Roller roller, String realPath) throws RollerException {
+ // no-op
+ }
+
+
+ public void run() {
+ rankSubscriptions();
+ }
+
+
+ /**
+ * Loop through all subscriptions get get Technorati rankings for each
+ */
+ private void rankSubscriptions() {
+
+ int count = 0;
+ int errorCount = 0;
+ try {
+ PlanetManager planet = RollerFactory.getRoller().getPlanetManager();
+ PlanetConfigData config = planet.getConfiguration();
+ Technorati technorati = null;
+ try {
+ if (config.getProxyHost()!=null && config.getProxyPort() != -1) {
+ technorati = new Technorati(
+ config.getProxyHost(), config.getProxyPort());
+ } else {
+ technorati = new Technorati();
+ }
+ } catch (IOException e) {
+ log.error("Aborting collection of Technorati rankings.\n"
+ +"technorati.license not found at root of classpath.\n"
+ +"Get license at http://technorati.com/developers/apikey.html\n"
+ +"Put the license string in a file called technorati.license.\n"
+ +"And place that file at the root of Roller's classpath.\n"
+ +"For example, in the /WEB-INF/classes directory.");
+ return;
+ }
+
+ UserManager userManager = RollerFactory.getRoller().getUserManager();
+ try {
+ int limit = RollerConfig.getIntProperty(
+ "planet.aggregator.technorati.limit", 500);
+ int userCount = planet.getSubscriptionCount();
+ int mod = (userCount / limit) + 1;
+
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(new Date());
+ int day = cal.get(Calendar.DAY_OF_YEAR);
+
+ int start = (day % mod) * limit;
+ int end = start + limit;
+ end = end > userCount ? userCount : end;
+ log.info("Updating subscriptions ["+start+":"+end+"]");
+
+ Iterator subs = planet.getAllSubscriptions();
+ while (subs.hasNext()) {
+ PlanetSubscriptionData sub =
+ (PlanetSubscriptionData)subs.next();
+ if (count >= start && count < end) {
+ try {
+ Technorati.Result result =
+ technorati.getBloginfo(sub.getSiteURL());
+ if (result != null && result.getWeblog() != null) {
+ sub.setInboundblogs(
+ result.getWeblog().getInboundblogs());
+ sub.setInboundlinks(
+ result.getWeblog().getInboundlinks());
+ log.debug("Adding rank for "
+ +sub.getFeedURL()+" ["+count+"|"
+ +sub.getInboundblogs()+"|"
+ +sub.getInboundlinks()+"]");
+ } else {
+ log.debug(
+ "No ranking available for "
+ +sub.getFeedURL()+" ["+count+"]");
+ sub.setInboundlinks(0);
+ sub.setInboundblogs(0);
+ }
+ planet.saveSubscription(sub);
+ } catch (Exception e) {
+ log.warn("WARN ranking subscription ["
+ + count + "]: " + e.getMessage());
+ if (errorCount++ > 5) {
+ log.warn(
+ " Stopping ranking, too many errors");
+ break;
+ }
+ }
+ }
+ count++;
+ }
+
+ // all done, flush results to db
+ RollerFactory.getRoller().flush();
+
+ } finally {
+ RollerFactory.getRoller().release();
+ }
+
+ } catch (Exception e) {
+ log.error("ERROR ranking subscriptions", e);
+ }
+ }
+
+}