You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@commons.apache.org by bu...@apache.org on 2004/10/22 02:37:09 UTC
cvs commit: jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/test TestFeedFilter.java TestFeedLocator.java TestProbeLocator.java
burton 2004/10/21 17:37:09
Modified: feedparser build.properties build.xml
feedparser/src/java/org/apache/commons/feedparser/locate
BlogServiceDiscovery.java FeedLocator.java
LinkLocator.java ProbeLocator.java
ResourceExpander.java
feedparser/src/java/org/apache/commons/feedparser/test
TestFeedFilter.java TestFeedLocator.java
TestProbeLocator.java
Log:
Brads changes to feedparser...
Revision Changes Path
1.2 +8 -0 jakarta-commons-sandbox/feedparser/build.properties
Index: build.properties
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/feedparser/build.properties,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- build.properties 21 Oct 2004 01:07:04 -0000 1.1
+++ build.properties 22 Oct 2004 00:37:08 -0000 1.2
@@ -0,0 +1,8 @@
+<!-- The full path to where the Jakarta Feed Parser is installed,
+ such as "c:/jakarta/feedparser"; use forward slashes instead of
+ backslashes on Windows. -->
+feedparser.home=c:/feedparser
+
+<!-- The file path location to where all of our external JARs are located
+ that are not bundled with the Jakarta Feed Parser, such as junit.jar. --%>
+ext.lib.path=c:/usr/local/ksa/lib
1.10 +134 -76 jakarta-commons-sandbox/feedparser/build.xml
Index: build.xml
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/feedparser/build.xml,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -r1.9 -r1.10
--- build.xml 18 Oct 2004 00:12:23 -0000 1.9
+++ build.xml 22 Oct 2004 00:37:08 -0000 1.10
@@ -1,6 +1,10 @@
<?xml version="1.0" encoding="ISO-8859-1"?>
-<project name="feedparser" default="jakarta-feedparser.jar" basedir=".">
+<project name="feedparser" default="help" basedir=".">
+
+ <property environment="env"/>
+
+ <property file="build.properties"/>
<property name="project" value="feedparser"/>
@@ -28,36 +32,76 @@
</fileset>
- <!--
- <fileset dir="c:/usr/local/ksa/lib">
+ <fileset dir="${ext.lib.path}">
<include name="*.jar"/>
</fileset>
- -->
+
+ <fileset dir="${ant.home}/lib">
+ <include name="*.jar"/>
+ </fileset>
</path>
+ <!-- ant-contrib tasks path http://ant-contrib.sourceforge.net;
+ this includes custom tags, such as an 'if' tag -->
+ <property name="ant-contrib.jar" value="./lib/ant-contrib-0.6.jar"/>
+ <taskdef resource="net/sf/antcontrib/antcontrib.properties">
+ <classpath>
+ <pathelement location="${ant-contrib.jar}"/>
+ </classpath>
+ </taskdef>
+
+ <taskdef name="junit"
+ classname="org.apache.tools.ant.taskdefs.optional.junit.JUnitTask"/>
+
<target name="compile">
<mkdir dir="compile"/>
- <javac srcdir="src/java/"
+ <javac srcdir="src/java/"
classpathref="project.classpath"
- destdir="compile"
+ destdir="compile"
debug="true"/>
</target>
+ <target name="clean">
+ <delete dir="compile"/>
+ </target>
+
+ <target name="clobber">
+ <delete dir="compile"/>
+ </target>
+
+ <target name="jakarta-feedparser.jar" depends="compile">
+
+ <mkdir dir="lib"/>
+
+ <jar compress="yes" jarfile="lib/jakarta-feedparser.jar">
+
+ <fileset dir="compile">
+
+ <include name="org/**"/>
+ <include name="feedparser/**"/>
+ <include name="*.class"/>
+
+ </fileset>
+
+ </jar>
+
+ </target>
+
<target name="run-example" description="Runs the example feed parser class"
depends="jakarta-feedparser.jar">
<java classpathref="project.classpath"
classname="org.apache.commons.feedparser.example.HelloFeedParser"
fork="true"
failonerror="true">
- </java>
+ </java>
</target>
-
+
<target name="debug-feed" description="Debugs a feed"
depends="jakarta-feedparser.jar">
<java classpathref="project.classpath"
@@ -65,77 +109,50 @@
fork="true"
failonerror="true">
<arg value="${feed-url}"/>
- </java>
+ </java>
</target>
-
+
<target name="test-autodiscover" description="Runs the testing class for autodiscovery"
depends="jakarta-feedparser.jar">
<java classpathref="project.classpath"
- classname="org.apache.commons.feedparser.locate.FeedLocator"
+ classname="org.apache.commons.feedparser.locate.TestFeedLocator"
fork="true"
failonerror="true">
- </java>
+ <sysproperty key="feedparser.home" value="${feedparser.home}"/>
+ </java>
</target>
-
- <target name="test-probing" description="Runs the testing class for aggresive probing"
- depends="jakarta-feedparser.jar">
+
+ <target name="test-probing" description="Runs the testing class for aggresive probing"
+ depends="jakarta-feedparser.jar">
<java classpathref="project.classpath"
classname="org.apache.commons.feedparser.test.TestProbeLocator"
fork="true"
failonerror="true">
- </java>
-
- </target>
-
- <target name="clean">
- <delete dir="compile"/>
- </target>
-
- <target name="jakarta-feedparser.jar" depends="compile">
-
- <mkdir dir="lib"/>
-
- <jar compress="yes" jarfile="lib/jakarta-feedparser.jar">
-
- <fileset dir="compile">
-
- <include name="org/**"/>
- <include name="feedparser/**"/>
- <include name="*.class"/>
-
- </fileset>
-
- </jar>
+ <sysproperty key="feedparser.home" value="${feedparser.home}"/>
+ </java>
</target>
<target name="test" depends="jakarta-feedparser.jar" description="Run junit tests.">
-
- <taskdef name="junit" classname="org.apache.tools.ant.taskdefs.optional.junit.JUnitTask"/>
-
<junit printsummary="withOutAndErr"
fork="true"
filtertrace="true"
- haltonfailure="true">
-
+ haltonfailure="true"
+ haltonerror="true">
+ <sysproperty key="feedparser.home" value="${feedparser.home}"/>
<classpath refid="project.classpath"/>
-
<formatter type="plain" usefile="false"/>
-
<test name="org.apache.commons.feedparser.test.TestFeedFilter"/>
- <test name="org.apache.commons.feedparser.test.TestFeedLocator"/>
-
- <test name="org.apache.commons.feedparser.test.TestProbeLocator"/>
<test name="org.apache.commons.feedparser.test.TestAtom"/>
<test name="org.apache.commons.feedparser.test.TestFeedParserUTF8"/>
-
+ <test name="org.apache.commons.feedparser.test.TestFeedLocator"
+ unless="test.only.local"/>
+ <test name="org.apache.commons.feedparser.test.TestProbeLocator"
+ unless="test.only.local"/>
</junit>
-
- <echo message="tests done"/>
-
</target>
<target name="javadoc">
@@ -156,7 +173,7 @@
</target>
<!--<target name="docs">-->
-
+
<!-- anakia config borrowed from Lucene -->
<!--<taskdef name="anakia"
@@ -167,7 +184,7 @@
<anakia basedir="xdocs"
destdir="docs/"
- extension=".html"
+ extension=".html"
style="./site.vsl"
projectFile="stylesheets/project.xml"
excludes="**/stylesheets/** empty.xml"
@@ -177,28 +194,69 @@
</target>-->
- <target name="test-feeds" description=""
- depends="jakarta-feedparser.jar">
-
- <!-- this test just performs raw local feed parsing -->
+ <target name="help">
+ <echo message="U S A G E :"/>
+ <echo message=""/>
+
+ <echo message="[clean,clobber]"/>
+ <echo message="Delete old builds"/>
+ <echo message=""/>
+
+ <echo message="[compile]"/>
+ <echo message="Compiles the source tree"/>
+ <echo message=""/>
+
+ <echo message="[jakarta-feedparser.jar]"/>
+ <echo message="Generates a JAR file for the feed parser"/>
+ <echo message=""/>
+
+ <echo message="[run-example]"/>
+ <echo message="Runs the example feed parser class"/>
+ <echo message=""/>
+
+ <echo message="[debug-feed]"/>
+ <echo message="Debugs a single feed by attempting to retrieve it."/>
+ <echo message="To use:"/>
+ <echo message="ant debug-feed -Dfeed-url=http://www.someblog.com"/>
+ <echo message=""/>
+
+ <echo message="[test]"/>
+ <echo message="Runs all the JUnit testing classes"/>
+ <echo message="This target has an optional value. If run as follows:"/>
+ <echo message="ant test -Dtest.only.local=true"/>
+ <echo message="Then any tests that involve network traffic are _not_ run. "/>
+ <echo message="This is useful because the remote tests can take a long time "/>
+ <echo message="to run. By default we run remote tests."/>
+ <echo message=""/>
+
+ <echo message="[test-autodiscover]"/>
+ <echo message="Runs the testing class for autodiscovery"/>
+ <echo message=""/>
+
+ <echo message="[test-probing]"/>
+ <echo message="Runs the testing class for aggresive probing"/>
+ <echo message=""/>
+
+ <echo message="[javadoc]"/>
+ <echo message="Generates JavaDoc for this project"/>
+ <echo message=""/>
+ <echo message=""/>
+
+ <echo message="General information:"/>
+ <echo message="A file named 'build.properties' exists in this"/>
+ <echo message="directory that can be used to control the build process."/>
+ <echo message="The following attributes are available:"/>
+ <echo message=""/>
+ <echo message="feedparser.home"/>
+ <echo message="The full path to where the Jakarta Feed Parser is installed,"/>
+ <echo message="such as 'c:/jakarta/feedparser'; use forward slashes instead of"/>
+ <echo message="backslashes on Windows."/>
+ <echo message=""/>
+ <echo message="ext.lib.path"/>
+ <echo message="The file path location to where all of our external "/>
+ <echo message="JARs are located that are not bundled with the "/>
+ <echo message="Jakarta Feed Parser, such as junit.jar."/>
- <junit printsummary="withOutAndErr"
- fork="true"
- filtertrace="true"
- haltonfailure="true">
-
- <classpath refid="project.classpath"/>
-
- <formatter type="plain" usefile="false"/>
-
- <test name="org.apache.commons.feedparser.test.TestFeedFilter"/>
- <test name="org.apache.commons.feedparser.test.TestAtom"/>
-
- <!-- FIXME: make sure this uses tests: -->
- <test name="org.apache.commons.feedparser.test.TestFeedParserUTF8"/>
-
- </junit>
-
- </target>
+ </target>
</project>
1.5 +14 -290 jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/locate/BlogServiceDiscovery.java
Index: BlogServiceDiscovery.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/locate/BlogServiceDiscovery.java,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -r1.4 -r1.5
--- BlogServiceDiscovery.java 30 Sep 2004 19:27:38 -0000 1.4
+++ BlogServiceDiscovery.java 22 Oct 2004 00:37:08 -0000 1.5
@@ -16,310 +16,34 @@
package org.apache.commons.feedparser.locate;
-import java.util.regex.*;
+import org.apache.commons.feedparser.*;
+import org.apache.commons.feedparser.locate.blogservice.*;
/**
- *
* Determines what blog provider a given URI is using,
* such as whether it is hosted on Blogspot, Radio Userland, etc.
*
* @author <a href="mailto:bkn3@columbia.edu">Brad Neuberg/a>
*/
public class BlogServiceDiscovery {
- /** Locates all the generator meta tags
- * (i.e. <meta content="generator" content="someGenerator"/>)
- */
- private static Pattern metaTagsPattern =
- Pattern.compile("<[\\s]*meta[\\w\\s=\"']*name=['\" ]generator[\"' ][\\w\\s=\"']*[^>]*",
- Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
-
- /** A pattern used to discover PMachine blogs. */
- private static Pattern pmachinePattern =
- Pattern.compile("pmachine", Pattern.CASE_INSENSITIVE);
- /** A pattern used to discover Blosxom blogs. */
- private static Pattern blosxomPattern =
- Pattern.compile("alt=[\"' ]powered by blosxom[\"' ]",
- Pattern.CASE_INSENSITIVE);
-
- /** Uses the given resource and content to determine what blog provider
- * a given URI is using.
- */
- public static BlogService discover( String resource, String content ) {
- return discoverService( resource, content );
- }
-
- /** Uses the given resource to determine what blog provider
- * a given URI is using; useful if you have no content but still want
- * to do a best guess of what blog service is being used.
- */
- public static BlogService discover( String resource ) {
- return discoverService( resource, null );
+ public static BlogService discover( String resource )
+ throws FeedParserException {
+ return discover(resource, null);
}
- // NOTE: ALL of this code should change to a visitor pattern. Bad design.
-
- protected static BlogService discoverService( String resource,
- String content ) {
+ public static BlogService discover( String resource,
+ String content )
+ throws FeedParserException {
resource = resource.toLowerCase();
-
- // check the weblogs in order of their popularity
- if (isBlogger(resource, content)) {
- return BlogService.BLOGGER;
- }
- else if (isLiveJournal(resource, content)) {
- return BlogService.LIVEJOURNAL;
- }
- else if (isDiaryLand(resource, content)) {
- return BlogService.DIARYLAND;
- }
- else if (isMovableType(resource, content)) {
- return BlogService.MOVABLE_TYPE;
- }
- else if (isXanga(resource, content)) {
- return BlogService.XANGA;
- }
- else if (isWordPress(resource, content)) {
- return BlogService.WORDPRESS;
- }
- else if (isAOLJournals(resource, content)) {
- return BlogService.AOL_JOURNAL;
- }
- else if (isTypePad(resource, content)) {
- return BlogService.TYPEPAD;
- }
- else if (isPMachine(resource, content)) {
- return BlogService.PMACHINE;
- }
- /* FIXME: No way to detect Expression Engine weblogs right now
- else if (isExpressionEngine(resource, content)) {
- return BlogService.EXPRESSION_ENGINE;
- }*/
- else if (isGreyMatter(resource, content)) {
- return BlogService.GREYMATTER;
- }
- /* FIXME: We can't detect iBlog sites
- else if (isIBlog(resource, content)) {
- return BlogService.IBLOG;
- }*/
- else if (isBlosxom(resource, content)) {
- return BlogService.BLOSXOM;
- }
- /* FIXME: We can't detect Manila sites.
- else if (isManila(resource, content)) {
- return BlogService.MANILA;
- }*/
- else if ( isRadioUserland( resource, content ) ) {
- return BlogService.RADIO_USERLAND;
- }
- else if ( isTextPattern( resource, content ) ) {
- return BlogService.TEXTPATTERN;
- }
- else if ( isTextAmerica( resource, content ) ) {
- return BlogService.TEXTAMERICA;
- }
- else if ( isYahooGroups( resource, content ) ) {
- return BlogService.YAHOOGROUPS;
- }
- else if ( BlogService.FLICKR.accept( resource, content ) ) {
- return BlogService.FLICKR;
- }
- else {
- return BlogService.UNKNOWN;
- }
- }
-
- // **** vendor specific CMS detection code **********************************
-
- protected static boolean isBlogger( String resource, String content ) {
- boolean results = false;
-
- results = containsDomain(resource, "blogspot.com");
-
- if (results == false) {
- results = hasGenerator(content, "blogger");
- }
-
- return results;
- }
-
- protected static boolean isGreyMatter( String resource, String content ) {
- boolean results = false;
-
- results = hasGenerator(content, "greymatter");
-
- return results;
- }
-
- /*protected static boolean isExpressionEngine( String resource,
- String content ) {
- boolean results = false;
-
- return results;
- } */
-
- protected static boolean isMovableType( String resource, String content ) {
- boolean results = false;
-
- results = hasGenerator(content, "movabletype");
-
- return results;
- }
-
- protected static boolean isAOLJournals( String resource, String content ) {
- boolean results = false;
-
- results = containsDomain(resource, "journals.aol.com");
-
- return results;
- }
-
- protected static boolean isDiaryLand( String resource, String content ) {
- boolean results = false;
-
- results = containsDomain(resource, "diaryland.com");
-
- return results;
- }
-
- protected static boolean isPMachine( String resource, String content ) {
- boolean results = false;
-
- Matcher pmachineMatcher = pmachinePattern.matcher(resource);
-
- results = pmachineMatcher.find();
-
- return results;
- }
-
- protected static boolean isTextPattern( String resource, String content ) {
- boolean results = false;
-
- results = hasGenerator(content, "textpattern");
-
- return results;
- }
-
- /* FIXME: We can't detect Manila sites.
- protected static boolean isManila( String resource, String content ) {
- boolean results = false;
-
- return results;
- }
- */
-
- protected static boolean isTypePad( String resource, String content ) {
- boolean results = false;
-
- results = containsDomain(resource, "typepad.com");
-
- if (results == false) {
- results = hasGenerator(content, "typepad");
- }
-
- return results;
- }
-
- protected static boolean isRadioUserland( String resource, String content ) {
- boolean results = false;
-
- results = containsDomain(resource, "radio.userland.com");
-
- if (results == false) {
- results = containsDomain(resource, "radio.weblogs.com");
- }
-
- return results;
- }
-
- protected static boolean isLiveJournal( String resource, String content ) {
- boolean results = false;
-
- results = containsDomain(resource, "livejournal.com");
-
- return results;
- }
-
- protected static boolean isWordPress( String resource, String content ) {
- boolean results = false;
+ BlogService[] blogServices = BlogService.getBlogServices();
- results = hasGenerator(content, "wordpress");
-
- return results;
- }
-
- /* FIXME: We can't detect iBlog sites.
- protected static boolean isIBlog( String resource, String content ) {
- boolean results = false;
-
- return results;
- }*/
-
- protected static boolean isXanga( String resource, String content ) {
- boolean results = false;
-
- results = containsDomain(resource, "xanga.com");
-
- return results;
- }
-
- protected static boolean isBlosxom( String resource, String content ) {
- boolean results = false;
-
- // This is the only kind of blog that we need to check for a
- // 'Powered by Blosxom'. We do this with the alt= value on the
- // Powered By image.
- // FIXME: This might be fragile, but it is used across all of the
- // Blosxom blogs I have looked at so far. Brad Neuberg, bkn3@columbia.edu
-
- Matcher blosxomMatcher = blosxomPattern.matcher(content);
- results = blosxomMatcher.find();
-
- return results;
- }
-
- protected static boolean isTextAmerica( String resource, String content ) {
- boolean results = false;
-
- results = containsDomain(resource, "textamerica.com");
-
- return results;
- }
-
- protected static boolean isYahooGroups( String resource, String content ) {
- boolean results = false;
-
- results = containsDomain( resource, "groups.yahoo.com" );
-
- return results;
- }
-
- // **** util code ***********************************************************
-
- /** Determines if the given resource contains the given domain name
- * fragment.
- */
- protected static boolean containsDomain(String resource, String domain) {
- return (resource.indexOf(domain) != -1);
- }
-
- /** Determines if the given content was generated by the given generator
- * (i.e. this document contains a meta tag with name="generator" and
- * content equal to the generatorType).
- */
- protected static boolean hasGenerator(String content, String generatorType) {
- if (content == null) {
- return false;
+ for (int i = 0; i < blogServices.length; i++) {
+ if (blogServices[i].isThisService(resource, content)) {
+ return blogServices[i];
+ }
}
- Matcher metaTagsMatcher = metaTagsPattern.matcher(content);
- if (metaTagsMatcher.find()) {
- String metaTag = metaTagsMatcher.group(0).toLowerCase();
- generatorType = generatorType.toLowerCase();
- return (metaTag.indexOf(generatorType) != -1);
- }
- else {
- return false;
- }
+ return new Unknown();
}
}
1.24 +5 -5 jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/locate/FeedLocator.java
Index: FeedLocator.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/locate/FeedLocator.java,v
retrieving revision 1.23
retrieving revision 1.24
diff -u -r1.23 -r1.24
--- FeedLocator.java 4 Oct 2004 00:59:47 -0000 1.23
+++ FeedLocator.java 22 Oct 2004 00:37:08 -0000 1.24
@@ -85,7 +85,7 @@
log.info( "Using LinkLocator..." );
//this failed... try looking for links
LinkLocator.locate( resource, content, list );
-
+
//this failed... try probe location. This is more reliable than
//LinkLocation but requires a few more HTTP gets.
log.info( "Using ProbeLocator..." );
@@ -141,17 +141,17 @@
Iterator it = l.iterator();
if ( it.hasNext() == false ) {
- System.out.println( "NO LINKS FOUND" );
+ log.info( "NO LINKS FOUND" );
}
- System.out.println( " FIXME: (debug): AD RSS: " + l.getAdRSSFeed() );
- System.out.println( " FIXME: (debug): AD Atom: " + l.getAdAtomFeed() );
+ log.info( " FIXME: (debug): AD RSS: " + l.getAdRSSFeed() );
+ log.info( " FIXME: (debug): AD Atom: " + l.getAdAtomFeed() );
while ( it.hasNext() ) {
FeedReference ref = (FeedReference)it.next();
- System.out.println( ref.resource );
+ log.info( ref.resource );
}
1.9 +8 -6 jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/locate/LinkLocator.java
Index: LinkLocator.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/locate/LinkLocator.java,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -r1.8 -r1.9
--- LinkLocator.java 31 Aug 2004 23:56:14 -0000 1.8
+++ LinkLocator.java 22 Oct 2004 00:37:08 -0000 1.9
@@ -72,9 +72,8 @@
}
public boolean onAnchor( String href, String rel, String title ) {
-
String current = ResourceExpander.expand( resource, href );
-
+
if ( current == null )
return true; //obviously not
@@ -89,13 +88,15 @@
//
// http://peerfear.org/index.rdf
- //this is on a different site.
-
+ //see if the link is on a different site
if ( ! site.equals( getSite( current ) ) ) {
return true;
}
//Radio style feed. Screw that.
+ //FIXME: What happens if the Feed Parser is used to
+ //aggregate feeds on the localhost? This will break that.
+ //Brad Neuberg, bkn3@columbia.edu
if ( current.startsWith( "http://127" ) )
return true;
@@ -109,7 +110,7 @@
//domain not a link to another feed.
boolean isRSSLink = current.endsWith( ".rss" );
-
+
//support ROLLER RSS links and explicit link discovery by
//non-extensions.
if ( isRSSLink == false ) {
@@ -151,7 +152,7 @@
hasFoundAtomFeed = true;
}
-
+
if ( current.endsWith( ".xml" ) ||
current.endsWith( ".rdf" ) ) {
@@ -190,6 +191,7 @@
}
//for coderman's blog at http://www.peertech.org
+ //FIXME: This is a hack, Brad Neuberg, bkn3@columbia.edu
if ( current.endsWith( "/node/feed" ) )
list.add( current );
1.16 +23 -255 jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/locate/ProbeLocator.java
Index: ProbeLocator.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/locate/ProbeLocator.java,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -r1.15 -r1.16
--- ProbeLocator.java 30 Sep 2004 19:27:38 -0000 1.15
+++ ProbeLocator.java 22 Oct 2004 00:37:08 -0000 1.16
@@ -17,12 +17,11 @@
package org.apache.commons.feedparser.locate;
import org.apache.commons.feedparser.*;
+import org.apache.commons.feedparser.locate.blogservice.*;
import org.peerfear.newsmonster.network.*;
import java.util.*;
-import java.util.regex.*;
-import java.net.*;
import org.apache.log4j.Logger;
@@ -65,124 +64,7 @@
* for client-side aggregators would be too great.
*/
public static boolean BLOG_SERVICE_PROBING_ENABLED = false;
-
- /**
- * A regex to find any trailing filename and strip it
- */
- private static Pattern patternToStrip = Pattern.compile("[^/](/\\w*\\.\\w*$)");
-
- /**
- * A regex to extract the user from a Xanga URL
- */
- private static Pattern xangaURLPattern = Pattern.compile(".*user=(\\w*)");
-
- /**
- * Internal map to store probe URIs and their services.
- */
- private static Map probeMapping = new HashMap();
-
- static {
- /** Associates a given BlogService with a list of usual locations to find
- * their RSS file. The locations are given as an array of FeedReferences,
- * with highest quality feeds put first. These blog providers don't
- * provide consistent autodiscovery.
- */
- FeedReference blosxomLocations[] =
- // there is sometimes an index.rss20 file, but Blosxom has a bug where
- // it incorrectly responds to HTTP HEAD requests for that file,
- // saying that it exists when it doesn't. Most sites don't seem
- // to have this file so we don't include it here.
- // Brad Neuberg, bkn3@columbia.edu
- { new FeedReference("index.rss", FeedReference.RSS_MEDIA_TYPE) };
-
- // Diaryland doesn't offer feeds
- //FeedReference diaryLandLocations[] = { "" };
- FeedReference bloggerLocations[] =
- { new FeedReference("atom.xml", FeedReference.ATOM_MEDIA_TYPE) };
-
- FeedReference aolJournalLocations[] =
- { new FeedReference("atom.xml", FeedReference.ATOM_MEDIA_TYPE),
- new FeedReference("rss.xml", FeedReference.RSS_MEDIA_TYPE) };
-
- FeedReference pmachineLocations[] =
- { new FeedReference("index.xml", FeedReference.RSS_MEDIA_TYPE) };
-
- FeedReference textPatternLocations[] =
- { new FeedReference("?atom=1", FeedReference.ATOM_MEDIA_TYPE),
- new FeedReference("?rss=1", FeedReference.RSS_MEDIA_TYPE) };
-
- FeedReference manilaLocations[] =
- { new FeedReference("xml/rss.xml", FeedReference.RSS_MEDIA_TYPE),
- new FeedReference("rss.xml", FeedReference.RSS_MEDIA_TYPE) };
-
- FeedReference typepadLocations[] =
- { new FeedReference("atom.xml", FeedReference.ATOM_MEDIA_TYPE),
- new FeedReference("index.rdf", FeedReference.RSS_MEDIA_TYPE) };
-
- FeedReference radioUserlandLocations[] =
- { new FeedReference("rss.xml", FeedReference.RSS_MEDIA_TYPE) };
-
- FeedReference liveJournalLocations[] =
- { new FeedReference("data/atom", FeedReference.ATOM_MEDIA_TYPE),
- new FeedReference("data/rss", FeedReference.RSS_MEDIA_TYPE) };
-
- FeedReference wordPressLocations[] =
- { new FeedReference("wp-atom.php", FeedReference.ATOM_MEDIA_TYPE),
- new FeedReference("wp-rss2.php", FeedReference.RSS_MEDIA_TYPE),
- new FeedReference("wp-rss.php", FeedReference.RSS_MEDIA_TYPE) };
-
- FeedReference iBlogLocations[] =
- { new FeedReference("rss.xml", FeedReference.RSS_MEDIA_TYPE) };
-
- // Xanga feeds have to be handled specially since they put their
- // feeds at the location: http://www.xanga.com/rss.aspx?user=username
- FeedReference xangaLocations[] =
- { new FeedReference("rss.aspx?user=", FeedReference.RSS_MEDIA_TYPE) };
-
- FeedReference textAmericaLocations[] =
- { new FeedReference("rss.aspx", FeedReference.RSS_MEDIA_TYPE) };
-
- FeedReference unknownLocations[] =
- { new FeedReference("atom.xml",FeedReference.ATOM_MEDIA_TYPE),
- new FeedReference("index.rss", FeedReference.RSS_MEDIA_TYPE),
- new FeedReference("rss.xml", FeedReference.RSS_MEDIA_TYPE),
- new FeedReference("index.rdf", FeedReference.RSS_MEDIA_TYPE),
- new FeedReference("index.xml", FeedReference.RSS_MEDIA_TYPE),
- new FeedReference("xml/rss.xml", FeedReference.RSS_MEDIA_TYPE) };
-
- FeedReference yahooGroupsLocations[] =
- { new FeedReference( "", FeedReference.RSS_MEDIA_TYPE) };
-
- probeMapping.put( BlogService.BLOSXOM, blosxomLocations );
-
- //Tue Aug 31 2004 04:21 PM (burton@rojo.com): Diaryland doesn't
- //currently offer RSS or Atom feeds (shame, shame, shame). This is a
- //placeholder until they see the light and provide Atom feeds.
-
- //probeMapping.put( BlogService.DIARYLAND, diaryLandLocations );
- probeMapping.put( BlogService.BLOGGER, bloggerLocations );
- probeMapping.put( BlogService.AOL_JOURNAL, aolJournalLocations );
- probeMapping.put( BlogService.PMACHINE, pmachineLocations );
- probeMapping.put( BlogService.TEXTPATTERN, textPatternLocations );
- probeMapping.put( BlogService.MANILA, manilaLocations );
- probeMapping.put( BlogService.TYPEPAD, typepadLocations );
- probeMapping.put( BlogService.RADIO_USERLAND, radioUserlandLocations );
- probeMapping.put( BlogService.LIVEJOURNAL, liveJournalLocations );
- probeMapping.put( BlogService.WORDPRESS, wordPressLocations );
- probeMapping.put( BlogService.IBLOG, iBlogLocations );
- probeMapping.put( BlogService.XANGA, xangaLocations);
- probeMapping.put( BlogService.YAHOOGROUPS, yahooGroupsLocations);
-
- //probeMapping.put( BlogService.YAHOOGROUPS, yahooGroupsLocations);
-
- probeMapping.put( BlogService.FLICKR, BlogService.FLICKR.getLocations() );
-
- probeMapping.put( BlogService.UNKNOWN, unknownLocations );
-
- probeMapping.put( BlogService.TEXTAMERICA, textAmericaLocations );
-
- }
/**
*
@@ -192,104 +74,54 @@
throws Exception {
// determine what blog service we are dealing with
- BlogService blogService = BlogServiceDiscovery.discover( resource, content );
+ BlogService blogService = BlogServiceDiscovery.discover( resource, content );
// fail-fast if we already have some results and if we determine that
// we can trust the results (TextAmerica has invalid autodiscovery,
// for example)
- if ( list.size() > 0 && blogService.hasValidAutodiscovery() )
+ if ( list.size() > 0 && blogService.hasValidAutoDiscovery() )
return list;
- else if ( blogService.hasValidAutodiscovery() == false ) {
+ else if ( blogService.hasValidAutoDiscovery() == false ) {
// clear out the list so far since we can't trust the results
list.clear();
}
if ( BLOG_SERVICE_PROBING_ENABLED || AGGRESIVE_PROBING_ENABLED ) {
-
- String baseFeedPath = getFeedPath( resource );
-
- FeedReference mapping[] = null;
-
- HashSet previousAttempts = new HashSet();
-
- boolean feedFound = false;
-
- if ( probeMapping.containsKey( blogService ) ) {
-
- mapping = (FeedReference[])probeMapping.get( blogService );
+ List servicesToTry = new ArrayList();
+ servicesToTry.add(blogService);
+ // only try the Unknown service if we want aggresive probing
+ if (AGGRESIVE_PROBING_ENABLED)
+ servicesToTry.add(new Unknown());
+ Iterator iter = servicesToTry.iterator();
+ Set previousAttempts = new HashSet();
+
+ while (iter.hasNext() && list.size() == 0) {
+ BlogService currentService = (BlogService)iter.next();
+ FeedReference[] mapping = currentService.getFeedLocations(resource, content);
log.info( "mapping = " + mapping );
- log.info( "baseFeedPath = " + baseFeedPath );
-
+
// try out each mapping
for (int i = 0; i < mapping.length; i++) {
+ String baseFeedPath = currentService.getBaseFeedPath(resource);
String pathToTest = baseFeedPath + mapping[i].resource;
-
- //FIXME: generalize this in the future. We should NOT have
- //custom tests here.
-
- // we have to do special probing for Xanga
- if ( blogService.equals( BlogService.XANGA ) ) {
- pathToTest += getXangaUser(resource);
- }
-
- if ( blogService.equals( BlogService.YAHOOGROUPS ) ) {
-
- pathToTest = BlogService.YAHOOGROUPS.getFeedResource( resource );
-
- }
-
- //right now this is ONLY for Flickr
-
- if ( blogService.useCustomFeedResource() ) {
- pathToTest = blogService.getFeedResource( resource );
- }
-
log.info( "pathToTest = " + pathToTest );
- if ( feedExists( pathToTest ) ) {
+ if ( !previousAttempts.contains( pathToTest )
+ && feedExists( pathToTest ) ) {
log.info("Feed exists");
FeedReference feedReference = new FeedReference( pathToTest,
mapping[i].type );
- feedReference.method = FeedReference.METHOD_PROBE_DISCOVERY;
-
+ feedReference.method = FeedReference.METHOD_PROBE_DISCOVERY;
+ previousAttempts.add( pathToTest );
onFeedReference( feedReference, list );
-
- feedFound = true;
-
}
-
+
// record this attempt so we don't repeat it again if
// we are doing aggresive probing
previousAttempts.add( pathToTest );
}
}
-
- // if we have nothing so far, do aggresive probing
- if ( AGGRESIVE_PROBING_ENABLED && feedFound == false ) {
-
- mapping = (FeedReference[])probeMapping.get( BlogService.UNKNOWN );
-
- // try out each mapping
- for (int i = 0; i < mapping.length; i++) {
- //NOTE: this shares duplicate code with the above tests.
-
- String pathToTest = baseFeedPath + mapping[i].resource;
- if ( previousAttempts.contains( pathToTest ) == false ) {
- if ( feedExists( pathToTest ) ) {
-
- FeedReference feedReference = new FeedReference( pathToTest,
- mapping[i].type);
-
- feedReference.method = FeedReference.METHOD_PROBE_DISCOVERY;
-
- onFeedReference( feedReference, list );
-
- }
- }
- }
- }
-
log.info( "Using aggresive probing, found the following:" );
log.info( "Blog service: " + blogService );
}
@@ -323,61 +155,6 @@
list.add( ref );
}
-
- /** This method takes a resource, such as "http://www.codinginparadise.org/myweblog.php",
- * and gets the path necessary to build up a feed, such as
- * "http://www.codinginparadise.org/". Basicly it appends a slash to the end if there
- * is not one, and removes any file names that might be at the end, such as
- * "myweblog.php".
- *
- * There is a special exception for some Blosxom blogs,
- * which have things inside of a cgi-script and 'hang' their RSS files
- * off of this cgi-bin. For example,
- * http://www.bitbucketheaven.com/cgi-bin/blosxom.cgi has its RSS file
- * at http://www.bitbucketheaven.com/cgi-bin/blosxom.cgi/index.rss, so
- * we must return the blosxom.cgi at the end as well for this method.
- *
- * @throws MalformedURLException Thrown if the given resource's URL is incorrectly
- * formatted.
- *
- * @author Brad Neuberg, bkn3@columbia.edu
- */
- protected static String getFeedPath( String resource )
- throws MalformedURLException {
-
- // strip off any query string or anchors
- int end = resource.lastIndexOf( "#" );
-
- if ( end != -1 )
- resource = resource.substring( 0, end );
-
- end = resource.lastIndexOf( "?" );
-
- if ( end != -1 )
- resource = resource.substring( 0, end );
-
- if ( ! resource.endsWith( "blosxom.cgi" ) ) {
- Matcher fileMatcher = patternToStrip.matcher(resource);
- if (fileMatcher.find()) {
- String stringToStrip = fileMatcher.group(1);
- int startStrip = resource.indexOf(stringToStrip);
- resource = resource.substring(0, startStrip);
- }
- }
-
- if ( ! resource.endsWith( "/" ) ) {
- resource = resource + "/";
- }
-
- return resource;
- }
-
- public static void main( String[] args ) throws Exception {
-
- log.info( getFeedPath( "http://foo.com/bar?cat=dog" ) );
- log.info( getFeedPath( "http://foo.com/bar?cat=dog#adf" ) );
-
- }
/** Does an HTTP HEAD to see if the given resource exists.
*
@@ -400,15 +177,6 @@
return response == 200;
}
- /** Xanga's feed locations are dependent on the 'user' attribute in a
- * Xanga URI. This method helps extract the user element from an
- * existing URI, such as http://www.xanga.com/home.aspx?user=wdfphillz.
- */
- protected static String getXangaUser(String resource) {
- Matcher xangaMatcher = xangaURLPattern.matcher(resource);
- xangaMatcher.matches();
-
- return xangaMatcher.group(1);
- }
+
}
1.6 +30 -16 jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/locate/ResourceExpander.java
Index: ResourceExpander.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/locate/ResourceExpander.java,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- ResourceExpander.java 3 Sep 2004 19:46:47 -0000 1.5
+++ ResourceExpander.java 22 Oct 2004 00:37:08 -0000 1.6
@@ -19,6 +19,7 @@
import java.io.*;
import java.net.*;
import java.util.*;
+import java.util.regex.*;
/**
*
@@ -26,7 +27,10 @@
* @version $Id$
*/
public class ResourceExpander {
-
+ /** A regexp to determine if a URL has a scheme, such as "http://foo.com".
+ */
+ protected static Pattern schemePattern = Pattern.compile("^\\w*://.*");
+
/**
* Expand a link relavant to the current site. This takes care of links
* such as
@@ -42,20 +46,20 @@
*
* Note that all resource URLs will have correct trailing slashes. If the URL
* does not end with / then it is a file URL and not a directory.
+ *
+ * @param resource The absolute base URL that will be used to expand the
+ * link, such as "http://www.codinginparadise.org".
+ * @param link The link to possibly expand, such as "/index.rdf" or
+ * "http://www.somehost.com/somepage.html".
*
* @author <a href="mailto:burton@openprivacy.org">Kevin A. Burton</a>
*/
public static String expand( String resource, String link ) {
-
if ( link == null )
return null;
- //obvioysly not
- if ( link.startsWith( "javascript:" ) )
- return link;
-
//make sure we can use this.
- if ( isInvalidScheme( link ) )
+ if ( !isValidScheme( link ) )
return link;
//nothing if ALREADY relativized
@@ -71,15 +75,14 @@
// is present, the entire remaining parse string is assigned to
// <net_loc>. The double- slash and <net_loc> are removed from the
// parse string before
-
+ //FIXME: What happens if resource is a "file://" scheme?
if ( link.startsWith( "//" ) ) {
return "http:" + link;
}
-
- //keep going
+ //keep going
if ( link.startsWith( "/" ) ) {
link = getSite( resource ) + link;
@@ -134,17 +137,19 @@
* @author <a href="mailto:burton@peerfear.org">Kevin Burton</a>
*/
public static boolean isExpanded( String resource ) {
- return resource.startsWith( "http://" );
+ return (resource.startsWith( "http://" ) ||
+ resource.startsWith( "file://" ));
}
/**
- * Return true if this is an invalid scheme and should be expanded
- * (javascript, mailto, etc)
+ * Return true if this is an valid scheme and should be expanded.
*
* @author <a href="mailto:burton@openprivacy.org">Kevin A. Burton</a>
*/
- public static boolean isInvalidScheme( String resource ) {
-
+ public static boolean isValidScheme( String resource ) {
+ if (hasScheme(resource) == false)
+ return true;
+
//only on file: and http:
if ( resource.startsWith( "http:" ) )
@@ -154,6 +159,15 @@
return true;
return false;
+
+ }
+
+ /**
+ * Determines if the given resource has a scheme (i.e. does it start
+ * with "http://foo.com" or does it just have "foo.com").
+ */
+ public static boolean hasScheme( String resource ) {
+ return schemePattern.matcher( resource ).matches();
}
1.5 +33 -25 jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/test/TestFeedFilter.java
Index: TestFeedFilter.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/test/TestFeedFilter.java,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -r1.4 -r1.5
--- TestFeedFilter.java 14 Sep 2004 01:32:04 -0000 1.4
+++ TestFeedFilter.java 22 Oct 2004 00:37:08 -0000 1.5
@@ -1,12 +1,12 @@
/*
* Copyright 1999,2004 The Apache Software Foundation.
- *
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -44,12 +44,19 @@
* @version $Id$
*/
public class TestFeedFilter extends TestCase {
-
- public static int current = 0;
+ protected String feedparserHome;
+ public static int current = 0;
+
public TestFeedFilter( String name ) throws Exception {
super( name );
-
+
+ feedparserHome = System.getProperty("feedparser.home", ".");
+ // these come in as forward slashes, even on Windows, because of Ant,
+ // so it is safe to check them as / rather than File.separator
+ // Brad Neuberg
+ if (feedparserHome.endsWith("/") == false)
+ feedparserHome = feedparserHome + "/";
}
private void doTest( String resource ) throws Exception {
@@ -59,14 +66,14 @@
URL url = new URL( resource );
FileOutputStream fos = new FileOutputStream( "/tmp/test-feed-filter-" + current + ".html" );
-
+
PrintStream out = new PrintStream( fos, true, "UTF-8" );
out.println( "<META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html; charset=UTF-8\"> " );
out.println( "<pre>" );
DebugFeedParserListener listener = new DebugFeedParserListener( out );
-
+
FeedParser.parse( listener, url.openStream(), resource );
out.println( "</pre>" );
@@ -74,36 +81,37 @@
++current;
}
-
+
public void test1() throws Exception {
+ String path = "file:" + feedparserHome;
- doTest( "file:tests/feeds/rss-1.0-EUC-JP.rdf" );
+ doTest( path + "tests/feeds/rss-1.0-EUC-JP.rdf" );
- doTest( "file:tests/filter/nbsp-1.xml" );
+ doTest( path + "tests/filter/nbsp-1.xml" );
- doTest( "file:tests/filter/entity-atom-1.xml" );
+ doTest( path + "tests/filter/entity-atom-1.xml" );
- doTest( "file:tests/filter/prolog-atom-1.xml" );
- doTest( "file:tests/filter/prolog-atom-2.xml" );
- doTest( "file:tests/filter/prolog-opml-1.xml" );
+ doTest( path + "tests/filter/prolog-atom-1.xml" );
+ doTest( path + "tests/filter/prolog-atom-2.xml" );
+ doTest( path + "tests/filter/prolog-opml-1.xml" );
- doTest( "file:tests/filter/lisa.opml" );
+ doTest( path + "tests/filter/lisa.opml" );
- doTest( "file:tests/feeds/utf16.rss1" );
- doTest( "file:tests/feeds/utf16.rss2" );
- doTest( "file:tests/feeds/i18n.atom" );
- doTest( "file:tests/feeds/utf16.atom" );
+ doTest( path + "tests/feeds/utf16.rss1" );
+ doTest( path + "tests/feeds/utf16.rss2" );
+ doTest( path + "tests/feeds/i18n.atom" );
+ doTest( path + "tests/feeds/utf16.atom" );
- doTest( "file:tests/feeds/atom-1.xml" );
- doTest( "file:tests/feeds/rss-1.0-EUC-JP.rdf" );
- doTest( "file:tests/feeds/rss-1.0-international-1.rdf" );
+ doTest( path + "tests/feeds/atom-1.xml" );
+ doTest( path + "tests/feeds/rss-1.0-EUC-JP.rdf" );
+ doTest( path + "tests/feeds/rss-1.0-international-1.rdf" );
}
public static void main( String[] args ) throws Exception {
TestFeedFilter test = new TestFeedFilter( null );
-
+
//test.testGetWeblogLinkForResource();
test.test1();
1.6 +21 -17 jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/test/TestFeedLocator.java
Index: TestFeedLocator.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/test/TestFeedLocator.java,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- TestFeedLocator.java 31 Aug 2004 21:00:32 -0000 1.5
+++ TestFeedLocator.java 22 Oct 2004 00:37:08 -0000 1.6
@@ -44,10 +44,17 @@
* @version $Id$
*/
public class TestFeedLocator extends TestCase {
-
+ protected String feedparserHome;
+
public TestFeedLocator( String name ) throws Exception {
super( name );
-
+
+ feedparserHome = System.getProperty("feedparser.home", ".");
+ // these come in as forward slashes, even on Windows, because of Ant,
+ // so it is safe to check them as / rather than File.separator
+ // Brad Neuberg
+ if (feedparserHome.endsWith("/") == false)
+ feedparserHome = feedparserHome + "/";
}
private void doTest( String resource ) throws Exception {
@@ -58,27 +65,24 @@
Iterator it = l.iterator();
- if ( it.hasNext() == false ) {
- throw new Exception( "NO LINKS FOUND" );
- }
-
+ // See if no links were found
+ assertTrue("No links were found for " + resource, it.hasNext());
System.out.println( "Atom: " + l.getAdAtomFeed() );
System.out.println( "RSS: " + l.getAdRSSFeed() );
}
public void test1() throws Exception {
+ String path = "file:" + feedparserHome;
- doTest( "file:tests/locate/locate10.html" );
-
- doTest( "file:tests/locate/locate1.html" );
- doTest( "file:tests/locate/locate2.html" );
- doTest( "file:tests/locate/locate3.html" );
- doTest( "file:tests/locate/locate4.html" );
- doTest( "file:tests/locate/locate5.html" );
- doTest( "file:tests/locate/locate6.html" );
- doTest( "file:tests/locate/locate7.html" );
-
+ doTest( path + "tests/locate/locate1.html" );
+ doTest( path + "tests/locate/locate2.html" );
+ doTest( path + "tests/locate/locate3.html" );
+ doTest( path + "tests/locate/locate4.html" );
+ doTest( path + "tests/locate/locate5.html" );
+ doTest( path + "tests/locate/locate6.html" );
+ doTest( path + "tests/locate/locate7.html" );
+ doTest( path + "tests/locate/locate10.html" );
}
public static void main( String[] args ) throws Exception {
1.6 +82 -57 jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/test/TestProbeLocator.java
Index: TestProbeLocator.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/feedparser/src/java/org/apache/commons/feedparser/test/TestProbeLocator.java,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- TestProbeLocator.java 23 Sep 2004 01:51:26 -0000 1.5
+++ TestProbeLocator.java 22 Oct 2004 00:37:08 -0000 1.6
@@ -20,14 +20,14 @@
import org.apache.commons.feedparser.*;
import org.apache.commons.feedparser.locate.*;
+import org.apache.commons.feedparser.locate.blogservice.*;
import java.net.*;
import java.io.*;
-import org.peerfear.newsmonster.network.*;
/**
*
- * @author <a href="mailto:burton@peerfear.org">Kevin A. Burton</a>
+ * @author <a href="mailto:bkn3@columbia.edu">Brad Neuberg</a>
* @version $Id$
*/
public class TestProbeLocator extends TestCase {
@@ -64,26 +64,26 @@
public void testBlogger() throws Exception {
System.out.println("\nTesting Blogger...");
- testSite("http://edpro.blogspot.com/", BlogService.BLOGGER, 1,
+ testSite("http://edpro.blogspot.com/", new Blogger(), 1,
new String[] { FeedReference.ATOM_MEDIA_TYPE },
new String[] { "http://edpro.blogspot.com/atom.xml" },
HAS_ATOM_FEED, "http://edpro.blogspot.com/atom.xml",
NO_RSS_FEED, null);
- testSite("http://carolinascl.blogspot.com/", BlogService.BLOGGER, 1,
+ testSite("http://carolinascl.blogspot.com/", new Blogger(), 1,
new String[] { FeedReference.ATOM_MEDIA_TYPE },
new String[] { "http://carolinascl.blogspot.com/atom.xml" },
HAS_ATOM_FEED, "http://carolinascl.blogspot.com/atom.xml",
NO_RSS_FEED, null);
- testSite("http://azizindia.blogspot.com/", BlogService.BLOGGER, 1,
+ testSite("http://azizindia.blogspot.com/", new Blogger(), 1,
new String[] { FeedReference.ATOM_MEDIA_TYPE },
new String[] { "http://azizindia.blogspot.com/atom.xml" },
HAS_ATOM_FEED, "http://azizindia.blogspot.com/atom.xml",
NO_RSS_FEED, null);
// This site has no blogs
- testSite("http://davebarry.blogspot.com/", BlogService.BLOGGER, 0,
+ testSite("http://davebarry.blogspot.com/", new Blogger(), 0,
new String[] { },
new String[] { },
NO_ATOM_FEED, null,
@@ -93,7 +93,7 @@
public void testLiveJournal() throws Exception {
System.out.println("\nTesting LiveJournal...");
testSite("http://www.livejournal.com/community/indiexiankids/",
- BlogService.LIVEJOURNAL, 2,
+ new LiveJournal(), 2,
new String[] {
FeedReference.ATOM_MEDIA_TYPE,
FeedReference.RSS_MEDIA_TYPE
@@ -106,7 +106,7 @@
HAS_RSS_FEED, "http://www.livejournal.com/community/indiexiankids/data/rss");
testSite("http://www.livejournal.com/community/ajoyforever/",
- BlogService.LIVEJOURNAL, 2,
+ new LiveJournal(), 2,
new String[] {
FeedReference.ATOM_MEDIA_TYPE,
FeedReference.RSS_MEDIA_TYPE
@@ -119,7 +119,7 @@
HAS_RSS_FEED, "http://www.livejournal.com/community/ajoyforever/data/rss");
testSite("http://www.livejournal.com/users/_jb_/12332.html",
- BlogService.LIVEJOURNAL, 2,
+ new LiveJournal(), 2,
new String[] {
FeedReference.ATOM_MEDIA_TYPE,
FeedReference.RSS_MEDIA_TYPE
@@ -133,33 +133,33 @@
}
public void testDiaryLand() throws Exception {
- System.out.println("\nTesting DiaryLand...");
+ System.out.println("\nTesting DiaryLand... No tests currently");
// FIXME: Test this
}
public void testMovableType() throws Exception {
- System.out.println("\nTesting MovableType...");
+ System.out.println("\nTesting MovableType... No tests currently");
// FIXME: Test this
}
public void testXanga() throws Exception {
System.out.println("\nTesting Xanga...");
testSite("http://www.xanga.com/home.aspx?user=lithium98",
- BlogService.XANGA, 1,
+ new Xanga(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://www.xanga.com/rss.aspx?user=lithium98" },
NO_ATOM_FEED, null,
HAS_RSS_FEED, "http://www.xanga.com/rss.aspx?user=lithium98");
testSite("http://www.xanga.com/home.aspx?user=ChUnSA_86",
- BlogService.XANGA, 1,
+ new Xanga(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://www.xanga.com/rss.aspx?user=ChUnSA_86" },
NO_ATOM_FEED, null,
HAS_RSS_FEED, "http://www.xanga.com/rss.aspx?user=ChUnSA_86");
testSite("http://www.xanga.com/home.aspx?user=wdfphillz",
- BlogService.XANGA, 1,
+ new Xanga(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://www.xanga.com/rss.aspx?user=wdfphillz" },
NO_ATOM_FEED, null,
@@ -169,7 +169,7 @@
// expand resources inside of the Feed Parser; we don't
// currently do this yet, Brad Neuberg, bkn3@columbia.edu
/*testSite("http://xanga.com/home.aspx?user=joe",
- BlogService.XANGA, 1,
+ new Xanga(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://www.xanga.com/rss.aspx?user=joe" },
NO_ATOM_FEED, null,
@@ -178,8 +178,9 @@
public void testWordPress() throws Exception {
System.out.println("\nTesting WordPress...");
- testSite("http://synflood.at/blog/",
- BlogService.WORDPRESS, 3,
+ // This site went down
+ /*testSite("http://zh.yazzy.org/blog/index.php",
+ new WordPress(), 3,
new String[] {
FeedReference.ATOM_MEDIA_TYPE,
FeedReference.RSS_MEDIA_TYPE,
@@ -192,9 +193,29 @@
},
HAS_ATOM_FEED, "http://synflood.at/blog/wp-atom.php",
HAS_RSS_FEED, "http://synflood.at/blog/wp-rss2.php");
-
+ */
+ // We need to firm up our autodiscovery regular expressions before
+ // this will pass; what happens is it has autodiscovery, but those
+ // point to a different location than the usual ones aggresive
+ // discovery would find (which are also there). We get back different
+ // results at different times
+ /*testSite("http://zh.yazzy.org/blog/index.php",
+ new WordPress(), 3,
+ new String[] {
+ FeedReference.ATOM_MEDIA_TYPE,
+ FeedReference.RSS_MEDIA_TYPE,
+ FeedReference.RSS_MEDIA_TYPE
+ },
+ new String[] {
+ "http://zh.yazzy.org/blog/wp-atom.php",
+ "http://zh.yazzy.org/blog/wp-rss2.php",
+ "http://zh.yazzy.org/blog/wp-rss.php"
+ },
+ HAS_ATOM_FEED, "http://zh.yazzy.org/blog/wp-atom.php",
+ HAS_RSS_FEED, "http://zh.yazzy.org/blog/wp-rss2.php");
+ */
testSite("http://holmes.hgen.pitt.edu/~dweeks/wordpress/",
- BlogService.WORDPRESS, 3,
+ new WordPress(), 3,
new String[] {
FeedReference.ATOM_MEDIA_TYPE,
FeedReference.RSS_MEDIA_TYPE,
@@ -221,7 +242,7 @@
*/
testProbeLocator(
"http://journals.aol.com/redhdka/BrandNewDay/",
- BlogService.AOL_JOURNAL, 1,
+ new AOLJournal(), 1,
new String[] {
FeedReference.RSS_MEDIA_TYPE
},
@@ -232,7 +253,7 @@
HAS_RSS_FEED, "http://journals.aol.com/redhdka/brandnewday/rss.xml");
testFeedLocator(
"http://journals.aol.com/redhdka/BrandNewDay/",
- BlogService.AOL_JOURNAL, 2,
+ new AOLJournal(), 2,
new String[] {
FeedReference.ATOM_MEDIA_TYPE,
FeedReference.RSS_MEDIA_TYPE
@@ -246,7 +267,7 @@
testProbeLocator(
"http://journals.aol.com/goldenchildnc/GCS/",
- BlogService.AOL_JOURNAL, 1,
+ new AOLJournal(), 1,
new String[] {
FeedReference.RSS_MEDIA_TYPE
},
@@ -257,7 +278,7 @@
HAS_RSS_FEED, "http://journals.aol.com/goldenchildnc/GCS/rss.xml");
testFeedLocator(
"http://journals.aol.com/goldenchildnc/GCS/",
- BlogService.AOL_JOURNAL, 2,
+ new AOLJournal(), 2,
new String[] {
FeedReference.ATOM_MEDIA_TYPE,
FeedReference.RSS_MEDIA_TYPE
@@ -272,7 +293,7 @@
testProbeLocator(
"http://journals.aol.com/mkgninja/MelissasMisunderstandingsofLife/",
- BlogService.AOL_JOURNAL, 1,
+ new AOLJournal(), 1,
new String[] {
FeedReference.RSS_MEDIA_TYPE
},
@@ -283,7 +304,7 @@
HAS_RSS_FEED, "http://journals.aol.com/mkgninja/MelissasMisunderstandingsofLife/rss.xml");
testFeedLocator(
"http://journals.aol.com/mkgninja/MelissasMisunderstandingsofLife/",
- BlogService.AOL_JOURNAL, 2,
+ new AOLJournal(), 2,
new String[] {
FeedReference.ATOM_MEDIA_TYPE,
FeedReference.RSS_MEDIA_TYPE
@@ -303,7 +324,7 @@
// However, we get a feed when we go through the FeedParser since
// the site has autodiscovery
testSite("http://lynikers.typepad.com/",
- BlogService.TYPEPAD, 0,
+ new Typepad(), 0,
new String[] { },
new String[] { },
HAS_ATOM_FEED, "http://lynikers.typepad.com/on_buck_lake/atom.xml",
@@ -312,14 +333,14 @@
// This site has no feed that we can link probe for (it's in a different
// location then usual).
testSite("http://emmeke.typepad.com/",
- BlogService.TYPEPAD, 0,
+ new Typepad(), 0,
new String[] { },
new String[] { },
NO_ATOM_FEED, null,
HAS_RSS_FEED, "http://emmeke.typepad.com/blog/index.rdf");
testSite("http://www.prettypolitical.com/",
- BlogService.TYPEPAD, 2,
+ new Typepad(), 2,
new String[] {
FeedReference.ATOM_MEDIA_TYPE,
FeedReference.RSS_MEDIA_TYPE
@@ -336,7 +357,7 @@
System.out.println("\nTesting GreyMatter...");
// No feeds supported
testSite("http://www.chattbike.com/gilligan/",
- BlogService.GREYMATTER, 0,
+ new GreyMatter(), 0,
new String[] { },
new String[] { },
NO_ATOM_FEED, null,
@@ -344,7 +365,7 @@
// No feeds supported
testSite("http://www.electricedge.com/greymatter/",
- BlogService.GREYMATTER, 0,
+ new GreyMatter(), 0,
new String[] { },
new String[] { },
NO_ATOM_FEED, null,
@@ -354,28 +375,28 @@
public void testPMachine() throws Exception {
System.out.println("\nTesting PMachine...");
testSite("http://bamph.com",
- BlogService.UNKNOWN, 1,
+ new Unknown(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://bamph.com/index.xml" },
NO_ATOM_FEED, null,
HAS_RSS_FEED, "http://bamph.com/index.xml");
testSite("http://bucsfishingreport.com/pMachine/weblog.php",
- BlogService.PMACHINE, 1,
+ new PMachine(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://bucsfishingreport.com/pMachine/index.xml" },
NO_ATOM_FEED, null,
HAS_RSS_FEED, "http://bucsfishingreport.com/pMachine/index.xml");
testSite("http://www.simplekindoflife.com/pMachine/weblog.php",
- BlogService.PMACHINE, 1,
+ new PMachine(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://www.simplekindoflife.com/pMachine/index.xml" },
NO_ATOM_FEED, null,
HAS_RSS_FEED, "http://www.simplekindoflife.com/pMachine/index.xml");
testSite("http://www.mondfish.net/pmachine/weblog.php",
- BlogService.PMACHINE, 1,
+ new PMachine(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://www.mondfish.net/pmachine/index.xml" },
NO_ATOM_FEED, null,
@@ -385,14 +406,14 @@
public void testBlosxom() throws Exception {
System.out.println("\nTesting Blosxom...");
testSite("http://mikemason.ca/",
- BlogService.BLOSXOM, 1,
+ new Blosxom(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://mikemason.ca/index.rss" },
NO_ATOM_FEED, null,
HAS_RSS_FEED, "http://mikemason.ca/index.rss");
testSite("http://www.foobargeek.com/",
- BlogService.BLOSXOM, 1,
+ new Blosxom(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://www.foobargeek.com/index.rss" },
NO_ATOM_FEED, null,
@@ -401,14 +422,14 @@
// The FeedParser gets a different location for the XML file then
// through the aggresive prober for this feed
testSite("http://www.pipetree.com/qmacro/",
- BlogService.BLOSXOM, 1,
+ new Blosxom(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://www.pipetree.com/qmacro/index.rss" },
NO_ATOM_FEED, null,
HAS_RSS_FEED, "http://www.pipetree.com/qmacro/xml");
testSite("http://www.bitbucketheaven.com/cgi-bin/blosxom.cgi",
- BlogService.BLOSXOM, 1,
+ new Blosxom(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://www.bitbucketheaven.com/cgi-bin/blosxom.cgi/index.rss" },
NO_ATOM_FEED, null,
@@ -418,21 +439,21 @@
public void testRadioUserland() throws Exception {
System.out.println("\nTesting Radio Userland...");
testSite("http://radio.weblogs.com/0131722/",
- BlogService.RADIO_USERLAND, 1,
+ new RadioUserland(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://radio.weblogs.com/0131722/rss.xml" },
NO_ATOM_FEED, null,
HAS_RSS_FEED, "http://radio.weblogs.com/0131722/rss.xml");
testSite("http://radio.weblogs.com/0131724/",
- BlogService.RADIO_USERLAND, 1,
+ new RadioUserland(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://radio.weblogs.com/0131724/rss.xml" },
NO_ATOM_FEED, null,
HAS_RSS_FEED, "http://radio.weblogs.com/0131724/rss.xml");
testSite("http://radio.weblogs.com/0131734/",
- BlogService.RADIO_USERLAND, 1,
+ new RadioUserland(), 1,
new String[] { FeedReference.RSS_MEDIA_TYPE },
new String[] { "http://radio.weblogs.com/0131734/rss.xml" },
NO_ATOM_FEED, null,
@@ -442,7 +463,7 @@
public void testTextPattern() throws Exception {
System.out.println("\nTesting TextPattern...");
testSite("http://www.digitalmediaminute.com/",
- BlogService.TEXTPATTERN, 2,
+ new TextPattern(), 2,
new String[] {
FeedReference.ATOM_MEDIA_TYPE,
FeedReference.RSS_MEDIA_TYPE
@@ -458,7 +479,7 @@
public void testTextAmerica() throws Exception {
System.out.println("\nTesting TextAmerica...");
testSite("http://morganwebb.textamerica.com/",
- BlogService.TEXTAMERICA, 1,
+ new TextAmerica(), 1,
new String[] {
FeedReference.RSS_MEDIA_TYPE
},
@@ -469,7 +490,7 @@
HAS_RSS_FEED, "http://morganwebb.textamerica.com/rss.aspx");
testSite("http://northlan.textamerica.com/",
- BlogService.TEXTAMERICA, 1,
+ new TextAmerica(), 1,
new String[] {
FeedReference.RSS_MEDIA_TYPE
},
@@ -480,7 +501,7 @@
HAS_RSS_FEED, "http://northlan.textamerica.com/rss.aspx");
testSite("http://mycamphone.textamerica.com/",
- BlogService.TEXTAMERICA, 1,
+ new TextAmerica(), 1,
new String[] {
FeedReference.RSS_MEDIA_TYPE
},
@@ -528,11 +549,11 @@
atomFeedURL, hasRSSFeed, rssFeedURL);
}
- private void testProbeLocator(String resource, BlogService correctBlogService,
- int numberOfFeeds, String feedType[],
- String feedURL[], boolean hasAtomFeed,
- String atomFeedURL, boolean hasRSSFeed,
- String rssFeedURL) throws Exception {
+ private FeedList testProbeLocator(String resource, BlogService correctBlogService,
+ int numberOfFeeds, String feedType[],
+ String feedURL[], boolean hasAtomFeed,
+ String atomFeedURL, boolean hasRSSFeed,
+ String rssFeedURL) throws Exception {
String content = getContent(resource);
assertNotNull(content);
@@ -542,7 +563,7 @@
FeedList list = new FeedList();
ProbeLocator.locate(resource, content, list);
assertEquals(numberOfFeeds, list.size());
-
+
FeedReference[] feeds = (FeedReference[])list.toArray(new FeedReference[list.size()]);
assertEquals(numberOfFeeds, feeds.length);
@@ -552,13 +573,15 @@
assertEquals(feedType[i], feeds[i].type);
assertEquals(feedURL[i], feeds[i].resource);
}
+
+ return list;
}
- private void testFeedLocator(String resource, BlogService correctBlogService,
- int numberOfFeeds, String feedType[],
- String feedURL[], boolean hasAtomFeed,
- String atomFeedURL, boolean hasRSSFeed,
- String rssFeedURL) throws Exception {
+ private FeedList testFeedLocator(String resource, BlogService correctBlogService,
+ int numberOfFeeds, String feedType[],
+ String feedURL[], boolean hasAtomFeed,
+ String atomFeedURL, boolean hasRSSFeed,
+ String rssFeedURL) throws Exception {
FeedList list = FeedLocator.locate(resource);
FeedReference atomFeed = list.getAdAtomFeed();
FeedReference rssFeed = list.getAdRSSFeed();
@@ -577,5 +600,7 @@
}
else
assertNull(rssFeed);
+
+ return list;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commons-dev-unsubscribe@jakarta.apache.org
For additional commands, e-mail: commons-dev-help@jakarta.apache.org