You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@nutch.apache.org by Stefan Groschupf <sg...@media-style.com> on 2005/11/19 13:57:48 UTC
Re: boost documents matching a url pattern
Hi Lawrence,
please in general use the nutch user mailing list for such questions,
since may more people are interested and you will may get a faster
answer.
To answer your question, check that you add the plugin in the plugin
include section in nutch-default.xml.
In general the plugin is only used until indexing time, since it
boost the lucene documents.
If you like to see the plugin in the nutch sources, feel free to vote
the issue. :)
Cheers,
Stefan
Am 18.11.2005 um 23:44 schrieb <la...@trebnet.com>:
> Hello,
>
> I tried your http://issues.apache.org/jira/browse/NUTCH-16 boost
> documents matching a url pattern.
>
> I put the files into my "/usr/local/nutch/nutch-0.7.1/src/plugin/
> boost-urlpattern/" directory I created, I put "boost-
> urlpattern.jar", plugin.xml and the altered "urlPattern.txt" files
> in there.
>
> I added the plugin to "usr/local/nutch/nutch-0.7.1/src/plugin/
> build.xml" shown below.
>
> I also added the plugin to my "usr/local/nutch/nutch-0.7.1/conf/
> nutch-default.xml shown below.
>
> Have I missed something because when I index it does not use the
> new plugin. Do I nee to also add something under <!-- urlfilter
> plugin properties --> in nutch-default.xml? I deleted all segments
> and DB, recreated db,segments, indexed etc. then restarted Tomcat.
> It still did not use new plugin?
>
> I appreciate your help as I am stumped.
>
> Regards,
> Lawrence
>
> //------------------------build.xml--------
>
>
> <?xml version="1.0"?>
>
> <project name="Nutch" default="deploy" basedir=".">
>
> <!-- ====================================================== -->
> <!-- Build & deploy all the plugin jars. -->
> <!-- ====================================================== -->
> <target name="deploy">
> <ant dir="nutch-extensionpoints" target="deploy"/>
> <ant dir="protocol-file" target="deploy"/>
> <ant dir="protocol-ftp" target="deploy"/>
> <ant dir="protocol-http" target="deploy"/>
> <ant dir="protocol-httpclient" target="deploy"/>
> <ant dir="parse-html" target="deploy"/>
> <ant dir="parse-js" target="deploy"/>
> <ant dir="parse-text" target="deploy"/>
> <ant dir="parse-pdf" target="deploy"/>
> <ant dir="parse-rss" target="deploy"/>
> <ant dir="parse-msword" target="deploy"/>
> <!-- <ant dir="parse-mp3" target="deploy"/> -->
> <!-- <ant dir="parse-rtf" target="deploy"/> -->
> <ant dir="parse-ext" target="deploy"/>
> <ant dir="index-basic" target="deploy"/>
> <ant dir="index-more" target="deploy"/>
> <ant dir="query-basic" target="deploy"/>
> <ant dir="query-more" target="deploy"/>
> <ant dir="query-site" target="deploy"/>
> <ant dir="query-url" target="deploy"/>
> <ant dir="urlfilter-regex" target="deploy"/>
> <ant dir="urlfilter-prefix" target="deploy"/>
> <ant dir="creativecommons" target="deploy"/>
> <ant dir="languageidentifier" target="deploy"/>
> <ant dir="boost-urlpattern" target="deploy"/>
> <ant dir="clustering-carrot2" target="deploy"/>
> <ant dir="ontology" target="deploy"/>
> </target>
>
> <!-- ====================================================== -->
> <!-- Test all of the plugins. -->
> <!-- ====================================================== -->
> <target name="test">
> <ant dir="protocol-http" target="test"/>
> <ant dir="parse-html" target="test"/>
> <ant dir="parse-pdf" target="test"/>
> <ant dir="parse-rss" target="test"/>
> <ant dir="parse-msword" target="test"/>
> <!-- <ant dir="parse-mp3" target="test"/> -->
> <!-- <ant dir="parse-rtf" target="test"/> -->
> <ant dir="parse-ext" target="test"/>
> <ant dir="creativecommons" target="test"/>
> <ant dir="languageidentifier" target="test"/>
> <ant dir="boost-urlpattern" target="test"/>
> <ant dir="ontology" target="test"/>
> </target>
> "build.xml" 89L, 3839C
>
> ----------------------------------------------------------------------
> -------------------------------------------------------------
> //---------------snippet from nutch-
> default.xml-----------------------------
>
> <property>
> <name>plugin.includes</name>
> <value>nutch-extensionpoints|protocol-http|urlfilter-regex|parse-
> (text|html)|index-basic|query-(basic|site|url)|boost-urlpattern</
> value>
> <description>Regular expression naming plugin directory names to
> include. Any plugin not matching this expression is excluded.
> In any case you need at least include the nutch-extensionpoints
> plugin. By
> default Nutch includes crawling just HTML and plain text via HTTP,
> and basic indexing and search plugins.
> </description>
> </property>
>
> <property>
> <name>plugin.excludes</name>
> <value></value>
> <description>Regular expression naming plugin directory names to
> exclude.
> </description>
> </property>
>
> <property>
> <name>parser.character.encoding.default</name>
> <value>windows-1252</value>
> <description>The character encoding to fall back to when no other
> information
> is available</description>
> </property>
>
> <property>
> <name>parser.html.impl</name>
> <value>neko</value>
> <description>HTML Parser implementation. Currently the following
> keywords
> are recognized: "neko" uses NekoHTML, "tagsoup" uses TagSoup.
> </description>
> </property>
>
> <!-- urlfilter plugin properties -->
>
> <property>
> <name>urlfilter.regex.file</name>
> <value>regex-urlfilter.txt</value>
>
>
>
>
---------------------------------------------------------------
company: http://www.media-style.com
forum: http://www.text-mining.org
blog: http://www.find23.net