You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by jn...@apache.org on 2010/06/30 12:36:29 UTC
svn commit: r959259 [1/12] - in /nutch/branches/nutchbase: ./ bin/ conf/
contrib/ docs/ ivy/ lib/ lib/jetty-ext/ src/engines/ src/gora/ src/java/
src/java/org/apache/nutch/analysis/ src/java/org/apache/nutch/clustering/
src/java/org/apache/nutch/crawl/...
Author: jnioche
Date: Wed Jun 30 10:36:20 2010
New Revision: 959259
URL: http://svn.apache.org/viewvc?rev=959259&view=rev
Log:
updated from http://github.com/dogacan/nutchbase. See patch on NUTCH-650
Added:
nutch/branches/nutchbase/conf/gora.properties
nutch/branches/nutchbase/eclipse-codeformat.xml
nutch/branches/nutchbase/ivy/
nutch/branches/nutchbase/ivy/ivy-2.1.0.jar (with props)
nutch/branches/nutchbase/ivy/ivy-configurations.xml
nutch/branches/nutchbase/ivy/ivy.xml
nutch/branches/nutchbase/ivy/ivysettings.xml
nutch/branches/nutchbase/ivy/libraries.properties
nutch/branches/nutchbase/lib/hbase-0.20.3-dev-test.jar (with props)
nutch/branches/nutchbase/lib/hbase-0.20.3.jar (with props)
nutch/branches/nutchbase/lib/zookeeper-3.2.1.jar (with props)
nutch/branches/nutchbase/src/gora/
nutch/branches/nutchbase/src/gora/webpage.avsc
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/CrawlStatus.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/DbUpdateMapper.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/DbUpdateReducer.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/GeneratorJob.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/InjectorJob.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/URLPartitioner.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/URLWebPage.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/WebTableReader.java
nutch/branches/nutchbase/src/java/org/apache/nutch/fetcher/FetcherJob.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerJob.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrIndexerJob.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrMappingReader.java
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatusCodes.java
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatusUtils.java
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserChecker.java
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserJob.java
nutch/branches/nutchbase/src/java/org/apache/nutch/plugin/FieldPluggable.java
nutch/branches/nutchbase/src/java/org/apache/nutch/protocol/ProtocolStatusCodes.java
nutch/branches/nutchbase/src/java/org/apache/nutch/storage/
nutch/branches/nutchbase/src/java/org/apache/nutch/storage/Mark.java
nutch/branches/nutchbase/src/java/org/apache/nutch/storage/ParseStatus.java
nutch/branches/nutchbase/src/java/org/apache/nutch/storage/ProtocolStatus.java
nutch/branches/nutchbase/src/java/org/apache/nutch/storage/ProtocolStatusUtils.java
nutch/branches/nutchbase/src/java/org/apache/nutch/storage/StorageUtils.java
nutch/branches/nutchbase/src/java/org/apache/nutch/storage/WebPage.java
nutch/branches/nutchbase/src/java/org/apache/nutch/storage/WebTableCreator.java
nutch/branches/nutchbase/src/java/org/apache/nutch/util/IdentityPageReducer.java
nutch/branches/nutchbase/src/java/org/apache/nutch/util/Pair.java
nutch/branches/nutchbase/src/java/org/apache/nutch/util/TableUtil.java
nutch/branches/nutchbase/src/java/org/apache/nutch/util/WebPageWritable.java
nutch/branches/nutchbase/src/plugin/creativecommons/ivy.xml
nutch/branches/nutchbase/src/plugin/feed/ivy.xml
nutch/branches/nutchbase/src/plugin/field-basic/ivy.xml
nutch/branches/nutchbase/src/plugin/field-boost/ivy.xml
nutch/branches/nutchbase/src/plugin/index-anchor/ivy.xml
nutch/branches/nutchbase/src/plugin/index-basic/ivy.xml
nutch/branches/nutchbase/src/plugin/index-more/ivy.xml
nutch/branches/nutchbase/src/plugin/ivy/
nutch/branches/nutchbase/src/plugin/ivy/ivy.xml
nutch/branches/nutchbase/src/plugin/languageidentifier/ivy.xml
nutch/branches/nutchbase/src/plugin/lib-http/ivy.xml
nutch/branches/nutchbase/src/plugin/lib-nekohtml/ivy.xml
nutch/branches/nutchbase/src/plugin/lib-regex-filter/ivy.xml
nutch/branches/nutchbase/src/plugin/lib-xml/ivy.xml
nutch/branches/nutchbase/src/plugin/microformats-reltag/ivy.xml
nutch/branches/nutchbase/src/plugin/nutch-extensionpoints/ivy.xml
nutch/branches/nutchbase/src/plugin/parse-ext/ivy.xml
nutch/branches/nutchbase/src/plugin/parse-html/ivy.xml
nutch/branches/nutchbase/src/plugin/parse-js/ivy.xml
nutch/branches/nutchbase/src/plugin/parse-rss/ivy.xml
nutch/branches/nutchbase/src/plugin/parse-swf/ivy.xml
nutch/branches/nutchbase/src/plugin/parse-tika/
nutch/branches/nutchbase/src/plugin/parse-tika/build-ivy.xml
nutch/branches/nutchbase/src/plugin/parse-tika/build.xml
nutch/branches/nutchbase/src/plugin/parse-tika/ivy.xml
nutch/branches/nutchbase/src/plugin/parse-tika/plugin.xml
nutch/branches/nutchbase/src/plugin/parse-tika/sample/
nutch/branches/nutchbase/src/plugin/parse-tika/sample/encrypted.pdf (with props)
nutch/branches/nutchbase/src/plugin/parse-tika/sample/nutch.html
nutch/branches/nutchbase/src/plugin/parse-tika/sample/ootest.odt (with props)
nutch/branches/nutchbase/src/plugin/parse-tika/sample/ootest.sxw (with props)
nutch/branches/nutchbase/src/plugin/parse-tika/sample/ootest.txt
nutch/branches/nutchbase/src/plugin/parse-tika/sample/pdftest.pdf
nutch/branches/nutchbase/src/plugin/parse-tika/sample/test.rtf
nutch/branches/nutchbase/src/plugin/parse-tika/sample/word97.doc (with props)
nutch/branches/nutchbase/src/plugin/parse-tika/src/
nutch/branches/nutchbase/src/plugin/parse-tika/src/java/
nutch/branches/nutchbase/src/plugin/parse-tika/src/java/org/
nutch/branches/nutchbase/src/plugin/parse-tika/src/java/org/apache/
nutch/branches/nutchbase/src/plugin/parse-tika/src/java/org/apache/nutch/
nutch/branches/nutchbase/src/plugin/parse-tika/src/java/org/apache/nutch/parse/
nutch/branches/nutchbase/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/
nutch/branches/nutchbase/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMBuilder.java
nutch/branches/nutchbase/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMContentUtils.java
nutch/branches/nutchbase/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/HTMLMetaProcessor.java
nutch/branches/nutchbase/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaConfig.java
nutch/branches/nutchbase/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
nutch/branches/nutchbase/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/XMLCharacterRecognizer.java
nutch/branches/nutchbase/src/plugin/parse-tika/src/test/
nutch/branches/nutchbase/src/plugin/parse-tika/src/test/org/
nutch/branches/nutchbase/src/plugin/parse-tika/src/test/org/apache/
nutch/branches/nutchbase/src/plugin/parse-tika/src/test/org/apache/nutch/
nutch/branches/nutchbase/src/plugin/parse-tika/src/test/org/apache/nutch/tika/
nutch/branches/nutchbase/src/plugin/parse-tika/src/test/org/apache/nutch/tika/TestMSWordParser.java
nutch/branches/nutchbase/src/plugin/parse-tika/src/test/org/apache/nutch/tika/TestOOParser.java
nutch/branches/nutchbase/src/plugin/parse-tika/src/test/org/apache/nutch/tika/TestPdfParser.java
nutch/branches/nutchbase/src/plugin/parse-tika/src/test/org/apache/nutch/tika/TestRTFParser.java
nutch/branches/nutchbase/src/plugin/parse-zip/ivy.xml
nutch/branches/nutchbase/src/plugin/protocol-file/ivy.xml
nutch/branches/nutchbase/src/plugin/protocol-file/sample/
nutch/branches/nutchbase/src/plugin/protocol-file/sample/testprotocolfile.txt
nutch/branches/nutchbase/src/plugin/protocol-ftp/ivy.xml
nutch/branches/nutchbase/src/plugin/protocol-http/ivy.xml
nutch/branches/nutchbase/src/plugin/protocol-httpclient/ivy.xml
nutch/branches/nutchbase/src/plugin/scoring-link/ivy.xml
nutch/branches/nutchbase/src/plugin/scoring-opic/ivy.xml
nutch/branches/nutchbase/src/plugin/subcollection/ivy.xml
nutch/branches/nutchbase/src/plugin/tld/ivy.xml
nutch/branches/nutchbase/src/plugin/urlfilter-automaton/ivy.xml
nutch/branches/nutchbase/src/plugin/urlfilter-domain/ivy.xml
nutch/branches/nutchbase/src/plugin/urlfilter-prefix/ivy.xml
nutch/branches/nutchbase/src/plugin/urlfilter-regex/ivy.xml
nutch/branches/nutchbase/src/plugin/urlfilter-suffix/ivy.xml
nutch/branches/nutchbase/src/plugin/urlfilter-validator/ivy.xml
nutch/branches/nutchbase/src/plugin/urlnormalizer-basic/ivy.xml
nutch/branches/nutchbase/src/plugin/urlnormalizer-pass/ivy.xml
nutch/branches/nutchbase/src/plugin/urlnormalizer-regex/ivy.xml
nutch/branches/nutchbase/src/test/hbase-site.xml
Removed:
nutch/branches/nutchbase/conf/hadoop-env.sh.template
nutch/branches/nutchbase/conf/hadoop-site.xml.template
nutch/branches/nutchbase/conf/masters
nutch/branches/nutchbase/conf/slaves.template
nutch/branches/nutchbase/contrib/
nutch/branches/nutchbase/docs/
nutch/branches/nutchbase/index.html
nutch/branches/nutchbase/lib/apache-solr-common-1.3.0.jar
nutch/branches/nutchbase/lib/apache-solr-solrj-1.3.0.jar
nutch/branches/nutchbase/lib/commons-beanutils-1.8.0.jar
nutch/branches/nutchbase/lib/commons-cli-2.0-SNAPSHOT.jar
nutch/branches/nutchbase/lib/commons-codec-1.3.jar
nutch/branches/nutchbase/lib/commons-collections-3.2.1.jar
nutch/branches/nutchbase/lib/commons-httpclient-3.0.1.jar
nutch/branches/nutchbase/lib/commons-lang-2.1.jar
nutch/branches/nutchbase/lib/commons-logging-1.0.4.jar
nutch/branches/nutchbase/lib/commons-logging-api-1.0.4.jar
nutch/branches/nutchbase/lib/hadoop-0.20.0-core.jar
nutch/branches/nutchbase/lib/hbase-0.20.0-r804408.jar
nutch/branches/nutchbase/lib/icu4j-4_0_1.LICENSE.txt
nutch/branches/nutchbase/lib/icu4j-4_0_1.jar
nutch/branches/nutchbase/lib/jakarta-oro-2.0.8.jar
nutch/branches/nutchbase/lib/jets3t-0.6.1.LICENSE.txt
nutch/branches/nutchbase/lib/jets3t-0.6.1.jar
nutch/branches/nutchbase/lib/jetty-5.1.4.LICENSE.txt
nutch/branches/nutchbase/lib/jetty-5.1.4.jar
nutch/branches/nutchbase/lib/jetty-ext/
nutch/branches/nutchbase/lib/junit-3.8.1.LICENSE.txt
nutch/branches/nutchbase/lib/junit-3.8.1.jar
nutch/branches/nutchbase/lib/log4j-1.2.15.jar
nutch/branches/nutchbase/lib/lucene-core-2.4.0.jar
nutch/branches/nutchbase/lib/lucene-misc-2.4.0.jar
nutch/branches/nutchbase/lib/servlet-api.jar
nutch/branches/nutchbase/lib/tika-0.1-incubating.jar
nutch/branches/nutchbase/lib/xerces-2_6_2-apis.jar
nutch/branches/nutchbase/lib/xerces-2_6_2.jar
nutch/branches/nutchbase/lib/zookeeper-r785019-hbase-1329.jar
nutch/branches/nutchbase/src/engines/
nutch/branches/nutchbase/src/java/org/apache/nutch/analysis/
nutch/branches/nutchbase/src/java/org/apache/nutch/clustering/
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/CrawlDatumHbase.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/Generator.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/Injector.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/PartitionSelectorByHost.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/TableUpdateMapper.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/TableUpdateReducer.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/TableUpdater.java
nutch/branches/nutchbase/src/java/org/apache/nutch/fetcher/Fetcher.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/DeleteDuplicates.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/FsDirectory.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/HighFreqTerms.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexMerger.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexSorter.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/Indexer.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/NutchSimilarity.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/field/
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/lucene/
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrIndexer.java
nutch/branches/nutchbase/src/java/org/apache/nutch/ontology/
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/TableParser.java
nutch/branches/nutchbase/src/java/org/apache/nutch/plugin/TablePluggable.java
nutch/branches/nutchbase/src/java/org/apache/nutch/scoring/webgraph/
nutch/branches/nutchbase/src/java/org/apache/nutch/searcher/
nutch/branches/nutchbase/src/java/org/apache/nutch/segment/
nutch/branches/nutchbase/src/java/org/apache/nutch/servlet/
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/SearchLoadTester.java
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/compat/
nutch/branches/nutchbase/src/java/org/apache/nutch/util/hbase/
nutch/branches/nutchbase/src/plugin/analysis-de/build.xml
nutch/branches/nutchbase/src/plugin/analysis-de/plugin.xml
nutch/branches/nutchbase/src/plugin/analysis-de/src/java/org/
nutch/branches/nutchbase/src/plugin/analysis-fr/build.xml
nutch/branches/nutchbase/src/plugin/analysis-fr/plugin.xml
nutch/branches/nutchbase/src/plugin/analysis-fr/src/java/org/
nutch/branches/nutchbase/src/plugin/clustering-carrot2/
nutch/branches/nutchbase/src/plugin/feed/lib/rome-0.9.jar
nutch/branches/nutchbase/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageQueryFilter.java
nutch/branches/nutchbase/src/plugin/lib-jakarta-poi/
nutch/branches/nutchbase/src/plugin/lib-lucene-analyzers/
nutch/branches/nutchbase/src/plugin/lib-nekohtml/lib/nekohtml-0.9.4.jar
nutch/branches/nutchbase/src/plugin/lib-parsems/
nutch/branches/nutchbase/src/plugin/lib-regex-filter/sample/
nutch/branches/nutchbase/src/plugin/lib-xml/lib/jdom.jar
nutch/branches/nutchbase/src/plugin/lib-xml/lib/xercesImpl.jar
nutch/branches/nutchbase/src/plugin/ontology/
nutch/branches/nutchbase/src/plugin/parse-html/lib/tagsoup-1.2.jar
nutch/branches/nutchbase/src/plugin/parse-mp3/
nutch/branches/nutchbase/src/plugin/parse-msexcel/
nutch/branches/nutchbase/src/plugin/parse-mspowerpoint/
nutch/branches/nutchbase/src/plugin/parse-msword/
nutch/branches/nutchbase/src/plugin/parse-oo/
nutch/branches/nutchbase/src/plugin/parse-pdf/
nutch/branches/nutchbase/src/plugin/parse-rss/lib/xmlrpc-1.2.jar
nutch/branches/nutchbase/src/plugin/parse-rtf/
nutch/branches/nutchbase/src/plugin/parse-text/
nutch/branches/nutchbase/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/testprotocolfile.txt
nutch/branches/nutchbase/src/plugin/protocol-ftp/lib/
nutch/branches/nutchbase/src/plugin/protocol-http/src/test/
nutch/branches/nutchbase/src/plugin/query-basic/build.xml
nutch/branches/nutchbase/src/plugin/query-basic/plugin.xml
nutch/branches/nutchbase/src/plugin/query-basic/src/java/org/
nutch/branches/nutchbase/src/plugin/query-custom/build.xml
nutch/branches/nutchbase/src/plugin/query-custom/plugin.xml
nutch/branches/nutchbase/src/plugin/query-custom/src/java/org/
nutch/branches/nutchbase/src/plugin/query-more/build.xml
nutch/branches/nutchbase/src/plugin/query-more/plugin.xml
nutch/branches/nutchbase/src/plugin/query-more/src/java/org/
nutch/branches/nutchbase/src/plugin/query-site/build.xml
nutch/branches/nutchbase/src/plugin/query-site/plugin.xml
nutch/branches/nutchbase/src/plugin/query-site/src/java/org/
nutch/branches/nutchbase/src/plugin/query-url/build.xml
nutch/branches/nutchbase/src/plugin/query-url/plugin.xml
nutch/branches/nutchbase/src/plugin/query-url/src/java/org/
nutch/branches/nutchbase/src/plugin/query-url/src/test/org/
nutch/branches/nutchbase/src/plugin/response-json/
nutch/branches/nutchbase/src/plugin/response-xml/
nutch/branches/nutchbase/src/plugin/summary-basic/
nutch/branches/nutchbase/src/plugin/summary-lucene/
nutch/branches/nutchbase/src/test/org/apache/nutch/analysis/
nutch/branches/nutchbase/src/test/org/apache/nutch/clustering/
nutch/branches/nutchbase/src/test/org/apache/nutch/db/
nutch/branches/nutchbase/src/test/org/apache/nutch/fetcher/TestFetcher.java
nutch/branches/nutchbase/src/test/org/apache/nutch/indexer/TestDeleteDuplicates.java
nutch/branches/nutchbase/src/test/org/apache/nutch/indexer/TestIndexSorter.java
nutch/branches/nutchbase/src/test/org/apache/nutch/ontology/
nutch/branches/nutchbase/src/test/org/apache/nutch/searcher/
nutch/branches/nutchbase/src/test/org/apache/nutch/tools/
nutch/branches/nutchbase/src/web/
nutch/branches/nutchbase/src/xmlcatalog/
Modified:
nutch/branches/nutchbase/bin/nutch
nutch/branches/nutchbase/build.xml
nutch/branches/nutchbase/conf/log4j.properties
nutch/branches/nutchbase/conf/nutch-default.xml
nutch/branches/nutchbase/conf/parse-plugins.xml
nutch/branches/nutchbase/conf/schema.xml
nutch/branches/nutchbase/conf/tika-mimetypes.xml
nutch/branches/nutchbase/default.properties
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/AbstractFetchSchedule.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/DefaultFetchSchedule.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/FetchSchedule.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/GeneratorMapper.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/GeneratorReducer.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/MD5Signature.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/NutchWritable.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/Signature.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/SignatureFactory.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/TextProfileSignature.java
nutch/branches/nutchbase/src/java/org/apache/nutch/fetcher/FetchEntry.java
nutch/branches/nutchbase/src/java/org/apache/nutch/fetcher/FetcherReducer.java
nutch/branches/nutchbase/src/java/org/apache/nutch/fetcher/PartitionUrlByHost.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerOutputFormat.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexerReducer.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexingFilter.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/IndexingFilters.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrDeleteDuplicates.java
nutch/branches/nutchbase/src/java/org/apache/nutch/indexer/solr/SolrWriter.java
nutch/branches/nutchbase/src/java/org/apache/nutch/metadata/HttpHeaders.java
nutch/branches/nutchbase/src/java/org/apache/nutch/metadata/Nutch.java
nutch/branches/nutchbase/src/java/org/apache/nutch/net/URLNormalizers.java
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/HtmlParseFilter.java
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/HtmlParseFilters.java
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/Parse.java
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseStatus.java
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParseUtil.java
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/Parser.java
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserFactory.java
nutch/branches/nutchbase/src/java/org/apache/nutch/plugin/ExtensionPoint.java
nutch/branches/nutchbase/src/java/org/apache/nutch/plugin/Pluggable.java
nutch/branches/nutchbase/src/java/org/apache/nutch/protocol/Protocol.java
nutch/branches/nutchbase/src/java/org/apache/nutch/protocol/ProtocolFactory.java
nutch/branches/nutchbase/src/java/org/apache/nutch/protocol/ProtocolOutput.java
nutch/branches/nutchbase/src/java/org/apache/nutch/protocol/ProtocolStatus.java
nutch/branches/nutchbase/src/java/org/apache/nutch/scoring/ScoringFilter.java
nutch/branches/nutchbase/src/java/org/apache/nutch/scoring/ScoringFilters.java
nutch/branches/nutchbase/src/java/org/apache/nutch/util/EncodingDetector.java
nutch/branches/nutchbase/src/java/org/apache/nutch/util/MimeUtil.java
nutch/branches/nutchbase/src/java/org/apache/nutch/util/NutchJob.java
nutch/branches/nutchbase/src/java/overview.html
nutch/branches/nutchbase/src/plugin/build-plugin.xml
nutch/branches/nutchbase/src/plugin/build.xml
nutch/branches/nutchbase/src/plugin/index-anchor/src/java/org/apache/nutch/indexer/anchor/AnchorIndexingFilter.java
nutch/branches/nutchbase/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
nutch/branches/nutchbase/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
nutch/branches/nutchbase/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java
nutch/branches/nutchbase/src/plugin/languageidentifier/plugin.xml
nutch/branches/nutchbase/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java
nutch/branches/nutchbase/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIdentifier.java
nutch/branches/nutchbase/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java
nutch/branches/nutchbase/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/NGramProfile.java
nutch/branches/nutchbase/src/plugin/languageidentifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java
nutch/branches/nutchbase/src/plugin/languageidentifier/src/test/org/apache/nutch/analysis/lang/TestLanguageIdentifier.java
nutch/branches/nutchbase/src/plugin/languageidentifier/src/test/org/apache/nutch/analysis/lang/TestNGramProfile.java
nutch/branches/nutchbase/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
nutch/branches/nutchbase/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java
nutch/branches/nutchbase/src/plugin/lib-nekohtml/build.xml
nutch/branches/nutchbase/src/plugin/lib-xml/build.xml
nutch/branches/nutchbase/src/plugin/nutch-extensionpoints/build.xml
nutch/branches/nutchbase/src/plugin/nutch-extensionpoints/plugin.xml
nutch/branches/nutchbase/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java
nutch/branches/nutchbase/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java
nutch/branches/nutchbase/src/plugin/parse-rss/src/java/org/apache/nutch/parse/rss/RSSParser.java
nutch/branches/nutchbase/src/plugin/parse-rss/src/test/org/apache/nutch/parse/rss/TestRSSParser.java
nutch/branches/nutchbase/src/plugin/protocol-file/build.xml
nutch/branches/nutchbase/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
nutch/branches/nutchbase/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
nutch/branches/nutchbase/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java
nutch/branches/nutchbase/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
nutch/branches/nutchbase/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java
nutch/branches/nutchbase/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java
nutch/branches/nutchbase/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
nutch/branches/nutchbase/src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java
nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java
nutch/branches/nutchbase/src/test/log4j.properties
nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java
nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/TestGenerator.java
nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/TestInjector.java
nutch/branches/nutchbase/src/test/org/apache/nutch/indexer/TestIndexingFilters.java
nutch/branches/nutchbase/src/test/org/apache/nutch/parse/TestParserFactory.java
nutch/branches/nutchbase/src/test/org/apache/nutch/parse/parse-plugin-test.xml
nutch/branches/nutchbase/src/test/org/apache/nutch/protocol/TestContent.java
nutch/branches/nutchbase/src/test/org/apache/nutch/util/TestEncodingDetector.java
nutch/branches/nutchbase/src/test/org/apache/nutch/util/TestTableUtil.java
Modified: nutch/branches/nutchbase/bin/nutch
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/bin/nutch?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/bin/nutch (original)
+++ nutch/branches/nutchbase/bin/nutch Wed Jun 30 10:36:20 2010
@@ -33,28 +33,14 @@ if [ $# = 0 ]; then
echo "Usage: nutch [-core] COMMAND"
echo "where COMMAND is one of:"
# echo " crawl one-step crawler for intranets"
-# echo " readdb read / dump crawl db"
-# echo " convdb convert crawl db from pre-0.9 format"
-# echo " mergedb merge crawldb-s, with optional filtering"
-# echo " readlinkdb read / dump link db"
- echo " createtable create a new webtable in hbase"
echo " inject inject new urls into the database"
echo " generate generate new segments to fetch from crawl db"
-# echo " freegen generate new segments to fetch from text files"
echo " fetch fetch URLs marked during generate"
echo " parse parse URLs marked during fetch"
-# echo " readseg read / dump segment data"
-# echo " mergesegs merge several segments, with optional filtering and slicing"
-# echo " updatedb update crawl db from segments after fetching"
-# echo " invertlinks create a linkdb from parsed segments"
-# echo " mergelinkdb merge linkdb-s, with optional filtering"
- echo " index run the indexer on parsed segments and linkdb"
+ echo " updatedb update web table after parsing"
echo " solrindex run the solr indexer on parsed segments and linkdb"
- echo " merge merge several segment indexes"
- echo " dedup remove duplicates from a set of segment indexes"
echo " solrdedup remove duplicates from solr"
echo " plugin load a plugin and run one of its classes main()"
- echo " server run a search server"
echo " or"
echo " CLASSNAME run the class named CLASSNAME"
echo "Most commands print help when invoked w/o parameters."
@@ -136,6 +122,11 @@ if [ -d "$NUTCH_HOME/plugins" ]; then
fi
# add libs to CLASSPATH
+for f in $NUTCH_HOME/build/lib/*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+done
+
+# add libs to CLASSPAT for binary distribution
for f in $NUTCH_HOME/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
@@ -197,53 +188,26 @@ fi
# figure out which class to run
if [ "$COMMAND" = "crawl" ] ; then
CLASS=org.apache.nutch.crawl.Crawl
-elif [ "$COMMAND" = "createtable" ] ; then
- CLASS=org.apache.nutch.util.hbase.WebTableCreator
elif [ "$COMMAND" = "inject" ] ; then
- CLASS=org.apache.nutch.crawl.Injector
+ CLASS=org.apache.nutch.crawl.InjectorJob
elif [ "$COMMAND" = "generate" ] ; then
- CLASS=org.apache.nutch.crawl.Generator
+ CLASS=org.apache.nutch.crawl.GeneratorJob
#elif [ "$COMMAND" = "freegen" ] ; then
# CLASS=org.apache.nutch.tools.FreeGenerator
elif [ "$COMMAND" = "fetch" ] ; then
- CLASS=org.apache.nutch.fetcher.Fetcher
+ CLASS=org.apache.nutch.fetcher.FetcherJob
elif [ "$COMMAND" = "parse" ] ; then
- CLASS=org.apache.nutch.parse.TableParser
-#elif [ "$COMMAND" = "readdb" ] ; then
-# CLASS=org.apache.nutch.crawl.CrawlDbReader
-#elif [ "$COMMAND" = "convdb" ] ; then
-# CLASS=org.apache.nutch.tools.compat.CrawlDbConverter
-#elif [ "$COMMAND" = "mergedb" ] ; then
-# CLASS=org.apache.nutch.crawl.CrawlDbMerger
-#elif [ "$COMMAND" = "readlinkdb" ] ; then
-# CLASS=org.apache.nutch.crawl.LinkDbReader
-#elif [ "$COMMAND" = "readseg" ] ; then
-# CLASS=org.apache.nutch.segment.SegmentReader
-#elif [ "$COMMAND" = "segread" ] ; then
-# echo "[DEPRECATED] Command 'segread' is deprecated, use 'readseg' instead."
-# CLASS=org.apache.nutch.segment.SegmentReader
-#elif [ "$COMMAND" = "mergesegs" ] ; then
-# CLASS=org.apache.nutch.segment.SegmentMerger
-elif [ "$COMMAND" = "updatetable" ] ; then
- CLASS=org.apache.nutch.crawl.TableUpdater
-#elif [ "$COMMAND" = "invertlinks" ] ; then
-# CLASS=org.apache.nutch.crawl.LinkDb
-#elif [ "$COMMAND" = "mergelinkdb" ] ; then
-# CLASS=org.apache.nutch.crawl.LinkDbMerger
-elif [ "$COMMAND" = "index" ] ; then
- CLASS=org.apache.nutch.indexer.Indexer
+ CLASS=org.apache.nutch.parse.ParserJob
+elif [ "$COMMAND" = "updatedb" ] ; then
+ CLASS=org.apache.nutch.crawl.DbUpdaterJob
elif [ "$COMMAND" = "solrindex" ] ; then
- CLASS=org.apache.nutch.indexer.solr.SolrIndexer
-elif [ "$COMMAND" = "dedup" ] ; then
- CLASS=org.apache.nutch.indexer.DeleteDuplicates
+ CLASS=org.apache.nutch.indexer.solr.SolrIndexerJob
elif [ "$COMMAND" = "solrdedup" ] ; then
CLASS=org.apache.nutch.indexer.solr.SolrDeleteDuplicates
elif [ "$COMMAND" = "merge" ] ; then
CLASS=org.apache.nutch.indexer.IndexMerger
elif [ "$COMMAND" = "plugin" ] ; then
CLASS=org.apache.nutch.plugin.PluginRepository
-elif [ "$COMMAND" = "server" ] ; then
- CLASS='org.apache.nutch.searcher.DistributedSearch$Server'
else
CLASS=$COMMAND
fi
Modified: nutch/branches/nutchbase/build.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/build.xml?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/build.xml (original)
+++ nutch/branches/nutchbase/build.xml Wed Jun 30 10:36:20 2010
@@ -15,7 +15,7 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project name="Nutch" default="job">
+<project name="Nutch" default="job" xmlns:ivy="antlib:org.apache.ivy.ant">
<!-- Load all the default properties, and any the user wants -->
<!-- to contribute (without having to type -D or edit this file -->
@@ -23,11 +23,12 @@
<property file="${basedir}/build.properties" />
<property file="${basedir}/default.properties" />
<property name="test.junit.output.format" value="plain"/>
+ <loadproperties srcfile="${ivy.dir}/libraries.properties"/>
<!-- the normal classpath -->
<path id="classpath">
<pathelement location="${build.classes}"/>
- <fileset dir="${lib.dir}">
+ <fileset dir="${build.lib.dir}">
<include name="*.jar" />
</fileset>
</path>
@@ -41,18 +42,15 @@
<pathelement location="${plugins.classpath.dir}"/>
<path refid="classpath"/>
<pathelement location="${build.dir}/${final.name}.job" />
+ <fileset dir="${build.lib.dir}">
+ <include name="*.jar" />
+ </fileset>
</path>
- <!-- xmlcatalog definition for xslt task -->
- <xmlcatalog id="docDTDs">
- <dtd publicId="-//W3C//DTD XHTML 1.0 Transitional//EN"
- location="${xmlcatalog.dir}/xhtml1-transitional.dtd"/>
- </xmlcatalog>
-
<!-- ====================================================== -->
<!-- Stuff needed by all targets -->
<!-- ====================================================== -->
- <target name="init">
+ <target name="init" depends="ivy-init">
<mkdir dir="${build.dir}"/>
<mkdir dir="${build.classes}"/>
@@ -67,11 +65,13 @@
<fileset dir="${conf.dir}" includes="**/*.template"/>
<mapper type="glob" from="*.template" to="*"/>
</copy>
+ </target>
+ <target name="extract-hadoop" depends="ivy-init">
<!-- unpack hadoop scripts from hadoop jar into bin directory -->
<mkdir dir="${build.dir}/hadoop"/>
<unjar dest="${build.dir}/hadoop">
- <fileset dir="${lib.dir}" includes="hadoop*.jar"/>
+ <fileset dir="${build.lib.dir}" includes="hadoop*.jar"/>
<patternset includes="bin.tgz"/>
</unjar>
@@ -80,21 +80,14 @@
<replace file="bin/hadoop" token="PlatformName" value="PlatformName | sed -e 's/ /_/g'"/>
<chmod dir="bin" perm="ugo+rx" includes="*.sh,hadoop"/>
- <!-- unpack hadoop webapp from hadoop jar into build directory -->
- <mkdir dir="${build.dir}/webapps"/>
- <unjar dest="${build.dir}">
- <fileset dir="${lib.dir}" includes="hadoop*.jar"/>
- <patternset includes="webapps/**"/>
- </unjar>
-
</target>
<!-- ====================================================== -->
<!-- Compile the Java files -->
<!-- ====================================================== -->
- <target name="compile" depends="compile-core, compile-plugins"/>
+ <target name="compile" depends="compile-core, compile-plugins, extract-hadoop"/>
- <target name="compile-core" depends="init">
+ <target name="compile-core" depends="init, resolve-default">
<javac
encoding="${build.encoding}"
srcdir="${src.dir}"
@@ -114,18 +107,6 @@
<ant dir="src/plugin" target="deploy" inheritAll="false"/>
</target>
- <target name="generate-src" depends="init">
- <javacc target="${src.dir}/org/apache/nutch/analysis/NutchAnalysis.jj"
- javacchome="${javacc.home}">
- </javacc>
-
- <fixcrlf srcdir="${src.dir}" eol="lf" includes="**/*.java"/>
-
- </target>
-
- <target name="dynamic" depends="generate-src, compile">
- </target>
-
<!-- ================================================================== -->
<!-- Make nutch.jar -->
<!-- ================================================================== -->
@@ -152,69 +133,17 @@
<jar jarfile="${build.dir}/${final.name}.job">
<zipfileset dir="${build.classes}"/>
<zipfileset dir="${conf.dir}" excludes="*.template,hadoop*.*"/>
- <zipfileset dir="${lib.dir}" prefix="lib"
+ <zipfileset dir="${build.lib.dir}" prefix="lib"
includes="**/*.jar" excludes="hadoop-*.jar"/>
<zipfileset dir="${build.plugins}" prefix="plugins"/>
</jar>
</target>
- <!-- ================================================================== -->
- <!-- Make nutch.war -->
- <!-- ================================================================== -->
- <!-- -->
- <!-- ================================================================== -->
- <target name="war" depends="jar,compile,generate-docs">
-
- <!-- generate the nutch.xml (servlet context) file -->
- <xslt in="${basedir}/conf/nutch-default.xml"
- out="${build.dir}/nutch.xml"
- style="${basedir}/conf/context.xsl">
- <xmlcatalog refid="docDTDs"/>
- <outputproperty name="indent" value="yes"/>
- </xslt>
- <war destfile="${build.dir}/${final.name}.war"
- webxml="${web.src.dir}/web.xml">
- <fileset dir="${web.src.dir}/jsp"/>
- <zipfileset dir="${docs.src}" includes="include/*.html"/>
- <zipfileset dir="${build.docs}" includes="*/include/*.html"/>
- <fileset dir="${docs.dir}"/>
- <lib dir="${lib.dir}">
- <include name="lucene*.jar"/>
- <include name="taglibs-*.jar"/>
- <include name="hadoop-*.jar"/>
- <include name="hbase-*.jar"/>
- <include name="zookeeper-*.jar"/>
- <include name="dom4j-*.jar"/>
- <include name="xerces-*.jar"/>
- <include name="tika-*.jar"/>
- <include name="apache-solr-*.jar"/>
- <include name="commons-httpclient-*.jar"/>
- <include name="commons-codec-*.jar"/>
- <include name="commons-collections-*.jar"/>
- <include name="commons-beanutils-*.jar"/>
- <include name="commons-cli-*.jar"/>
- <include name="commons-lang-*.jar"/>
- <include name="commons-logging-*.jar"/>
- <include name="log4j-*.jar"/>
- </lib>
- <lib dir="${build.dir}">
- <include name="${final.name}.jar"/>
- </lib>
- <classes dir="${conf.dir}" excludes="**/*.template"/>
- <classes dir="${web.src.dir}/locale"/>
- <classes file="${web.src.dir}/log4j.properties"/>
- <zipfileset prefix="WEB-INF/classes/plugins" dir="${build.plugins}"/>
- <webinf dir="${lib.dir}">
- <include name="taglibs-*.tld"/>
- </webinf>
- </war>
- </target>
-
<!-- ================================================================== -->
<!-- Compile test code -->
<!-- ================================================================== -->
- <target name="compile-core-test" depends="compile-core">
+ <target name="compile-core-test" depends="compile-core, resolve-test">
<javac
encoding="${build.encoding}"
srcdir="${test.src.dir}"
@@ -231,45 +160,6 @@
</target>
<!-- ================================================================== -->
- <!-- Run code checks (PMD) -->
- <!-- ================================================================== -->
- <target name="pmd" depends="compile">
- <property name="pmd.report" location="${build.dir}/pmd-report.html" />
- <taskdef name="pmd" classname="net.sourceforge.pmd.ant.PMDTask">
- <classpath>
- <fileset dir="${lib.dir}">
- <include name="pmd-ext/*.jar" />
- <include name="xerces*.jar" />
- </fileset>
- </classpath>
- </taskdef>
- <pmd shortFilenames="true" failonerror="true" failOnRuleViolation="false"
- encoding="${build.encoding}" failuresPropertyName="pmd.failures">
- <ruleset>unusedcode</ruleset>
- <!--ruleset>basic</ruleset-->
- <!--ruleset>optimizations</ruleset-->
- <formatter type="html" toFile="${pmd.report}" />
- <!-- <formatter type="xml" toFile="${tempbuild}/$report_pmd.xml"/> -->
- <fileset dir="${basedir}/src">
- <include name="java/**/*.java"/>
- <include name="plugin/**/*.java"/>
- <!-- Exclude generated sources -->
- <exclude name="**/NutchAnalysis.java" />
- <exclude name="**/NutchAnalysisTokenManager.java" />
- </fileset>
- </pmd>
- <condition property="pmd.stop" value="true">
- <and>
- <isset property="pmd.failures" />
- <not>
- <equals arg1="0" arg2="${pmd.failures}" trim="true" />
- </not>
- </and>
- </condition>
- <fail if="pmd.stop">FAILURE: PMD shows ${pmd.failures} rule violations. See ${pmd.report} for details.</fail>
- </target>
-
- <!-- ================================================================== -->
<!-- Run unit tests -->
<!-- ================================================================== -->
<target name="test" depends="test-core, test-plugins"/>
@@ -294,6 +184,7 @@
errorProperty="tests.failed" failureProperty="tests.failed" maxmemory="1000m">
<sysproperty key="test.build.data" value="${test.build.data}"/>
<sysproperty key="test.src.dir" value="${test.src.dir}"/>
+ <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
<classpath refid="test.classpath"/>
<formatter type="${test.junit.output.format}" />
<batchtest todir="${test.build.dir}" unless="testcase">
@@ -317,6 +208,89 @@
</target>
<!-- ================================================================== -->
+ <!-- Ivy targets -->
+ <!-- ================================================================== -->
+
+ <!-- target: resolve ================================================= -->
+ <target name="resolve-default" depends="clean-lib" description="--> resolve and retrieve dependencies with ivy">
+ <ivy:resolve file="${ivy.file}" conf="default" log="download-only"/>
+ <ivy:retrieve pattern="${build.lib.dir}/[artifact]-[revision].[ext]" symlink="false" log="quiet"/>
+ <antcall target="copy-libs"/>
+ </target>
+
+ <target name="resolve-test" depends="clean-lib" description="--> resolve and retrieve dependencies with ivy">
+ <ivy:resolve file="${ivy.file}" conf="test" log="download-only"/>
+ <ivy:retrieve pattern="${build.lib.dir}/[artifact]-[revision].[ext]" symlink="false" log="quiet"/>
+ <antcall target="copy-libs"/>
+ </target>
+
+ <target name="copy-libs">
+ <!-- copy the libs in lib, which are not ivy enabled-->
+ <copy todir="${build.lib.dir}/" failonerror="false">
+ <fileset dir="${lib.dir}" includes="**/*.jar"/>
+ </copy>
+ </target>
+
+ <!-- target: publish-local =========================================== -->
+ <target name="publish-local" depends="jar" description="--> publish this project in the local ivy repository">
+ <ivy:publish artifactspattern="${build.dir}/[artifact]-${version}.[ext]"
+ resolver="local"
+ pubrevision="${version}"
+ pubdate="${now}"
+ status="integration"
+ forcedeliver="true"
+ overwrite="true"
+ />
+ <echo message="project ${ant.project.name} published locally with version ${version}" />
+ </target>
+
+ <!-- target: report ================================================== -->
+ <target name="report" depends="resolve-test" description="--> generates a report of dependencies">
+ <ivy:report todir="${build.dir}"/>
+ </target>
+
+ <!-- target: ivy-init ================================================ -->
+ <target name="ivy-init" depends="ivy-probe-antlib, ivy-init-antlib">
+ <ivy:settings file="${ivy.dir}/ivysettings.xml" />
+ </target>
+
+ <!-- target: ivy-probe-antlib ======================================== -->
+ <target name="ivy-probe-antlib">
+ <condition property="ivy.found">
+ <typefound uri="antlib:org.apache.ivy.ant" name="cleancache" />
+ </condition>
+ </target>
+
+ <!-- target: ivy-download ============================================ -->
+ <target name="ivy-download" description="Download ivy">
+ <available file="${ivy.jar}" property="ivy.jar.found"/>
+ <antcall target="-ivy-download-unchecked"/>
+ </target>
+
+ <!-- target: ivy-download-unchecked ================================== -->
+ <target name="-ivy-download-unchecked" unless="ivy.jar.found">
+ <get src="${ivy.repo.url}" dest="${ivy.jar}" usetimestamp="true" />
+ </target>
+
+ <!-- target: ivy-init-antlib ========================================= -->
+ <target name="ivy-init-antlib" depends="ivy-download" unless="ivy.found">
+ <typedef uri="antlib:org.apache.ivy.ant" onerror="fail" loaderRef="ivyLoader">
+ <classpath>
+ <pathelement location="${ivy.jar}" />
+ </classpath>
+ </typedef>
+ <fail>
+ <condition>
+ <not>
+ <typefound uri="antlib:org.apache.ivy.ant" name="cleancache" />
+ </not>
+ </condition>
+ You need Apache Ivy 2.0 or later from http://ant.apache.org/
+ It could not be loaded from ${ivy.repo.url}
+ </fail>
+ </target>
+
+ <!-- ================================================================== -->
<!-- Documentation -->
<!-- ================================================================== -->
<target name="javadoc" depends="compile">
@@ -336,44 +310,27 @@
<packageset dir="${src.dir}"/>
<packageset dir="${plugins.dir}/lib-http/src/java"/>
- <packageset dir="${plugins.dir}/lib-parsems/src/java"/>
<packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
<packageset dir="${plugins.dir}/microformats-reltag/src/java"/>
- <packageset dir="${plugins.dir}/ontology/src/java"/>
<packageset dir="${plugins.dir}/protocol-file/src/java"/>
<packageset dir="${plugins.dir}/protocol-ftp/src/java"/>
<packageset dir="${plugins.dir}/protocol-http/src/java"/>
<packageset dir="${plugins.dir}/protocol-httpclient/src/java"/>
+ <packageset dir="${plugins.dir}/parse-tika/src/java"/>
<packageset dir="${plugins.dir}/parse-ext/src/java"/>
<packageset dir="${plugins.dir}/parse-html/src/java"/>
<packageset dir="${plugins.dir}/parse-js/src/java"/>
- <packageset dir="${plugins.dir}/parse-text/src/java"/>
- <packageset dir="${plugins.dir}/parse-pdf/src/java"/>
-<!-- <packageset dir="${plugins.dir}/parse-rtf/src/java"/> plugin excluded from build due to licensing issues-->
-<!-- <packageset dir="${plugins.dir}/parse-mp3/src/java"/> plugin excluded from build due to licensing issues-->
- <packageset dir="${plugins.dir}/parse-msexcel/src/java"/>
- <packageset dir="${plugins.dir}/parse-mspowerpoint/src/java"/>
- <packageset dir="${plugins.dir}/parse-msword/src/java"/>
- <packageset dir="${plugins.dir}/parse-oo/src/java"/>
<packageset dir="${plugins.dir}/parse-rss/src/java"/>
<packageset dir="${plugins.dir}/parse-swf/src/java"/>
<packageset dir="${plugins.dir}/parse-zip/src/java"/>
<packageset dir="${plugins.dir}/index-basic/src/java"/>
<packageset dir="${plugins.dir}/index-more/src/java"/>
- <packageset dir="${plugins.dir}/query-basic/src/java"/>
- <packageset dir="${plugins.dir}/query-more/src/java"/>
- <packageset dir="${plugins.dir}/query-site/src/java"/>
- <packageset dir="${plugins.dir}/query-url/src/java"/>
<packageset dir="${plugins.dir}/scoring-opic/src/java"/>
- <packageset dir="${plugins.dir}/summary-basic/src/java"/>
- <packageset dir="${plugins.dir}/summary-lucene/src/java"/>
<packageset dir="${plugins.dir}/urlfilter-automaton/src/java"/>
<packageset dir="${plugins.dir}/urlfilter-regex/src/java"/>
<packageset dir="${plugins.dir}/urlfilter-prefix/src/java"/>
<packageset dir="${plugins.dir}/creativecommons/src/java"/>
<packageset dir="${plugins.dir}/languageidentifier/src/java"/>
- <packageset dir="${plugins.dir}/clustering-carrot2/src/java"/>
- <packageset dir="${plugins.dir}/ontology/src/java"/>
<link href="${javadoc.link.java}"/>
<link href="${javadoc.link.lucene}"/>
@@ -392,12 +349,7 @@
<group title="URL Filter Plugins" packages="${plugins.urlfilter}"/>
<group title="Scoring Plugins" packages="${plugins.scoring}"/>
<group title="Parse Plugins" packages="${plugins.parse}"/>
- <group title="Analysis Plugins" packages="${plugins.analysis}"/>
<group title="Indexing Filter Plugins" packages="${plugins.index}"/>
- <group title="Query Filter Plugins" packages="${plugins.query}"/>
- <group title="Summary Plugins" packages="${plugins.summary}"/>
- <group title="Clustering Plugins" packages="${plugins.clustering}"/>
- <group title="Ontology Plugins" packages="${plugins.ontology}"/>
<group title="Misc. Plugins" packages="${plugins.misc}"/>
</javadoc>
<!-- Copy the plugin.dtd file to the plugin doc-files dir -->
@@ -405,190 +357,63 @@
todir="${build.javadoc}/org/apache/nutch/plugin/doc-files"/>
</target>
- <target name="default-doc">
- <style basedir="${conf.dir}" destdir="${docs.dir}"
- includes="nutch-default.xml" style="conf/nutch-conf.xsl"/>
- </target>
-
- <target name="generate-locale" if="doc.locale">
- <echo message="Generating docs for locale=${doc.locale}"/>
-
- <mkdir dir="${build.docs}/${doc.locale}/include"/>
- <xslt in="${docs.src}/include/${doc.locale}/header.xml"
- out="${build.docs}/${doc.locale}/include/header.html"
- style="${docs.src}/style/nutch-header.xsl">
- <xmlcatalog refid="docDTDs"/>
- </xslt>
-
- <dependset>
- <srcfileset dir="${docs.src}/include/${doc.locale}" includes="*.xml"/>
- <srcfileset dir="${docs.src}/style" includes="*.xsl"/>
- <targetfileset dir="${docs.dir}/${doc.locale}" includes="*.html"/>
- </dependset>
-
- <copy file="${docs.src}/style/nutch-page.xsl"
- todir="${build.docs}/${doc.locale}"
- preservelastmodified="true"/>
-
- <xslt basedir="${docs.src}/pages/${doc.locale}"
- destdir="${docs.dir}/${doc.locale}"
- includes="*.xml"
- style="${build.docs}/${doc.locale}/nutch-page.xsl">
- <xmlcatalog refid="docDTDs"/>
- </xslt>
- </target>
-
-
- <target name="generate-docs" depends="init">
- <dependset>
- <srcfileset dir="${docs.src}/include" includes="*.html"/>
- <targetfileset dir="${docs.dir}" includes="**/*.html"/>
- </dependset>
-
- <mkdir dir="${build.docs}/include"/>
- <copy todir="${build.docs}/include">
- <fileset dir="${docs.src}/include"/>
- </copy>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="ca"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="de"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="en"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="es"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="fi"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="fr"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="hu"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="it"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="jp"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="ms"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="nl"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="pl"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="pt"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="sh"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="sr"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="sv"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="th"/>
- </antcall>
-
- <antcall target="generate-locale">
- <param name="doc.locale" value="zh"/>
- </antcall>
-
- <fixcrlf srcdir="${docs.dir}" eol="lf" encoding="utf-8"
- includes="**/*.html"/>
-
- </target>
-
<!-- ================================================================== -->
<!-- D I S T R I B U T I O N -->
<!-- ================================================================== -->
<!-- -->
<!-- ================================================================== -->
- <target name="package" depends="jar, job, war, javadoc">
- <mkdir dir="${dist.dir}"/>
- <mkdir dir="${dist.dir}/lib"/>
- <mkdir dir="${dist.dir}/bin"/>
- <mkdir dir="${dist.dir}/docs"/>
- <mkdir dir="${dist.dir}/docs/api"/>
- <mkdir dir="${dist.dir}/plugins"/>
+ <target name="package" depends="jar, job, javadoc">
+ <mkdir dir="${dist.version.dir}"/>
+ <mkdir dir="${dist.version.dir}/lib"/>
+ <mkdir dir="${dist.version.dir}/bin"/>
+ <mkdir dir="${dist.version.dir}/docs"/>
+ <mkdir dir="${dist.version.dir}/docs/api"/>
+ <mkdir dir="${dist.version.dir}/plugins"/>
- <copy todir="${dist.dir}/lib" includeEmptyDirs="false">
- <fileset dir="lib"/>
+ <copy todir="${dist.version.dir}/lib" includeEmptyDirs="false">
+ <fileset dir="${build.lib.dir}"/>
</copy>
- <copy todir="${dist.dir}/plugins">
+ <copy todir="${dist.version.dir}/plugins">
<fileset dir="${build.plugins}"/>
</copy>
- <copy todir="${dist.dir}/webapps">
- <fileset dir="${build.webapps}"/>
- </copy>
-
- <copy file="${build.dir}/${final.name}.jar" todir="${dist.dir}"/>
- <copy file="${build.dir}/${final.name}.job" todir="${dist.dir}"/>
- <copy file="${build.dir}/${final.name}.war" todir="${dist.dir}"/>
-
- <copy todir="${dist.dir}/bin">
+ <copy file="${build.dir}/${final.name}.jar" todir="${dist.version.dir}"/>
+ <copy file="${build.dir}/${final.name}.job" todir="${dist.version.dir}"/>
+
+ <copy todir="${dist.version.dir}/bin">
<fileset dir="bin"/>
</copy>
- <copy todir="${dist.dir}/conf">
+ <copy todir="${dist.version.dir}/conf">
<fileset dir="${conf.dir}" excludes="**/*.template"/>
</copy>
<chmod perm="ugo+x" type="file">
- <fileset dir="${dist.dir}/bin"/>
+ <fileset dir="${dist.version.dir}/bin"/>
</chmod>
- <copy todir="${dist.dir}/docs">
- <fileset dir="${docs.dir}"/>
+ <copy todir="${dist.version.dir}/docs">
+ <fileset dir="site" />
</copy>
- <copy todir="${dist.dir}/docs/api">
+ <copy todir="${dist.version.dir}/docs/api">
<fileset dir="${build.javadoc}"/>
</copy>
- <copy todir="${dist.dir}">
+ <copy todir="${dist.version.dir}">
<fileset dir=".">
<include name="*.txt" />
<include name="KEYS" />
</fileset>
</copy>
- <copy todir="${dist.dir}/src" includeEmptyDirs="true">
+ <copy todir="${dist.version.dir}/src" includeEmptyDirs="true">
<fileset dir="src"/>
</copy>
- <copy todir="${dist.dir}/" file="build.xml"/>
- <copy todir="${dist.dir}/" file="default.properties"/>
+ <copy todir="${dist.version.dir}/" file="build.xml"/>
+ <copy todir="${dist.version.dir}/" file="default.properties"/>
</target>
@@ -597,22 +422,64 @@
<!-- ================================================================== -->
<target name="tar" depends="package">
<tar compression="gzip" longfile="gnu"
- destfile="${build.dir}/${final.name}.tar.gz">
- <tarfileset dir="${build.dir}" mode="664">
+ destfile="${dist.dir}/${final.name}.tar.gz">
+ <tarfileset dir="${dist.dir}" mode="664">
<exclude name="${final.name}/bin/*" />
<include name="${final.name}/**" />
</tarfileset>
- <tarfileset dir="${build.dir}" mode="755">
+ <tarfileset dir="${dist.dir}" mode="755">
<include name="${final.name}/bin/*" />
</tarfileset>
</tar>
</target>
+
+ <!-- ================================================================== -->
+ <!-- Make release tarball -->
+ <!-- ================================================================== -->
+ <target name="zip" depends="package">
+ <zip compress="true" casesensitive="yes" destfile="${dist.dir}/${final.name}.zip">
+ <zipfileset dir="${dist.dir}" filemode="664">
+ <exclude name="${final.name}/bin/*" />
+ <include name="${final.name}/**" />
+ </zipfileset>
+ <zipfileset dir="${dist.dir}" filemode="755">
+ <include name="${final.name}/bin/*" />
+ </zipfileset>
+ </zip>
+ </target>
<!-- ================================================================== -->
<!-- Clean. Delete the build files, and their directories -->
<!-- ================================================================== -->
- <target name="clean">
- <delete dir="${build.dir}"/>
+
+ <!-- target: clean =================================================== -->
+ <target name="clean" depends="clean-build, clean-lib, clean-dist" description="--> clean the project" />
+
+ <!-- target: clean-local ============================================= -->
+ <target name="clean-local" depends=""
+ description="--> cleans the local repository for the current module">
+ <delete dir="${ivy.local.default.root}/${ivy.organisation}/${ivy.module}"/>
+ </target>
+
+ <!-- target: clean-lib =============================================== -->
+ <target name="clean-lib" description="--> clean the project libraries directory (dependencies)">
+ <delete includeemptydirs="true" dir="${build.lib.dir}"/>
+ </target>
+
+ <!-- target: clean-build ============================================= -->
+ <target name="clean-build" description="--> clean the project built files">
+ <delete includeemptydirs="true" dir="${build.dir}"/>
+ </target>
+
+ <!-- target: clean-dist ============================================= -->
+ <target name="clean-dist" description="--> clean the project dist files">
+ <delete includeemptydirs="true" dir="${dist.dir}"/>
+ </target>
+
+ <!-- target: clean-cache ============================================= -->
+ <target name="clean-cache" depends=""
+ description="delete ivy cache">
+ <ivy:cleancache />
</target>
<!-- ================================================================== -->
@@ -635,5 +502,45 @@
</fileset>
</rat:report>
</target>
+
+
+ <!-- ================================================================== -->
+ <!-- Run code checks (PMD) -->
+ <!-- ================================================================== -->
+ <target name="pmd" depends="compile">
+ <property name="pmd.report" location="${build.dir}/pmd-report.html" />
+ <taskdef name="pmd" classname="net.sourceforge.pmd.ant.PMDTask">
+ <classpath>
+ <fileset dir="${build.lib.dir}">
+ <include name="pmd-ext/*.jar" />
+ <include name="xerces*.jar" />
+ </fileset>
+ </classpath>
+ </taskdef>
+ <pmd shortFilenames="true" failonerror="true" failOnRuleViolation="false"
+ encoding="${build.encoding}" failuresPropertyName="pmd.failures">
+ <ruleset>unusedcode</ruleset>
+ <!--ruleset>basic</ruleset-->
+ <!--ruleset>optimizations</ruleset-->
+ <formatter type="html" toFile="${pmd.report}" />
+ <!-- <formatter type="xml" toFile="${tempbuild}/$report_pmd.xml"/> -->
+ <fileset dir="${basedir}/src">
+ <include name="java/**/*.java"/>
+ <include name="plugin/**/*.java"/>
+ <!-- Exclude generated sources -->
+ <exclude name="**/NutchAnalysis.java" />
+ <exclude name="**/NutchAnalysisTokenManager.java" />
+ </fileset>
+ </pmd>
+ <condition property="pmd.stop" value="true">
+ <and>
+ <isset property="pmd.failures" />
+ <not>
+ <equals arg1="0" arg2="${pmd.failures}" trim="true" />
+ </not>
+ </and>
+ </condition>
+ <fail if="pmd.stop">FAILURE: PMD shows ${pmd.failures} rule violations. See ${pmd.report} for details.</fail>
+ </target>
</project>
Added: nutch/branches/nutchbase/conf/gora.properties
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/conf/gora.properties?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/conf/gora.properties (added)
+++ nutch/branches/nutchbase/conf/gora.properties Wed Jun 30 10:36:20 2010
@@ -0,0 +1,2 @@
+output.path=test-avrostore-output
+input.path=test-avrostore-input
Modified: nutch/branches/nutchbase/conf/log4j.properties
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/conf/log4j.properties?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/conf/log4j.properties (original)
+++ nutch/branches/nutchbase/conf/log4j.properties Wed Jun 30 10:36:20 2010
@@ -10,21 +10,16 @@ log4j.threshhold=ALL
#special logging requirements for some commandline tools
log4j.logger.org.apache.nutch.crawl.Crawl=INFO,cmdstdout
-log4j.logger.org.apache.nutch.crawl.Injector=INFO,cmdstdout
-log4j.logger.org.apache.nutch.crawl.Generator=INFO,cmdstdout
-log4j.logger.org.apache.nutch.fetcher.Fetcher=INFO,cmdstdout
-log4j.logger.org.apache.nutch.parse.ParseSegment=INFO,cmdstdout
-log4j.logger.org.apache.nutch.crawl.CrawlDbReader=INFO,cmdstdout
-log4j.logger.org.apache.nutch.crawl.CrawlDbMerger=INFO,cmdstdout
-log4j.logger.org.apache.nutch.crawl.LinkDbReader=INFO,cmdstdout
-log4j.logger.org.apache.nutch.segment.SegmentReader=INFO,cmdstdout
-log4j.logger.org.apache.nutch.segment.SegmentMerger=INFO,cmdstdout
-log4j.logger.org.apache.nutch.crawl.CrawlDb=INFO,cmdstdout
-log4j.logger.org.apache.nutch.crawl.LinkDb=INFO,cmdstdout
-log4j.logger.org.apache.nutch.crawl.LinkDbMerger=INFO,cmdstdout
-log4j.logger.org.apache.nutch.indexer.Indexer=INFO,cmdstdout
+log4j.logger.org.apache.nutch.crawl.InjectorJob=INFO,cmdstdout
+log4j.logger.org.apache.nutch.crawl.GeneratorJob=INFO,cmdstdout
+log4j.logger.org.apache.nutch.crawl.DbUpdaterJob=INFO,cmdstdout
+log4j.logger.org.apache.nutch.fetcher.FetcherJob=INFO,cmdstdout
+log4j.logger.org.apache.nutch.parse.ParserJob=INFO,cmdstdout
+log4j.logger.org.apache.nutch.indexer.IndexerJob=INFO,cmdstdout
+log4j.logger.org.apache.nutch.indexer.solr.SolrIndexerJob=INFO,cmdstdout
log4j.logger.org.apache.nutch.indexer.DeleteDuplicates=INFO,cmdstdout
log4j.logger.org.apache.nutch.indexer.IndexMerger=INFO,cmdstdout
+log4j.logger.org.apache.nutch.crawl.WebTableReader=INFO,cmdstdout
log4j.logger.org.apache.nutch=INFO
log4j.logger.org.apache.hadoop=WARN
Modified: nutch/branches/nutchbase/conf/nutch-default.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/conf/nutch-default.xml?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/conf/nutch-default.xml (original)
+++ nutch/branches/nutchbase/conf/nutch-default.xml Wed Jun 30 10:36:20 2010
@@ -497,24 +497,21 @@
<!-- generate properties -->
<property>
- <name>generate.max.per.host</name>
+ <name>generate.max.count</name>
<value>-1</value>
- <description>The maximum number of urls per host in a single
- fetchlist. -1 if unlimited.</description>
+ <description>The maximum number of urls in a single
+ fetchlist. -1 if unlimited. The urls are counted according
+ to the value of the parameter generator.count.mode.
+ </description>
</property>
<property>
- <name>generate.max.per.host.by.ip</name>
- <value>false</value>
- <description>If false, same host names are counted. If true,
- hosts' IP addresses are resolved and the same IP-s are counted.
-
- -+-+-+- WARNING !!! -+-+-+-
- When set to true, Generator will create a lot of DNS lookup
- requests, rapidly. This may cause a DOS attack on
- remote DNS servers, not to mention increased external traffic
- and latency. For these reasons when using this option it is
- required that a local caching DNS be used.</description>
+ <name>generate.count.mode</name>
+ <value>host</value>
+ <description>Determines how the URLs are counted for generator.max.count.
+ Default value is 'host' but can be 'domain'. Note that we do not count
+ per IP in the new version of the Generator.
+ </description>
</property>
<property>
@@ -528,6 +525,14 @@
updatedb will generate identical fetchlists.</description>
</property>
+<property>
+ <name>partition.url.mode</name>
+ <value>byHost</value>
+ <description>Determines how to partition URLs. Default value is 'byHost',
+ also takes 'byDomain' or 'byIP'.
+ </description>
+</property>
+
<!-- fetcher properties -->
<property>
@@ -567,20 +572,19 @@
</property>
<property>
- <name>fetcher.threads.per.host</name>
+ <name>fetcher.threads.per.queue</name>
<value>1</value>
<description>This number is the maximum number of threads that
- should be allowed to access a host at one time.</description>
+ should be allowed to access a queue at one time.</description>
</property>
<property>
- <name>fetcher.threads.per.host.by.ip</name>
- <value>true</value>
- <description>If true, then fetcher will count threads by IP address,
- to which the URL's host name resolves. If false, only host name will be
- used. NOTE: this should be set to the same value as
- "generate.max.per.host.by.ip" - default settings are different only for
- reasons of backward-compatibility.</description>
+ <name>fetcher.queue.mode</name>
+ <value>byHost</value>
+ <description>Determines how the URLs are placed into queues.
+ Allowed values are 'byHost', 'byDomain' and 'byIP'.
+ The value would usually correspond to that of 'partition.url.mode'.
+ </description>
</property>
<property>
@@ -886,7 +890,7 @@
<property>
<name>plugin.includes</name>
- <value>protocol-http|urlfilter-regex|parse-(html|js)|index-basic|urlnormalizer-(pass|regex|basic)|scoring-opic|query-(basic|site|url)|summary-basic</value>
+ <value>protocol-http|urlfilter-regex|parse-tika|index-(basic|anchor)|urlnormalizer-(pass|regex|basic)|scoring-opic|query-(basic|site|url)|summary-basic</value>
<description>Regular expression naming plugin directory names to
include. Any plugin not matching this expression is excluded.
In any case you need at least include the nutch-extensionpoints plugin. By
@@ -966,6 +970,18 @@
for most people would be "img,script,link".</description>
</property>
+<property>
+ <name>htmlparsefilter.order</name>
+ <value></value>
+ <description>The order by which HTMLParse filters are applied.
+ If empty, all available HTMLParse filters (as dictated by properties
+ plugin-includes and plugin-excludes above) are loaded and applied in system
+ defined order. If not empty, only named filters are loaded and applied
+ in given order.
+ HTMLParse filter ordering MAY have an impact
+ on end result, as some filters could rely on the metadata generated by a previous filter.
+ </description>
+</property>
<!-- urlfilter plugin properties -->
@@ -1277,4 +1293,12 @@
</description>
</property>
+<!-- storage properties -->
+
+<property>
+ <name>storage.data.store.class</name>
+ <value>org.gora.hbase.store.HBaseStore</value>
+ <description>Default class for storing data</description>
+</property>
+
</configuration>
Modified: nutch/branches/nutchbase/conf/parse-plugins.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/conf/parse-plugins.xml?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/conf/parse-plugins.xml (original)
+++ nutch/branches/nutchbase/conf/parse-plugins.xml Wed Jun 30 10:36:20 2010
@@ -22,6 +22,11 @@
<parse-plugins>
+<!--
+ The following mimetype are now handled by the default parser (parse-tika).
+ You can uncomment the associations below to override parse-tika
+ and chose which plugin should be used for a given content type
+
<mimeType name="application/msword">
<plugin id="parse-msword" />
</mimeType>
@@ -34,11 +39,6 @@
<plugin id="parse-pdf" />
</mimeType>
- <mimeType name="application/rss+xml">
- <plugin id="parse-rss" />
- <plugin id="feed" />
- </mimeType>
-
<mimeType name="application/vnd.ms-excel">
<plugin id="parse-msexcel" />
</mimeType>
@@ -102,47 +102,19 @@
<mimeType name="application/vnd.sun.xml.writer.template">
<plugin id="parse-oo" />
</mimeType>
-
- <mimeType name="application/xhtml+xml">
- <plugin id="parse-html" />
- </mimeType>
-
- <mimeType name="application/x-bzip2">
- <!-- try and parse it with the zip parser -->
- <plugin id="parse-zip" />
- </mimeType>
-
+
<mimeType name="application/x-csh">
<plugin id="parse-text" />
</mimeType>
- <mimeType name="application/x-gzip">
- <!-- try and parse it with the zip parser -->
- <plugin id="parse-zip" />
- </mimeType>
-
- <mimeType name="application/x-javascript">
- <plugin id="parse-js" />
- </mimeType>
-
<mimeType name="application/x-kword">
- <!-- try and parse it with the word parser -->
<plugin id="parse-msword" />
</mimeType>
<mimeType name="application/x-kspread">
- <!-- try and parse it with the msexcel parser -->
<plugin id="parse-msexcel" />
</mimeType>
-
- <mimeType name="application/x-shockwave-flash">
- <plugin id="parse-swf" />
- </mimeType>
-
- <mimeType name="application/zip">
- <plugin id="parse-zip" />
- </mimeType>
-
+
<mimeType name="text/html">
<plugin id="parse-html" />
</mimeType>
@@ -152,13 +124,11 @@
</mimeType>
<mimeType name="text/richtext">
- <plugin id="parse-rtf" />
- <plugin id="parse-msword" />
+ <plugin id="parse-tika" />
</mimeType>
<mimeType name="text/rtf">
- <plugin id="parse-rtf" />
- <plugin id="parse-msword" />
+ <plugin id="parse-tika" />
</mimeType>
<mimeType name="text/sgml">
@@ -168,6 +138,38 @@
<mimeType name="text/tab-separated-values">
<plugin id="parse-msexcel" />
</mimeType>
+-->
+
+ <mimeType name="application/xhtml+xml">
+ <plugin id="parse-html" />
+ </mimeType>
+
+ <mimeType name="application/rss+xml">
+ <plugin id="parse-rss" />
+ <plugin id="feed" />
+ </mimeType>
+
+ <mimeType name="application/x-bzip2">
+ <!-- try and parse it with the zip parser -->
+ <plugin id="parse-zip" />
+ </mimeType>
+
+ <mimeType name="application/x-gzip">
+ <!-- try and parse it with the zip parser -->
+ <plugin id="parse-zip" />
+ </mimeType>
+
+ <mimeType name="application/x-javascript">
+ <plugin id="parse-js" />
+ </mimeType>
+
+ <mimeType name="application/x-shockwave-flash">
+ <plugin id="parse-swf" />
+ </mimeType>
+
+ <mimeType name="application/zip">
+ <plugin id="parse-zip" />
+ </mimeType>
<mimeType name="text/xml">
<plugin id="parse-html" />
@@ -188,12 +190,12 @@
<!-- alias mappings for parse-xxx names to the actual extension implementation
ids described in each plugin's plugin.xml file -->
<aliases>
+ <alias name="parse-tika"
+ extension-id="org.apache.nutch.parse.tika.Parser" />
<alias name="parse-ext" extension-id="ExtParser" />
<alias name="parse-html"
extension-id="org.apache.nutch.parse.html.HtmlParser" />
<alias name="parse-js" extension-id="JSParser" />
- <alias name="parse-mp3"
- extension-id="org.apache.nutch.parse.mp3.MP3Parser" />
<alias name="parse-msexcel"
extension-id="org.apache.nutch.parse.msexcel.MSExcelParser" />
<alias name="parse-mspowerpoint"
@@ -206,10 +208,8 @@
extension-id="org.apache.nutch.parse.pdf.PdfParser" />
<alias name="parse-rss"
extension-id="org.apache.nutch.parse.rss.RSSParser" />
- <alias name="feed"
- extension-id="org.apache.nutch.parse.feed.FeedParser" />
- <alias name="parse-rtf"
- extension-id="org.apache.nutch.parse.rtf.RTFParseFactory" />
+ <alias name="feed"
+ extension-id="org.apache.nutch.parse.feed.FeedParser" />
<alias name="parse-swf"
extension-id="org.apache.nutch.parse.swf.SWFParser" />
<alias name="parse-text"
Modified: nutch/branches/nutchbase/conf/schema.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/conf/schema.xml?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/conf/schema.xml (original)
+++ nutch/branches/nutchbase/conf/schema.xml Wed Jun 30 10:36:20 2010
@@ -105,5 +105,4 @@
<uniqueKey>id</uniqueKey>
<defaultSearchField>content</defaultSearchField>
<solrQueryParser defaultOperator="OR"/>
- <copyField source="url" dest="id"/>
-</schema>
\ No newline at end of file
+</schema>