You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by sn...@apache.org on 2026/06/30 06:45:00 UTC
(nutch) 01/01: Merge pull request #924 from sebastian-nagel/NUTCH-1807-forbidden-apis
This is an automated email from the ASF dual-hosted git repository.
sebastian-nagel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
commit 10b9a56e6b3f7e0924b175b025f9d7705dacc24f
Merge: ca8538abf b845812d3
Author: Sebastian Nagel <sn...@apache.org>
AuthorDate: Tue Jun 30 08:44:53 2026 +0200
Merge pull request #924 from sebastian-nagel/NUTCH-1807-forbidden-apis
.github/workflows/master-build.yml | 6 +-
.yetus/blanks-tabs.txt | 3 +
build.xml | 50 ++++++-
default.properties | 3 +
.../apache/nutch/crawl/AdaptiveFetchSchedule.java | 45 +++----
src/java/org/apache/nutch/crawl/CrawlDbReader.java | 45 ++++---
.../org/apache/nutch/crawl/DeduplicationJob.java | 14 +-
src/java/org/apache/nutch/crawl/Generator.java | 62 ++++-----
src/java/org/apache/nutch/crawl/LinkDb.java | 23 ++--
src/java/org/apache/nutch/crawl/MD5Signature.java | 6 +-
.../apache/nutch/crawl/TextProfileSignature.java | 7 +-
src/java/org/apache/nutch/fetcher/FetchItem.java | 4 +-
src/java/org/apache/nutch/fetcher/Fetcher.java | 6 +-
.../org/apache/nutch/fetcher/FetcherThread.java | 59 ++++----
src/java/org/apache/nutch/hostdb/HostDatum.java | 6 +-
.../org/apache/nutch/indexer/IndexerMapReduce.java | 50 +++----
.../org/apache/nutch/parse/ParseOutputFormat.java | 85 ++++++------
.../org/apache/nutch/protocol/ProtocolFactory.java | 48 ++++---
.../apache/nutch/protocol/RobotRulesParser.java | 32 ++---
.../apache/nutch/scoring/webgraph/LinkRank.java | 53 ++++----
.../nutch/segment/ContentAsTextInputFormat.java | 10 +-
.../org/apache/nutch/segment/SegmentReader.java | 20 +--
.../nutch/tools/AbstractCommonCrawlFormat.java | 4 +-
.../apache/nutch/tools/CommonCrawlDataDumper.java | 15 +--
.../nutch/tools/CommonCrawlFormatJackson.java | 33 ++---
.../apache/nutch/tools/CommonCrawlFormatWARC.java | 9 +-
src/java/org/apache/nutch/tools/DmozParser.java | 40 +++---
src/java/org/apache/nutch/tools/FileDumper.java | 50 ++++---
src/java/org/apache/nutch/tools/ResolveUrls.java | 13 +-
.../org/apache/nutch/tools/ShowProperties.java | 22 +--
.../apache/nutch/tools/arc/ArcRecordReader.java | 43 +++---
.../apache/nutch/tools/arc/ArcSegmentCreator.java | 44 +++---
.../org/apache/nutch/tools/warc/WARCExporter.java | 27 ++--
.../org/apache/nutch/util/AbstractChecker.java | 16 +--
src/java/org/apache/nutch/util/DumpFileUtil.java | 47 ++++---
.../org/apache/nutch/util/EncodingDetector.java | 25 ++--
src/java/org/apache/nutch/util/NutchJob.java | 18 +--
.../org/apache/nutch/util/PrefixStringMatcher.java | 4 +-
.../org/apache/nutch/util/SitemapProcessor.java | 12 +-
src/java/org/apache/nutch/util/TimingUtil.java | 17 ++-
src/java/org/apache/nutch/util/URLUtil.java | 52 +++----
.../org/apache/nutch/parse/feed/FeedParser.java | 44 +++---
.../parse/headings/TestHeadingsParseFilter.java | 19 ++-
.../nutch/indexer/anchor/AnchorIndexingFilter.java | 7 +-
.../nutch/indexer/links/LinksIndexingFilter.java | 22 +--
.../cloudsearch/CloudSearchIndexWriter.java | 7 +-
.../nutch/indexwriter/csv/CSVIndexWriter.java | 12 +-
.../nutch/indexwriter/dummy/DummyIndexWriter.java | 7 +-
.../nutch/indexwriter/rabbit/RabbitDocument.java | 7 +-
.../indexwriter/rabbit/RabbitIndexWriter.java | 20 ++-
.../nutch/indexwriter/rabbit/RabbitMessage.java | 7 +-
.../nutch/analysis/lang/HTMLLanguageParser.java | 7 +-
.../analysis/lang/TestHTMLLanguageParser.java | 11 +-
.../apache/nutch/protocol/http/api/HttpBase.java | 29 ++--
.../protocol/http/api/HttpRobotRulesParser.java | 23 ++--
.../protocol/http/api/TestRobotRulesParser.java | 38 +++---
.../nutch/urlfilter/api/RegexURLFilterBase.java | 58 ++++----
.../urlfilter/api/RegexURLFilterBaseTest.java | 28 ++--
.../nutch/microformats/reltag/RelTagParser.java | 23 ++--
.../indexer/filter/MimeTypeIndexingFilter.java | 47 +++----
.../indexer/filter/MimeTypeIndexingFilterTest.java | 13 +-
.../java/org/apache/nutch/parse/ext/ExtParser.java | 46 +++----
.../apache/nutch/parse/html/DOMContentUtils.java | 55 ++++----
.../apache/nutch/parse/html/HTMLMetaProcessor.java | 16 ++-
.../nutch/parse/html/TestDOMContentUtils.java | 22 +--
.../nutch/parse/html/TestRobotsMetaProcessor.java | 34 ++---
.../org/apache/nutch/parse/js/JSParseFilter.java | 27 ++--
.../org/apache/nutch/parse/tika/DOMBuilder.java | 150 ++++++++++++---------
.../apache/nutch/parse/tika/DOMContentUtils.java | 48 +++----
.../nutch/parse/tika/TestBoilerpipeExtraction.java | 49 ++++---
.../parse/tika/TestLinkExtractionEdgeCases.java | 69 +++++-----
.../nutch/parse/tika/TestMetadataExtraction.java | 59 ++++----
.../org/apache/nutch/parse/tika/TestOOParser.java | 6 +-
.../parse/tika/TestParserFailureHandling.java | 68 +++++-----
.../nutch/parse/tika/TestRobotsMetaProcessor.java | 21 +--
.../nutch/parsefilter/naivebayes/Classify.java | 11 +-
.../apache/nutch/parsefilter/naivebayes/Train.java | 7 +-
.../nutch/parsefilter/regex/RegexParseFilter.java | 43 +++---
.../parsefilter/regex/TestRegexParseFilter.java | 24 ++--
.../java/org/apache/nutch/protocol/file/File.java | 15 +--
.../apache/nutch/protocol/file/FileResponse.java | 20 ++-
.../java/org/apache/nutch/protocol/foo/Foo.java | 11 +-
.../java/org/apache/nutch/protocol/ftp/Client.java | 63 ++++-----
.../java/org/apache/nutch/protocol/ftp/Ftp.java | 41 +++---
.../org/apache/nutch/protocol/ftp/FtpResponse.java | 32 ++---
.../nutch/protocol/ftp/FtpRobotRulesParser.java | 10 +-
.../nutch/protocol/htmlunit/HttpResponse.java | 9 +-
.../apache/nutch/protocol/http/HttpResponse.java | 12 +-
.../protocol/http/TestBadServerResponses.java | 15 ++-
.../httpclient/HttpBasicAuthentication.java | 28 ++--
.../httpclient/TestProtocolHttpClient.java | 35 +++--
.../protocol/interactiveselenium/HttpResponse.java | 18 +--
.../nutch/protocol/okhttp/IPFilterRules.java | 5 +-
.../nutch/protocol/okhttp/OkHttpResponse.java | 10 +-
.../protocol/okhttp/TestBadServerResponses.java | 33 +++--
.../protocol/okhttp/TestIPAddressFiltering.java | 13 +-
.../nutch/protocol/selenium/HttpResponse.java | 13 +-
.../publisher/rabbitmq/RabbitMQPublisherImpl.java | 3 +-
.../subcollection/SubcollectionIndexingFilter.java | 27 ++--
.../apache/nutch/collection/TestSubcollection.java | 12 +-
.../nutch/urlfilter/domain/DomainURLFilter.java | 23 ++--
.../domaindenylist/DomainDenylistURLFilter.java | 21 +--
.../apache/nutch/urlfilter/fast/FastURLFilter.java | 56 ++++----
.../nutch/urlfilter/prefix/PrefixURLFilter.java | 33 +++--
.../nutch/urlfilter/suffix/SuffixURLFilter.java | 61 +++++----
.../net/urlnormalizer/ajax/AjaxURLNormalizer.java | 9 +-
.../net/urlnormalizer/host/HostURLNormalizer.java | 15 ++-
.../protocol/ProtocolURLNormalizer.java | 18 +--
.../urlnormalizer/regex/RegexURLNormalizer.java | 24 ++--
.../regex/TestRegexURLNormalizer.java | 29 ++--
.../urlnormalizer/slash/SlashURLNormalizer.java | 32 ++---
src/test/domain-urlfilter.txt | 22 ---
src/test/host-protocol-mapping.txt | 35 +++++
.../org/apache/nutch/crawl/CrawlDBTestUtil.java | 41 +++---
src/test/org/apache/nutch/fetcher/TestFetcher.java | 22 +--
.../org/apache/nutch/plugin/TestPluginSystem.java | 26 ++--
.../apache/nutch/protocol/TestProtocolFactory.java | 30 ++++-
.../apache/nutch/segment/TestSegmentMerger.java | 9 +-
.../segment/TestSegmentMergerCrawlDatums.java | 9 +-
.../apache/nutch/util/TestEncodingDetector.java | 22 ++-
src/test/org/apache/nutch/util/TestGZIPUtils.java | 29 ++--
src/test/org/apache/nutch/util/TestMimeUtil.java | 5 +-
src/test/org/apache/nutch/util/TestNodeWalker.java | 5 +-
src/testresources/forbidden-apis-signatures.txt | 24 ++++
124 files changed, 1802 insertions(+), 1535 deletions(-)