You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2025/09/19 11:23:55 UTC
(stormcrawler) 01/01: Merge remote-tracking branch 'origin/main' into docs
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch docs
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
commit 18bb91041bbfb5ccd18f6bd2cfae1a48611461ab
Merge: a706f266 038152df
Author: Richard Zowalla <rz...@apache.org>
AuthorDate: Fri Sep 19 13:23:43 2025 +0200
Merge remote-tracking branch 'origin/main' into docs
.github/workflows/main.yml | 2 +-
.github/workflows/maven.yml | 4 +-
.github/workflows/snapshots.yaml | 2 +-
THIRD-PARTY.txt | 124 +++++-----
core/pom.xml | 33 +--
.../filtering/basic/BasicURLFilter.java | 7 +-
.../filtering/basic/BasicURLNormalizer.java | 2 +-
.../filtering/metadata/MetadataFilter.java | 246 +++++++++++++++++--
.../apache/stormcrawler/jsoup/LinkParseFilter.java | 2 +-
.../org/apache/stormcrawler/parse/ParseResult.java | 4 +-
.../parse/filter/CollectionTagger.java | 4 +-
.../stormcrawler/parse/filter/LinkParseFilter.java | 2 +-
.../stormcrawler/parse/filter/XPathFilter.java | 4 +-
.../persistence/urlbuffer/AbstractURLBuffer.java | 3 +-
.../protocol/HttpRobotRulesParser.java | 5 +-
.../protocol/httpclient/HttpProtocol.java | 2 +-
.../apache/stormcrawler/util/CookieConverter.java | 2 +-
.../java/org/apache/stormcrawler/util/URLUtil.java | 24 +-
core/src/main/resources/crawler-default.yaml | 24 --
.../filtering/MetadataFilterFromJsonTest.java | 152 ++++++++++++
.../stormcrawler/filtering/MetadataFilterTest.java | 262 +++++++++++++++++++++
.../test/resources/test.metadata.1.urlfilters.json | 11 +
.../test/resources/test.metadata.2.urlfilters.json | 15 ++
.../test/resources/test.metadata.3.urlfilters.json | 21 ++
external/ai/pom.xml | 4 +-
external/aws/pom.xml | 2 +-
external/playwright/pom.xml | 2 +-
external/selenium/README.md | 43 ++++
external/{playwright => selenium}/pom.xml | 56 +++--
external/selenium/selenium-conf.yaml | 41 ++++
.../protocol/selenium/NavigationFilter.java | 0
.../protocol/selenium/NavigationFilters.java | 0
.../protocol/selenium/RemoteDriverProtocol.java | 0
.../protocol/selenium/SeleniumProtocol.java | 0
.../protocol/selenium/ProtocolTest.java | 0
.../apache/stormcrawler/sql/StatusUpdaterBolt.java | 2 +-
.../apache/stormcrawler/warc/WARCHdfsBoltTest.java | 2 +-
.../apache/stormcrawler/warc/WARCSpoutTest.java | 2 +-
pom.xml | 25 +-
39 files changed, 953 insertions(+), 183 deletions(-)