You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by jn...@apache.org on 2024/07/05 14:16:37 UTC
(incubator-stormcrawler) branch issues/1248 updated (2c2a0de6 -> 815a3519)
This is an automated email from the ASF dual-hosted git repository.
jnioche pushed a change to branch issues/1248
in repository https://gitbox.apache.org/repos/asf/incubator-stormcrawler.git
from 2c2a0de6 #1248 - Use pre-compiled patterns for mime type matching in TikaParser
add 2eafa0a3 1024 Playwright protocol implementation, fixes #1024 (#1228)
add abd9918d Fix #1230: Set sitemap key before outlink processing (#1231)
add 3256ae33 Add disclaimer or binaries. Fixes #1220 (#1234)
add 5d1a9b29 Fix #1221
add 9e8d89f3 Update RAT exclusions. Fixes licenses. Fix #1215
add bf87568f #1236 - Fix Typos in StormCrawler (#1237)
add 6f5463a5 Fixes #1222
add b7523019 #1238 - Avoid use of star imports (#1239)
add 5b772479 Fix #1244 "Migrate to JUnit 5" (#1245)
add dc84c569 Fix #1216 - Add RAT Exclusion File for standalone RAT (#1243)
add 815a3519 Merge branch 'main' into issues/1248
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 15 ++
.github/workflows/code_coverage.yml | 15 ++
.github/workflows/maven.yml | 18 +-
.rat-excludes | 26 ++
DISCLAIMER-BINARIES.txt | 9 +
RELEASING.md | 29 +-
THIRD-PARTY.txt | 15 +-
.../archetype-resources/crawler-conf.yaml | 15 ++
.../src/main/java/CrawlTopology.java | 13 +-
core/pom.xml | 19 +-
.../org/apache/stormcrawler/bolt/FetcherBolt.java | 2 +-
.../stormcrawler/bolt/SimpleFetcherBolt.java | 4 +-
.../stormcrawler/bolt/SiteMapParserBolt.java | 8 +-
.../filtering/basic/BasicURLFilter.java | 10 +-
.../stormcrawler/persistence/DefaultScheduler.java | 9 +-
.../apache/stormcrawler/persistence/Scheduler.java | 2 +-
.../persistence/urlbuffer/URLBuffer.java | 2 +-
.../protocol/httpclient/HttpProtocol.java | 2 +-
.../stormcrawler/protocol/okhttp/HttpProtocol.java | 9 +-
.../org/apache/stormcrawler/spout/FileSpout.java | 8 +-
.../stormcrawler/util/InitialisationUtil.java | 4 +-
.../apache/stormcrawler/util/MetadataTransfer.java | 2 +-
.../java/org/apache/stormcrawler/util/URLUtil.java | 8 +-
core/src/main/resources/crawler-default.yaml | 19 +-
.../java/org/apache/stormcrawler/MetadataTest.java | 12 +-
.../stormcrawler/TestMetadataSerialization.java | 16 +-
.../stormcrawler/bolt/AbstractFetcherBoltTest.java | 51 ++--
.../stormcrawler/bolt/FeedParserBoltTest.java | 38 +--
.../apache/stormcrawler/bolt/FetcherBoltTest.java | 6 +-
.../stormcrawler/bolt/JSoupParserBoltTest.java | 132 ++++-----
.../stormcrawler/bolt/SimpleFetcherBoltTest.java | 6 +-
.../stormcrawler/bolt/SiteMapParserBoltTest.java | 178 ++++++------
.../stormcrawler/filtering/BasicURLFilterTest.java | 26 +-
.../filtering/BasicURLNormalizerTest.java | 112 ++++----
.../stormcrawler/filtering/FastURLFilterTest.java | 21 +-
.../stormcrawler/filtering/HostURLFilterTest.java | 41 ++-
.../stormcrawler/filtering/MaxDepthFilterTest.java | 22 +-
.../stormcrawler/filtering/MetadataFilterTest.java | 18 +-
.../stormcrawler/filtering/RegexFilterTest.java | 29 +-
...> ClassInheritingFromAbstractAndInterface.java} | 2 +-
.../stormcrawler/indexer/BasicIndexingTest.java | 138 ++++------
.../apache/stormcrawler/indexer/IndexerTester.java | 8 +-
.../apache/stormcrawler/json/JsoupFilterTest.java | 29 +-
.../stormcrawler/jsoup/JSoupFiltersTest.java | 61 ++---
.../stormcrawler/parse/DuplicateLinksTest.java | 16 +-
.../apache/stormcrawler/parse/ParsingTester.java | 8 +-
.../stormcrawler/parse/StackOverflowTest.java | 23 +-
.../stormcrawler/parse/TextExtractorTest.java | 34 +--
.../parse/filter/CSVMetadataFilterTest.java | 25 +-
.../parse/filter/CollectionTaggerTest.java | 21 +-
.../parse/filter/SubDocumentsFilterTest.java | 19 +-
.../parse/filter/SubDocumentsParseFilter.java | 6 +-
.../stormcrawler/parse/filter/XPathFilterTest.java | 54 ++--
.../persistence/AdaptiveSchedulerTest.java | 54 ++--
.../persistence/DefaultSchedulerTest.java | 39 ++-
.../stormcrawler/persistence/URLBufferTest.java | 49 ++--
.../protocol/AbstractProtocolTest.java | 20 +-
.../protocol/DelegationProtocolTest.java | 45 +---
.../stormcrawler/protocol/HttpHeadersTest.java | 13 +-
....java => HttpRobotRulesParserRedirectTest.java} | 224 +++++++---------
.../protocol/HttpRobotRulesParserTest.java | 248 +++--------------
.../protocol/selenium/ProtocolTest.java | 50 ++--
.../stormcrawler/proxy/MultiProxyManagerTest.java | 130 ++++-----
.../org/apache/stormcrawler/proxy/SCProxyTest.java | 50 ++--
.../stormcrawler/proxy/SingleProxyManagerTest.java | 25 +-
.../apache/stormcrawler/spout/FileSpoutTest.java | 28 +-
.../apache/stormcrawler/util/ConfUtilsTest.java | 30 +--
.../stormcrawler/util/CookieConverterTest.java | 180 ++++++-------
.../stormcrawler/util/InitialisationUtilTest.java | 84 +++---
.../stormcrawler/util/MetadataTransferTest.java | 39 ++-
.../apache/stormcrawler/util/RefreshTagTest.java | 11 +-
.../apache/stormcrawler/util/RobotsTagsTest.java | 36 ++-
core/src/test/resources/delegator-conf.yaml | 16 +-
core/src/test/resources/digitalpebble.com.html | 18 ++
.../digitalpebble.sitemap.extensions.all.xml | 18 ++
.../digitalpebble.sitemap.extensions.image.xml | 18 ++
.../digitalpebble.sitemap.extensions.links.xml | 18 ++
.../digitalpebble.sitemap.extensions.mobile.xml | 18 ++
.../digitalpebble.sitemap.extensions.news.xml | 18 ++
.../digitalpebble.sitemap.extensions.video.xml | 18 ++
core/src/test/resources/digitalpebble.sitemap.xml | 18 ++
core/src/test/resources/duplicateLinks.html | 18 ++
core/src/test/resources/javascriptLinks.html | 18 ++
core/src/test/resources/longtext.html | 18 ++
core/src/test/resources/redir.html | 20 +-
core/src/test/resources/stackexception.html | 155 ++++++-----
.../test/resources/tripadvisor.sitemap.index.xml | 21 +-
external/aws/README.md | 2 +-
external/aws/aws-conf.yaml | 15 ++
.../resources/archetype-resources/OS_IndexInit.sh | 15 ++
.../archetype-resources/crawler-conf.yaml | 15 ++
.../dashboards/importDashboards.sh | 15 ++
.../archetype-resources/opensearch-conf.yaml | 17 +-
external/opensearch/opensearch-conf.yaml | 17 +-
external/opensearch/pom.xml | 6 +
.../opensearch/bolt/AbstractOpenSearchTest.java | 21 +-
.../opensearch/bolt/IndexerBoltTest.java | 47 ++--
.../opensearch/bolt/StatusBoltTest.java | 62 ++---
external/playwright/README.md | 30 +++
.../playwright-conf.yaml} | 17 +-
external/{solr => playwright}/pom.xml | 39 +--
.../protocol/playwright/HttpProtocol.java | 297 +++++++++++++++++++++
.../protocol/playwright/LocalResourceHandler.java | 66 +++++
.../protocol/playwright/ProtocolTest.java | 156 +++++++++++
.../src/test/resources/dynamic-scraping.html | 50 ++++
external/pom.xml | 11 +-
external/solr/cores/docs/core.properties | 14 +
external/solr/cores/metrics/core.properties | 14 +
external/solr/cores/status/core.properties | 14 +
external/solr/solr-conf.yaml | 15 ++
external/sql/sql-conf.yaml | 15 ++
external/sql/tableCreation.script | 17 ++
.../org/apache/stormcrawler/tika/DOMBuilder.java | 4 +-
.../apache/stormcrawler/tika/ParserBoltTest.java | 45 +---
external/urlfrontier/pom.xml | 6 +
.../org/apache/stormcrawler/urlfrontier/Spout.java | 9 +-
.../urlfrontier/StatusUpdaterBolt.java | 11 +-
.../urlfrontier/StatusUpdaterBoltTest.java | 70 +++--
.../apache/stormcrawler/warc/WARCHdfsBoltTest.java | 90 ++++---
.../stormcrawler/warc/WARCRecordFormatTest.java | 160 +++++------
.../apache/stormcrawler/warc/WARCSpoutTest.java | 25 +-
pom.xml | 58 ++--
122 files changed, 2663 insertions(+), 1982 deletions(-)
create mode 100644 .rat-excludes
create mode 100644 DISCLAIMER-BINARIES.txt
rename core/src/test/java/org/apache/stormcrawler/helper/initialisation/{ClassInheritingFomAbstractAndInterface.java => ClassInheritingFromAbstractAndInterface.java} (93%)
copy core/src/test/java/org/apache/stormcrawler/protocol/{HttpRobotRulesParserTest.java => HttpRobotRulesParserRedirectTest.java} (59%)
create mode 100644 external/playwright/README.md
copy external/{solr/cores/docs/conf/stopwords.txt => playwright/playwright-conf.yaml} (57%)
copy external/{solr => playwright}/pom.xml (66%)
create mode 100644 external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
create mode 100644 external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/LocalResourceHandler.java
create mode 100644 external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ProtocolTest.java
create mode 100644 external/playwright/src/test/resources/dynamic-scraping.html