You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by jn...@apache.org on 2026/10/22 10:13:29 UTC
(stormcrawler) 01/01: #158 remove the DOM-based LDJsonParseFilter and LinkParseFilter
This is an automated email from the ASF dual-hosted git repository.
jnioche pushed a commit to branch remove-dom-ldjson-link-filters
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
commit 8eb73ee5cad9f76dbc5cfcfa499edacce9c96821
Author: Julien Nioche <ju...@digitalpebble.com>
AuthorDate: Thu Oct 22 11:13:20 2026 +0100
#158 remove the DOM-based LDJsonParseFilter and LinkParseFilter
Their JSoupFilter equivalents in org.apache.stormcrawler.jsoup do not need a DOM.
Also delegate needsDOM() in the OpenSearch JSONResourceWrapper and remove the
unused DOM-based RobotsTags.extractMetaTags(DocumentFragment).
Co-Authored-By: Claude Opus 5.5 <no...@anthropic.com>
---
THIRD-PARTY.txt | 295 ---------------------
.../apache/stormcrawler/jsoup/LinkParseFilter.java | 2 +-
.../parse/filter/LDJsonParseFilter.java | 123 ---------
.../stormcrawler/parse/filter/LinkParseFilter.java | 131 ---------
.../org/apache/stormcrawler/util/RobotsTags.java | 60 -----
.../stormcrawler/parse/filter/XPathFilterTest.java | 12 -
core/src/test/resources/test.parsefilters.json | 7 -
docs/src/main/asciidoc/internals.adoc | 4 +-
.../parse/filter/JSONResourceWrapper.java | 5 +
.../parse/filter/JSONResourceWrapperTest.java | 87 ++++++
10 files changed, 95 insertions(+), 631 deletions(-)
diff --git a/THIRD-PARTY.txt b/THIRD-PARTY.txt
index 9c5aabb8..c30afb4a 100644
--- a/THIRD-PARTY.txt
+++ b/THIRD-PARTY.txt
@@ -2,10 +2,6 @@
List of third-party dependencies grouped by their license type.
- 0BSD
-
- * XZ for Java (org.tukaani:xz:1.12 - https://tukaani.org/xz/java.html)
-
3-Clause BSD License
* Kryo (com.esotericsoftware:kryo:5.6.2 - https://github.com/EsotericSoftware/kryo/kryo)
@@ -18,306 +14,73 @@ List of third-party dependencies grouped by their license type.
Apache License, Version 2.0
- * aggs-matrix-stats (org.opensearch.plugin:aggs-matrix-stats-client:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * Apache Avro (org.apache.avro:avro:1.12.2 - https://avro.apache.org)
* Apache Commons CLI (commons-cli:commons-cli:1.11.0 - https://commons.apache.org/proper/commons-cli/)
* Apache Commons Codec (commons-codec:commons-codec:1.23.0 - https://commons.apache.org/proper/commons-codec/)
- * Apache Commons Collections (org.apache.commons:commons-collections4:4.6.0 - https://commons.apache.org/proper/commons-collections/)
* Apache Commons Compress (org.apache.commons:commons-compress:1.28.0 - https://commons.apache.org/proper/commons-compress/)
- * Apache Commons CSV (org.apache.commons:commons-csv:1.14.1 - https://commons.apache.org/proper/commons-csv/)
- * Apache Commons Exec (org.apache.commons:commons-exec:1.6.0 - https://commons.apache.org/proper/commons-exec/)
- * Apache Commons IO (commons-io:commons-io:2.17.0 - https://commons.apache.org/proper/commons-io/)
* Apache Commons IO (commons-io:commons-io:2.20.0 - https://commons.apache.org/proper/commons-io/)
* Apache Commons IO (commons-io:commons-io:2.22.0 - https://commons.apache.org/proper/commons-io/)
* Apache Commons Lang (org.apache.commons:commons-lang3:3.21.0 - https://commons.apache.org/proper/commons-lang/)
* Apache Commons Logging (commons-logging:commons-logging:1.2 - http://commons.apache.org/proper/commons-logging/)
* Apache Commons Logging (commons-logging:commons-logging:1.3.6 - https://commons.apache.org/proper/commons-logging/)
- * Apache Commons Logging (commons-logging:commons-logging:1.4.0 - https://commons.apache.org/proper/commons-logging/)
- * Apache Commons Math (org.apache.commons:commons-math3:3.6.1 - http://commons.apache.org/proper/commons-math/)
- * Apache FontBox (org.apache.pdfbox:fontbox:3.0.8 - http://pdfbox.apache.org/)
- * Apache Hadoop Client API (org.apache.hadoop:hadoop-client-api:3.5.0 - no url defined)
- * Apache Hadoop Client Runtime (org.apache.hadoop:hadoop-client-runtime:3.5.0 - no url defined)
- * Apache HttpAsyncClient (org.apache.httpcomponents:httpasyncclient:4.1.5 - http://hc.apache.org/httpcomponents-asyncclient)
* Apache HttpClient (org.apache.httpcomponents:httpclient:4.5.14 - http://hc.apache.org/httpcomponents-client-ga)
* Apache HttpClient (org.apache.httpcomponents.client5:httpclient5:5.6.4 - https://hc.apache.org/httpcomponents-client-5.6.x/5.6.4/httpclient5/)
* Apache HttpComponents Core HTTP/1.1 (org.apache.httpcomponents.core5:httpcore5:5.4.3 - https://hc.apache.org/httpcomponents-core-5.4.x/5.4.3/httpcore5/)
* Apache HttpComponents Core HTTP/2 (org.apache.httpcomponents.core5:httpcore5-h2:5.4.3 - https://hc.apache.org/httpcomponents-core-5.4.x/5.4.3/httpcore5-h2/)
* Apache HttpCore (org.apache.httpcomponents:httpcore:4.4.16 - http://hc.apache.org/httpcomponents-core-ga)
- * Apache HttpCore NIO (org.apache.httpcomponents:httpcore-nio:4.4.16 - http://hc.apache.org/httpcomponents-core-ga)
- * Apache James :: Mime4j :: Core (org.apache.james:apache-mime4j-core:0.8.15 - http://james.apache.org/mime4j/apache-mime4j-core)
- * Apache James :: Mime4j :: DOM (org.apache.james:apache-mime4j-dom:0.8.15 - http://james.apache.org/mime4j/apache-mime4j-dom)
* Apache Log4j API (org.apache.logging.log4j:log4j-api:2.26.1 - https://logging.apache.org/log4j/2.x/)
* Apache Log4j Core (org.apache.logging.log4j:log4j-core:2.26.1 - https://logging.apache.org/log4j/2.x/)
- * Apache Log4j JUL Adapter (org.apache.logging.log4j:log4j-jul:2.25.4 - https://logging.apache.org/log4j/2.x/)
- * Apache Lucene (module: backward-codecs) (org.apache.lucene:lucene-backward-codecs:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: common) (org.apache.lucene:lucene-analysis-common:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: core) (org.apache.lucene:lucene-core:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: grouping) (org.apache.lucene:lucene-grouping:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: highlighter) (org.apache.lucene:lucene-highlighter:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: join) (org.apache.lucene:lucene-join:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: memory) (org.apache.lucene:lucene-memory:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: misc) (org.apache.lucene:lucene-misc:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: queries) (org.apache.lucene:lucene-queries:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: queryparser) (org.apache.lucene:lucene-queryparser:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: sandbox) (org.apache.lucene:lucene-sandbox:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: spatial3d) (org.apache.lucene:lucene-spatial3d:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: spatial-extras) (org.apache.lucene:lucene-spatial-extras:9.12.3 - https://lucene.apache.org/)
- * Apache Lucene (module: suggest) (org.apache.lucene:lucene-suggest:9.12.3 - https://lucene.apache.org/)
- * Apache PDFBox (org.apache.pdfbox:pdfbox:3.0.8 - https://www.apache.org/pdfbox-parent/pdfbox/)
- * Apache PDFBox io (org.apache.pdfbox:pdfbox-io:3.0.8 - https://www.apache.org/pdfbox-parent/pdfbox-io/)
- * Apache PDFBox tools (org.apache.pdfbox:pdfbox-tools:3.0.8 - https://www.apache.org/pdfbox-parent/pdfbox-tools/)
- * Apache POI (org.apache.poi:poi-scratchpad:5.5.1 - https://poi.apache.org/)
- * Apache POI - API based on OPC and OOXML schemas (org.apache.poi:poi-ooxml:5.5.1 - https://poi.apache.org/)
- * Apache POI - Common (org.apache.poi:poi:5.5.1 - https://poi.apache.org/)
- * Apache Solr (module: api) (org.apache.solr:solr-api:10.0.0 - https://solr.apache.org/)
- * Apache Solr (module: solrj) (org.apache.solr:solr-solrj:10.0.0 - https://solr.apache.org/)
- * Apache Solr (module: solrj-jetty) (org.apache.solr:solr-solrj-jetty:10.0.0 - https://solr.apache.org/)
- * Apache Solr (module: solrj-zookeeper) (org.apache.solr:solr-solrj-zookeeper:10.0.0 - https://solr.apache.org/)
- * Apache Tika Apple parser module (org.apache.tika:tika-parser-apple-module:4.1.0 - https://tika.apache.org/tika-parser-apple-module/)
- * Apache Tika audiovideo parser module (org.apache.tika:tika-parser-audiovideo-module:4.1.0 - https://tika.apache.org/tika-parser-audiovideo-module/)
- * Apache Tika cad parser module (org.apache.tika:tika-parser-cad-module:4.1.0 - https://tika.apache.org/tika-parser-cad-module/)
- * Apache Tika code parser module (org.apache.tika:tika-parser-code-module:4.1.0 - https://tika.apache.org/tika-parser-code-module/)
* Apache Tika core (org.apache.tika:tika-core:4.1.0 - https://tika.apache.org/)
- * Apache Tika crypto parser module (org.apache.tika:tika-parser-crypto-module:4.1.0 - https://tika.apache.org/tika-parser-crypto-module/)
- * Apache Tika data URI commons (org.apache.tika:tika-parser-datauri-commons:4.1.0 - https://tika.apache.org/tika-parser-datauri-commons/)
- * Apache Tika digest commons (org.apache.tika:tika-parser-digest-commons:4.1.0 - https://tika.apache.org/tika-parser-digest-commons/)
- * Apache Tika eval core (org.apache.tika:tika-eval-core:4.1.0 - https://tika.apache.org/tika-eval-core/)
- * Apache Tika font parser module (org.apache.tika:tika-parser-font-module:4.1.0 - https://tika.apache.org/tika-parser-font-module/)
- * Apache Tika HTML encoding detector (org.apache.tika:tika-encoding-detector-html:4.1.0 - https://tika.apache.org/tika-encoding-detectors/tika-encoding-detector-html/)
- * Apache Tika html parser module (org.apache.tika:tika-parser-html-module:4.1.0 - https://tika.apache.org/tika-parser-html-module/)
- * Apache Tika image parser module (org.apache.tika:tika-parser-image-module:4.1.0 - https://tika.apache.org/tika-parser-image-module/)
- * Apache Tika langdetect (built-in charsoup) (org.apache.tika:tika-langdetect-charsoup:4.1.0 - https://tika.apache.org/tika-langdetect-charsoup/)
- * Apache Tika langdetect (charsoup core — no Tika dependencies) (org.apache.tika:tika-langdetect-charsoup-core:4.1.0 - https://tika.apache.org/tika-langdetect-charsoup-core/)
- * Apache Tika mail commons (org.apache.tika:tika-parser-mail-commons:4.1.0 - https://tika.apache.org/tika-parser-mail-commons/)
- * Apache Tika mail parser module (org.apache.tika:tika-parser-mail-module:4.1.0 - https://tika.apache.org/tika-parser-mail-module/)
- * Apache Tika Microsoft parser module (org.apache.tika:tika-parser-microsoft-module:4.1.0 - https://tika.apache.org/tika-parser-microsoft-module/)
- * Apache Tika miscellaneous office format parser module (org.apache.tika:tika-parser-miscoffice-module:4.1.0 - https://tika.apache.org/tika-parser-miscoffice-module/)
- * Apache Tika ML charset encoding detector (org.apache.tika:tika-encoding-detector-mojibuster:4.1.0 - https://tika.apache.org/tika-encoding-detectors/tika-encoding-detector-mojibuster/)
- * Apache Tika ML core (no Tika dependencies) (org.apache.tika:tika-ml-core:4.1.0 - https://tika.apache.org/tika-ml-core/)
- * Apache Tika ML junk detector — runtime and training tools (org.apache.tika:tika-ml-junkdetect:4.1.0 - https://tika.apache.org/tika-ml-junkdetect/)
- * Apache Tika news parser module (org.apache.tika:tika-parser-news-module:4.1.0 - https://tika.apache.org/tika-parser-news-module/)
- * Apache Tika OCR parser module (org.apache.tika:tika-parser-ocr-module:4.1.0 - https://tika.apache.org/tika-parser-ocr-module/)
- * Apache Tika package parser module (org.apache.tika:tika-parser-pkg-module:4.1.0 - https://tika.apache.org/tika-parser-pkg-module/)
- * Apache Tika PDF parser module (org.apache.tika:tika-parser-pdf-module:4.1.0 - https://tika.apache.org/tika-parser-pdf-module/)
- * Apache Tika pipes api (org.apache.tika:tika-pipes-api:4.1.0 - https://tika.apache.org/)
- * Apache Tika pipes core (org.apache.tika:tika-pipes-core:4.1.0 - https://tika.apache.org/)
- * Apache Tika Pipes File System (org.apache.tika:tika-pipes-file-system:4.1.0 - https://tika.apache.org/tika-pipes-file-system/)
- * Apache Tika pipes fork parser (org.apache.tika:tika-pipes-fork-parser:4.1.0 - https://tika.apache.org/)
- * Apache Tika Pipes iterators - base (org.apache.tika:tika-pipes-iterator-commons:4.1.0 - https://tika.apache.org/)
- * Apache Tika Pipes Reporter - base (org.apache.tika:tika-pipes-reporter-commons:4.1.0 - https://tika.apache.org/)
- * Apache Tika plugins core (org.apache.tika:tika-plugins-core:4.1.0 - https://tika.apache.org)
- * Apache Tika serialization (org.apache.tika:tika-serialization:4.1.0 - https://tika.apache.org)
- * Apache Tika standard parser package (org.apache.tika:tika-parsers-standard-package:4.1.0 - https://tika.apache.org/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/)
- * Apache Tika text parser module (org.apache.tika:tika-parser-text-module:4.1.0 - https://tika.apache.org/tika-parser-text-module/)
- * Apache Tika WARC parser module (org.apache.tika:tika-parser-webarchive-module:4.1.0 - https://tika.apache.org/tika-parser-webarchive-module/)
- * Apache Tika XML parser module (org.apache.tika:tika-parser-xml-module:4.1.0 - https://tika.apache.org/tika-parser-xml-module/)
- * Apache Tika XMP commons (org.apache.tika:tika-parser-xmp-commons:4.1.0 - https://tika.apache.org/tika-parser-xmp-commons/)
- * Apache Tika ZIP commons (org.apache.tika:tika-parser-zip-commons:4.1.0 - https://tika.apache.org/tika-parser-zip-commons/)
- * Apache ZooKeeper - Jute (org.apache.zookeeper:zookeeper-jute:3.9.4 - http://zookeeper.apache.org/zookeeper-jute)
- * Apache ZooKeeper - Server (org.apache.zookeeper:zookeeper:3.9.4 - http://zookeeper.apache.org/zookeeper)
- * AWS Event Stream (software.amazon.eventstream:eventstream:1.0.1 - https://github.com/awslabs/aws-eventstream-java)
- * AWS Java SDK :: Annotations (software.amazon.awssdk:annotations:2.56.0 - https://aws.amazon.com/sdkforjava/core/annotations)
- * AWS Java SDK :: Arns (software.amazon.awssdk:arns:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Auth (software.amazon.awssdk:auth:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: AWS Core (software.amazon.awssdk:aws-core:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: AWS CRT Core (software.amazon.awssdk:crt-core:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Checksums (software.amazon.awssdk:checksums:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Checksums SPI (software.amazon.awssdk:checksums-spi:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Core :: Protocols :: AWS Json Protocol (software.amazon.awssdk:aws-json-protocol:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Core :: Protocols :: AWS Query Protocol (software.amazon.awssdk:aws-query-protocol:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Core :: Protocols :: AWS Xml Protocol (software.amazon.awssdk:aws-xml-protocol:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Core :: Protocols :: Json Utils (software.amazon.awssdk:json-utils:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Core :: Protocols :: Protocol Core (software.amazon.awssdk:protocol-core:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Endpoints SPI (software.amazon.awssdk:endpoints-spi:2.56.0 - https://aws.amazon.com/sdkforjava/core/endpoints-spi)
- * AWS Java SDK :: HTTP Auth (software.amazon.awssdk:http-auth:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: HTTP Auth AWS (software.amazon.awssdk:http-auth-aws:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: HTTP Auth Event Stream (software.amazon.awssdk:http-auth-aws-eventstream:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: HTTP Auth SPI (software.amazon.awssdk:http-auth-spi:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: HTTP Client Interface (software.amazon.awssdk:http-client-spi:2.56.0 - https://aws.amazon.com/sdkforjava/http-client-spi)
- * AWS Java SDK :: HTTP Clients :: Apache5 (software.amazon.awssdk:apache5-client:2.56.0 - https://aws.amazon.com/sdkforjava/http-clients/apache5-client)
- * AWS Java SDK :: HTTP Clients :: Netty Non-Blocking I/O (software.amazon.awssdk:netty-nio-client:2.56.0 - https://aws.amazon.com/sdkforjava/http-clients/netty-nio-client)
- * AWS Java SDK :: Identity SPI (software.amazon.awssdk:identity-spi:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Metrics SPI (software.amazon.awssdk:metrics-spi:2.56.0 - https://aws.amazon.com/sdkforjava/core/metrics-spi)
- * AWS Java SDK :: Profiles (software.amazon.awssdk:profiles:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Regions (software.amazon.awssdk:regions:2.56.0 - https://aws.amazon.com/sdkforjava/core/regions)
- * AWS Java SDK :: Retries (software.amazon.awssdk:retries:2.56.0 - https://aws.amazon.com/sdkforjava/core/retries)
- * AWS Java SDK :: Retries API (software.amazon.awssdk:retries-spi:2.56.0 - https://aws.amazon.com/sdkforjava/core/retries-spi)
- * AWS Java SDK :: SDK Core (software.amazon.awssdk:sdk-core:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Services :: Amazon CloudSearch (software.amazon.awssdk:cloudsearch:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Services :: Amazon CloudSearch Domain (software.amazon.awssdk:cloudsearchdomain:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Services :: Amazon S3 (software.amazon.awssdk:s3:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Third Party :: Jackson-core (software.amazon.awssdk:third-party-jackson-core:2.56.0 - https://aws.amazon.com/sdkforjava)
- * AWS Java SDK :: Utilities (software.amazon.awssdk:utils:2.56.0 - https://aws.amazon.com/sdkforjava/utils)
- * AWS Java SDK :: Utils Lite (software.amazon.awssdk:utils-lite:2.56.0 - https://aws.amazon.com/sdkforjava)
* Caffeine cache (com.github.ben-manes.caffeine:caffeine:3.3.1 - https://github.com/ben-manes/caffeine)
- * com.drewnoakes:metadata-extractor (com.drewnoakes:metadata-extractor:2.21.0 - https://drewnoakes.com/code/exif/)
- * compiler (com.github.spullara.mustache.java:compiler:0.9.14 - http://github.com/spullara/mustache.java)
* Crawler-commons (com.github.crawler-commons:crawler-commons:1.6 - https://github.com/crawler-commons/crawler-commons)
- * Curator Client (org.apache.curator:curator-client:5.9.0 - https://curator.apache.org/curator-client)
- * Curator Framework (org.apache.curator:curator-framework:5.9.0 - https://curator.apache.org/curator-framework)
- * error-prone annotations (com.google.errorprone:error_prone_annotations:2.45.0 - https://errorprone.info/error_prone_annotations)
- * error-prone annotations (com.google.errorprone:error_prone_annotations:2.48.0 - https://errorprone.info/error_prone_annotations)
* error-prone annotations (com.google.errorprone:error_prone_annotations:2.50.0 - https://errorprone.info/error_prone_annotations)
* FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/)
- * Google Android Annotations Library (com.google.android:annotations:4.1.1.4 - http://source.android.com/)
* Graphite Integration for Metrics (io.dropwizard.metrics:metrics-graphite:4.2.40 - https://metrics.dropwizard.io/metrics-graphite)
- * Gson (com.google.code.gson:gson:2.11.0 - https://github.com/google/gson)
- * Gson (com.google.code.gson:gson:2.14.0 - https://github.com/google/gson)
- * Guava: Google Core Libraries for Java (com.google.guava:guava:18.0 - http://code.google.com/p/guava-libraries/guava)
* Guava: Google Core Libraries for Java (com.google.guava:guava:33.7.2-jre - https://github.com/google/guava)
* Guava InternalFutureFailureAccess and InternalFutures (com.google.guava:failureaccess:1.0.3 - https://github.com/google/guava/failureaccess)
* Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture)
- * IntelliJ IDEA Annotations (com.intellij:annotations:12.0 - http://www.jetbrains.org)
- * io.grpc:grpc-api (io.grpc:grpc-api:1.76.0 - https://github.com/grpc/grpc-java)
- * io.grpc:grpc-context (io.grpc:grpc-context:1.76.0 - https://github.com/grpc/grpc-java)
- * io.grpc:grpc-core (io.grpc:grpc-core:1.76.0 - https://github.com/grpc/grpc-java)
- * io.grpc:grpc-netty-shaded (io.grpc:grpc-netty-shaded:1.76.0 - https://github.com/grpc/grpc-java)
- * io.grpc:grpc-protobuf (io.grpc:grpc-protobuf:1.76.0 - https://github.com/grpc/grpc-java)
- * io.grpc:grpc-protobuf-lite (io.grpc:grpc-protobuf-lite:1.76.0 - https://github.com/grpc/grpc-java)
- * io.grpc:grpc-services (io.grpc:grpc-services:1.76.0 - https://github.com/grpc/grpc-java)
- * io.grpc:grpc-stub (io.grpc:grpc-stub:1.76.0 - https://github.com/grpc/grpc-java)
- * io.grpc:grpc-util (io.grpc:grpc-util:1.76.0 - https://github.com/grpc/grpc-java)
* J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.1 - https://github.com/google/j2objc/)
- * Jackcess (com.healthmarketscience.jackcess:jackcess:5.0.0 - https://jackcess.sourceforge.io)
- * Jackcess Encrypt (com.healthmarketscience.jackcess:jackcess-encrypt:5.0.0 - http://jackcessencrypt.sf.net)
* Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.22 - https://github.com/FasterXML/jackson)
* Jackson-core (com.fasterxml.jackson.core:jackson-core:2.22.0 - https://github.com/FasterXML/jackson-core)
* jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.22.0 - https://github.com/FasterXML/jackson)
- * Jackson dataformat: CBOR (com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.22.0 - https://github.com/FasterXML/jackson-dataformats-binary)
- * Jackson dataformat: Smile (com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.22.0 - https://github.com/FasterXML/jackson-dataformats-binary)
- * Jackson-dataformat-YAML (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.22.0 - https://github.com/FasterXML/jackson-dataformats-text)
- * Jackson datatype: JSR310 (com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.22.0 - https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jsr310)
- * java-libpst (com.pff:java-libpst:0.9.3 - https://github.com/rjohnsondev/java-libpst)
- * JCL 1.2 implemented over SLF4J (org.slf4j:jcl-over-slf4j:2.0.17 - http://www.slf4j.org)
- * JCL 1.2 implemented over SLF4J (org.slf4j:jcl-over-slf4j:2.0.20 - http://www.slf4j.org)
* JetBrains Java Annotations (org.jetbrains:annotations:26.1.0 - https://github.com/JetBrains/java-annotations)
- * Joda-Time (joda-time:joda-time:2.12.7 - https://www.joda.org/joda-time/)
- * jsonic (net.arnx:jsonic:1.2.11 - http://jsonic.sourceforge.jp/)
* JSpecify annotations (org.jspecify:jspecify:1.0.0 - http://jspecify.org/)
- * JSpecify annotations (org.jspecify:jspecify:1.0.1 - https://jspecify.dev/)
* JVM Integration for Metrics (io.dropwizard.metrics:metrics-jvm:4.2.40 - https://metrics.dropwizard.io/metrics-jvm)
- * jwarc (org.netpreserve:jwarc:0.37.0 - https://github.com/iipc/jwarc)
* Kotlin Stdlib (org.jetbrains.kotlin:kotlin-stdlib:2.1.21 - https://kotlinlang.org/)
- * LangChain4j :: Core (dev.langchain4j:langchain4j-core:1.22.0 - https://github.com/langchain4j/langchain4j/tree/main/langchain4j-core)
- * LangChain4j :: HTTP Client :: JDK HttpClient (dev.langchain4j:langchain4j-http-client-jdk:1.22.0 - https://github.com/langchain4j/langchain4j/tree/main/langchain4j-http-client-jdk)
- * LangChain4j :: HTTP Client (dev.langchain4j:langchain4j-http-client:1.22.0 - https://github.com/langchain4j/langchain4j/tree/main/langchain4j-http-client)
- * LangChain4j :: Integration :: OpenAI (dev.langchain4j:langchain4j-open-ai:1.22.0 - https://github.com/langchain4j/langchain4j/tree/main/langchain4j-open-ai)
- * LangChain4j :: Reactive Streaming (dev.langchain4j:langchain4j-reactive-streaming:1.22.0-beta32 - https://github.com/langchain4j/langchain4j/tree/main/langchain4j-reactive-streaming)
- * LangChain4j (dev.langchain4j:langchain4j:1.22.0 - https://github.com/langchain4j/langchain4j/tree/main/langchain4j)
- * lang-mustache (org.opensearch.plugin:lang-mustache-client:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * language-detector (com.optimaize.languagedetector:language-detector:0.6 - https://github.com/optimaize/language-detector)
- * mapper-extras (org.opensearch.plugin:mapper-extras-client:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
* Metrics Core (io.dropwizard.metrics:metrics-core:4.2.40 - https://metrics.dropwizard.io/metrics-core)
* Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.40 - https://metrics.dropwizard.io/metrics-jmx)
- * Netty/Buffer (io.netty:netty-buffer:4.1.119.Final - https://netty.io/netty-buffer/)
- * Netty/Buffer (io.netty:netty-buffer:4.1.138.Final - https://netty.io/netty-buffer/)
* Netty/Buffer (io.netty:netty-buffer:4.2.17.Final - https://netty.io/netty-buffer/)
* Netty/Codec/Base (io.netty:netty-codec-base:4.2.17.Final - https://netty.io/netty-codec-base/)
* Netty/Codec/Compression (io.netty:netty-codec-compression:4.2.17.Final - https://netty.io/netty-codec-compression/)
- * Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.138.Final - https://netty.io/netty-codec-http/)
- * Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.138.Final - https://netty.io/netty-codec-http2/)
* Netty/Codec/Marshalling (io.netty:netty-codec-marshalling:4.2.17.Final - https://netty.io/netty-codec-marshalling/)
* Netty/Codec/Protobuf (io.netty:netty-codec-protobuf:4.2.17.Final - https://netty.io/netty-codec-protobuf/)
- * Netty/Codec (io.netty:netty-codec:4.1.119.Final - https://netty.io/netty-codec/)
- * Netty/Codec (io.netty:netty-codec:4.1.138.Final - https://netty.io/netty-codec/)
* Netty/Codec (io.netty:netty-codec:4.2.17.Final - https://netty.io/netty-codec/)
- * Netty/Common (io.netty:netty-common:4.1.119.Final - https://netty.io/netty-common/)
- * Netty/Common (io.netty:netty-common:4.1.138.Final - https://netty.io/netty-common/)
* Netty/Common (io.netty:netty-common:4.2.17.Final - https://netty.io/netty-common/)
- * Netty/Handler (io.netty:netty-handler:4.1.119.Final - https://netty.io/netty-handler/)
- * Netty/Handler (io.netty:netty-handler:4.1.138.Final - https://netty.io/netty-handler/)
* Netty/Handler (io.netty:netty-handler:4.2.17.Final - https://netty.io/netty-handler/)
- * Netty/Resolver (io.netty:netty-resolver:4.1.119.Final - https://netty.io/netty-resolver/)
- * Netty/Resolver (io.netty:netty-resolver:4.1.138.Final - https://netty.io/netty-resolver/)
* Netty/Resolver (io.netty:netty-resolver:4.2.17.Final - https://netty.io/netty-resolver/)
- * Netty/TomcatNative [BoringSSL - Static] (io.netty:netty-tcnative-boringssl-static:2.0.70.Final - https://github.com/netty/netty-tcnative/netty-tcnative-boringssl-static/)
- * Netty/TomcatNative [OpenSSL - Classes] (io.netty:netty-tcnative-classes:2.0.70.Final - https://github.com/netty/netty-tcnative/netty-tcnative-classes/)
- * Netty/Transport/Classes/Epoll (io.netty:netty-transport-classes-epoll:4.1.119.Final - https://netty.io/netty-transport-classes-epoll/)
- * Netty/Transport/Classes/Epoll (io.netty:netty-transport-classes-epoll:4.1.138.Final - https://netty.io/netty-transport-classes-epoll/)
- * Netty/Transport/Native/Epoll (io.netty:netty-transport-native-epoll:4.1.119.Final - https://netty.io/netty-transport-native-epoll/)
- * Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.119.Final - https://netty.io/netty-transport-native-unix-common/)
- * Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.138.Final - https://netty.io/netty-transport-native-unix-common/)
* Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.2.17.Final - https://netty.io/netty-transport-native-unix-common/)
- * Netty/Transport (io.netty:netty-transport:4.1.119.Final - https://netty.io/netty-transport/)
- * Netty/Transport (io.netty:netty-transport:4.1.138.Final - https://netty.io/netty-transport/)
* Netty/Transport (io.netty:netty-transport:4.2.17.Final - https://netty.io/netty-transport/)
- * Non-Blocking Reactive Foundation for the JVM (io.projectreactor:reactor-core:3.7.19 - https://github.com/reactor/reactor-core)
* Objenesis (org.objenesis:objenesis:3.3 - http://objenesis.org/objenesis)
- * Ogg and Vorbis for Java, Core (org.gagravarr:vorbis-java-core:0.8 - https://github.com/Gagravarr/VorbisJava)
* okhttp (com.squareup.okhttp3:okhttp:5.5.0 - https://lysine.dev/okhttp/)
* okhttp (com.squareup.okhttp3:okhttp-jvm:5.5.0 - https://lysine.dev/okhttp/)
* okhttp-brotli (com.squareup.okhttp3:okhttp-brotli:5.5.0 - https://lysine.dev/okhttp/)
* okhttp-zstd (com.squareup.okhttp3:okhttp-zstd:5.5.0 - https://lysine.dev/okhttp/)
* okio (com.squareup.okio:okio:3.18.1 - https://github.com/lysine-dev/okio/)
* okio (com.squareup.okio:okio-jvm:3.18.1 - https://github.com/lysine-dev/okio/)
- * opensearch-cli (org.opensearch:opensearch-cli:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * opensearch-common (org.opensearch:opensearch-common:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * opensearch-compress (org.opensearch:opensearch-compress:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * opensearch-core (org.opensearch:opensearch-core:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * opensearch-geo (org.opensearch:opensearch-geo:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
* OpenSearch Java Client (org.opensearch.client:opensearch-java:3.10.0 - https://github.com/opensearch-project/opensearch-java/)
- * opensearch-secure-sm (org.opensearch:opensearch-secure-sm:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * opensearch-task-commons (org.opensearch:opensearch-task-commons:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * opensearch-telemetry (org.opensearch:opensearch-telemetry:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * opensearch-x-content (org.opensearch:opensearch-x-content:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * org.opentest4j:opentest4j (org.opentest4j:opentest4j:1.3.0 - https://github.com/ota4j-team/opentest4j)
- * org.roaringbitmap:RoaringBitmap (org.roaringbitmap:RoaringBitmap:1.3.0 - https://github.com/RoaringBitmap/RoaringBitmap)
- * parent-join (org.opensearch.plugin:parent-join-client:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * parso (com.epam:parso:2.0.14 - https://github.com/epam/parso)
- * PDFBox JBIG2 ImageIO plugin (org.apache.pdfbox:jbig2-imageio:3.0.5 - https://www.apache.org/jbig2-imageio/)
- * perfmark:perfmark-api (io.perfmark:perfmark-api:0.27.0 - https://github.com/perfmark/perfmark)
- * PF4J (org.pf4j:pf4j:3.16.0 - https://pf4j.org/pf4j)
- * Playwright - Driver (com.microsoft.playwright:driver:1.63.0 - https://github.com/microsoft/playwright-java/driver)
- * Playwright - Main Library (com.microsoft.playwright:playwright:1.63.0 - https://github.com/microsoft/playwright-java/playwright)
- * Playwright - Node.js For All Platforms (com.microsoft.playwright:driver-bundle:1.63.0 - https://github.com/microsoft/playwright-java/driver-bundle)
- * proto-google-common-protos (com.google.api.grpc:proto-google-common-protos:2.59.2 - https://github.com/googleapis/sdk-platform-java)
- * rank-eval (org.opensearch.plugin:rank-eval-client:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * rest (org.opensearch.client:opensearch-rest-client:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * rest-high-level (org.opensearch.client:opensearch-rest-high-level-client:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
* rome (com.rometools:rome:2.1.0 - http://rometools.com/rome)
* rome-utils (com.rometools:rome-utils:2.1.0 - http://rometools.com/rome-utils)
- * server (org.opensearch:opensearch:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * SmallRye Mutiny Zero (io.smallrye.reactive:mutiny-zero:1.3.1 - https://smallrye.io)
* SnakeYAML (org.yaml:snakeyaml:2.7 - https://codeberg.org/snakeyaml/snakeyaml)
- * snappy-java (org.xerial.snappy:snappy-java:1.1.10.4 - https://github.com/xerial/snappy-java)
- * sniffer (org.opensearch.client:opensearch-rest-client-sniffer:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * SparseBitSet (com.zaxxer:SparseBitSet:1.3 - https://github.com/brettwooldridge/SparseBitSet)
- * Storm Autocreds (org.apache.storm:storm-autocreds:3.1.0 - https://storm.apache.org/storm-external/storm-autocreds)
* Storm Client (org.apache.storm:storm-client:3.1.0 - https://storm.apache.org/storm-client)
- * Storm HDFS (org.apache.storm:storm-hdfs:3.1.0 - https://storm.apache.org/storm-external/storm-hdfs)
* Storm Shaded Deps (org.apache.storm:storm-shaded-deps:3.1.0 - https://storm.apache.org/storm-shaded-deps)
- * swagger-annotations-jakarta (io.swagger.core.v3:swagger-annotations-jakarta:2.2.22 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations-jakarta)
- * T-Digest (com.tdunning:t-digest:3.2 - https://github.com/tdunning/t-digest)
- * urlfrontier-API (com.github.crawler-commons:urlfrontier-API:2.6 - https://github.com/crawler-commons/url-frontier/urlfrontier-API)
* Xerces2-j (xerces:xercesImpl:2.12.2 - https://xerces.apache.org/xerces2-j/)
- * XmlBeans (org.apache.xmlbeans:xmlbeans:5.3.0 - https://xmlbeans.apache.org/)
* zstd-kmp (com.squareup.zstd:zstd-kmp-jvm:0.4.0 - https://github.com/square/okio-zstd/)
* zstd-kmp-okio (com.squareup.zstd:zstd-kmp-okio-jvm:0.4.0 - https://github.com/square/okio-zstd/)
- Apache License, Version 2.0, Eclipse Public License - Version 2.0
-
- * Core :: ALPN :: Client (org.eclipse.jetty:jetty-alpn-client:12.0.27 - https://jetty.org/jetty-core/jetty-alpn/jetty-alpn-client)
- * Core :: ALPN :: Java Client (org.eclipse.jetty:jetty-alpn-java-client:12.0.27 - https://jetty.org/jetty-core/jetty-alpn/jetty-alpn-java-client)
- * Core :: HTTP (org.eclipse.jetty:jetty-http:12.0.27 - https://jetty.org/jetty-core/jetty-http)
- * Core :: HTTP2 :: Client (org.eclipse.jetty.http2:jetty-http2-client:12.0.27 - https://jetty.org/jetty-core/jetty-http2/jetty-http2-client)
- * Core :: HTTP2 :: Client Transport (org.eclipse.jetty.http2:jetty-http2-client-transport:12.0.27 - https://jetty.org/jetty-core/jetty-http2/jetty-http2-client-transport)
- * Core :: HTTP2 :: Common (org.eclipse.jetty.http2:jetty-http2-common:12.0.27 - https://jetty.org/jetty-core/jetty-http2/jetty-http2-common)
- * Core :: HTTP2 :: HPACK (org.eclipse.jetty.http2:jetty-http2-hpack:12.0.27 - https://jetty.org/jetty-core/jetty-http2/jetty-http2-hpack)
- * Core :: HTTP Client (org.eclipse.jetty:jetty-client:12.0.27 - https://jetty.org/jetty-core/jetty-client)
- * Core :: IO (org.eclipse.jetty:jetty-io:12.0.27 - https://jetty.org/jetty-core/jetty-io)
- * Core :: Utilities (org.eclipse.jetty:jetty-util:12.0.27 - https://jetty.org/jetty-core/jetty-util)
-
- Apache License, Version 2.0, LGPL-2.1-or-later
-
- * Java Native Access (net.java.dev.jna:jna:5.13.0 - https://github.com/java-native-access/jna)
-
Bouncy Castle Licence
* Bouncy Castle ASN.1 Extension and Utility APIs (org.bouncycastle:bcutil-jdk18on:1.85 - https://www.bouncycastle.org/download/bouncy-castle-java/)
- * Bouncy Castle JavaMail Jakarta S/MIME APIs (org.bouncycastle:bcjmail-jdk18on:1.86 - https://www.bouncycastle.org/download/bouncy-castle-java/)
* Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (org.bouncycastle:bcpkix-jdk18on:1.85 - https://www.bouncycastle.org/download/bouncy-castle-java/)
* Bouncy Castle Provider (org.bouncycastle:bcprov-jdk18on:1.85 - https://www.bouncycastle.org/download/bouncy-castle-java/)
- * Bouncy Castle Provider (org.bouncycastle:bcprov-jdk18on:1.86 - https://www.bouncycastle.org/download/bouncy-castle-java/)
BSD-2-Clause
@@ -325,44 +88,14 @@ List of third-party dependencies grouped by their license type.
* commonmark-java extension for strikethrough (org.commonmark:commonmark-ext-gfm-strikethrough:0.30.0 - https://github.com/commonmark/commonmark-java/commonmark-ext-gfm-strikethrough)
* commonmark-java extension for tables (org.commonmark:commonmark-ext-gfm-tables:0.30.0 - https://github.com/commonmark/commonmark-java/commonmark-ext-gfm-tables)
- BSD-2-Clause, Public Domain, per Creative Commons CC0
-
- * HdrHistogram (org.hdrhistogram:HdrHistogram:2.2.2 - http://hdrhistogram.github.io/HdrHistogram/)
-
BSD 2-Clause License
* zstd-jni (com.github.luben:zstd-jni:1.5.7-15 - https://github.com/luben/zstd-jni)
- BSD 3-Clause License
-
- * Adobe XMPCore (com.adobe.xmp:xmpcore:6.1.11 - https://www.adobe.com/devnet/xmp/library/eula-xmp-library-java.html)
- * asm (org.ow2.asm:asm:9.10.1 - http://asm.ow2.io/)
- * Protocol Buffers [Core] (com.google.protobuf:protobuf-java:3.25.8 - https://developers.google.com/protocol-buffers/protobuf-java/)
- * Protocol Buffers [Core] (com.google.protobuf:protobuf-java:4.31.1 - https://developers.google.com/protocol-buffers/protobuf-java/)
- * Protocol Buffers [Util] (com.google.protobuf:protobuf-java-util:3.25.8 - https://developers.google.com/protocol-buffers/protobuf-java-util/)
-
- BSD License
-
- * curvesapi (com.github.virtuald:curvesapi:1.08 - https://github.com/virtuald/curvesapi)
- * JMatIO (org.tallison:jmatio:1.5 - https://github.com/tballison/jmatio)
- * JZlib (com.jcraft:jzlib:1.1.3 - http://www.jcraft.com/jzlib/)
-
- CDDL, v1.0, LGPL, v2.1 or later
-
- * JHighlight (org.codelibs:jhighlight:2.0.0 - https://github.com/codelibs/jhighlight)
-
- CDDL/GPLv2+CE
-
- * JavaBeans Activation Framework API jar (javax.activation:javax.activation-api:1.2.0 - http://java.net/all/javax.activation-api/)
-
Common Development and Distribution License
* javax.annotation API (javax.annotation:javax.annotation-api:1.3.2 - http://jcp.org/en/jsr/detail?id=250)
- Common Development and Distribution License (CDDL) v1.1, The GNU General Public License (GPL), Version 2, With Classpath Exception
-
- * jaxb-api (javax.xml.bind:jaxb-api:2.3.1 - https://github.com/javaee/jaxb-spec/jaxb-api)
-
Eclipse Distribution License, Version 1.0
* Jakarta Activation (com.sun.activation:jakarta.activation:2.0.1 - https://github.com/eclipse-ee4j/jaf/jakarta.activation)
@@ -373,10 +106,6 @@ List of third-party dependencies grouped by their license type.
* org.eclipse.yasson (org.eclipse:yasson:2.0.2 - https://projects.eclipse.org/projects/ee4j.yasson)
- Eclipse Public License, Version 2.0, GPL-2.0-with-classpath-exception
-
- * Jakarta RESTful WS API (jakarta.ws.rs:jakarta.ws.rs-api:3.1.0 - https://github.com/eclipse-ee4j/jaxrs-api)
-
Eclipse Public License, Version 2.0, The GNU General Public License (GPL), Version 2, With Classpath Exception
* Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:1.3.5 - https://projects.eclipse.org/projects/ee4j.ca)
@@ -388,23 +117,11 @@ List of third-party dependencies grouped by their license type.
* JSON-B API (jakarta.json.bind:jakarta.json.bind-api:2.0.0 - https://eclipse-ee4j.github.io/jsonb-api)
* JSON-P Default Provider (org.glassfish:jakarta.json:2.0.0 - https://github.com/eclipse-ee4j/jsonp)
- MIT-0
-
- * reactive-streams (org.reactivestreams:reactive-streams:1.0.4 - http://www.reactive-streams.org/)
-
MIT License
- * Animal Sniffer Annotations (org.codehaus.mojo:animal-sniffer-annotations:1.24 - https://www.mojohaus.org/animal-sniffer/animal-sniffer-annotations)
- * dd-plist (com.googlecode.plist:dd-plist:1.30 - http://www.github.com/3breadt/dd-plist)
- * Java SemVer (com.github.zafarkhaja:java-semver:0.10.2 - https://github.com/zafarkhaja/jsemver)
- * JOpt Simple (net.sf.jopt-simple:jopt-simple:5.0.4 - http://jopt-simple.github.io/jopt-simple)
* jsoup Java HTML Parser (org.jsoup:jsoup:1.23.2 - https://jsoup.org/)
- * JTokkit (com.knuddels:jtokkit:1.1.0 - https://github.com/knuddelsgmbh/jtokkit)
* org.brotli:dec (org.brotli:dec:0.1.2 - http://brotli.org/dec)
- * semver4j (org.semver4j:semver4j:6.0.0 - https://github.com/semver4j/semver4j)
* SLF4J API Module (org.slf4j:slf4j-api:1.7.36 - http://www.slf4j.org)
- * SLF4J API Module (org.slf4j:slf4j-api:1.7.6 - http://www.slf4j.org)
- * SLF4J API Module (org.slf4j:slf4j-api:2.0.17 - http://www.slf4j.org)
* SLF4J API Module (org.slf4j:slf4j-api:2.0.18 - http://www.slf4j.org)
* SLF4J API Module (org.slf4j:slf4j-api:2.0.20 - http://www.slf4j.org)
@@ -412,18 +129,6 @@ List of third-party dependencies grouped by their license type.
* JDOM (org.jdom:jdom2:2.0.6.1 - http://www.jdom.org)
- The Apache Software License, version 2.0
-
- * picocli (info.picocli:picocli:4.7.7 - https://picocli.info)
-
- The GNU General Public License, v2 with Universal FOSS Exception, v1.0
-
- * MySQL Connector/J (com.mysql:mysql-connector-j:26.7.0 - http://dev.mysql.com/doc/connector-j/en/)
-
Unicode-3.0
* ICU4J (com.ibm.icu:icu4j:78.3 - https://icu.unicode.org/)
-
- UnRar License
-
- * Java Unrar (com.github.junrar:junrar:8.1.1 - https://github.com/junrar/junrar)
diff --git a/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java b/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
index 92a114f0..91d7fb29 100644
--- a/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
@@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory;
*
* <pre>{@code
* {
- * "class": "org.apache.stormcrawler.parse.filter.LinkParseFilter",
+ * "class": "org.apache.stormcrawler.jsoup.LinkParseFilter",
* "name": "LinkParseFilter",
* "params": {
* "pattern": "//IMG/@src",
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/LDJsonParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/LDJsonParseFilter.java
deleted file mode 100644
index 8a91c6aa..00000000
--- a/core/src/main/java/org/apache/stormcrawler/parse/filter/LDJsonParseFilter.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.parse.filter;
-
-import com.fasterxml.jackson.core.JsonPointer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpression;
-import javax.xml.xpath.XPathFactory;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.parse.ParseData;
-import org.apache.stormcrawler.parse.ParseFilter;
-import org.apache.stormcrawler.parse.ParseResult;
-import org.jetbrains.annotations.NotNull;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.DocumentFragment;
-import org.w3c.dom.Node;
-
-/** Extracts data from JSON-LD representation (https://json-ld.org/). */
-public class LDJsonParseFilter extends ParseFilter {
-
- public static final Logger LOG = LoggerFactory.getLogger(LDJsonParseFilter.class);
-
- private static XPathFactory factory = XPathFactory.newInstance();
- private static XPath xpath = factory.newXPath();
-
- private static ObjectMapper mapper = new ObjectMapper();
-
- private List<LabelledJsonPointer> expressions = new LinkedList<>();
-
- @Override
- public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {
- if (doc == null) {
- return;
- }
- try {
- JsonNode json = filterJson(doc);
- if (json == null) {
- return;
- }
-
- ParseData parseData = parse.getOrCreate(url);
- Metadata metadata = parseData.getMetadata();
-
- // extract patterns and store as metadata
- for (LabelledJsonPointer expression : expressions) {
- JsonNode match = json.at(expression.pointer);
- if (match.isMissingNode()) {
- continue;
- }
- metadata.addValue(expression.label, match.asText());
- }
-
- } catch (Exception e) {
- LOG.error("Exception caught when extracting json", e);
- }
- }
-
- public static JsonNode filterJson(DocumentFragment doc) throws Exception {
- XPathExpression expressionJobPosting =
- xpath.compile("//SCRIPT[@type=\"application/ld+json\"]");
- Node scriptNode = (Node) expressionJobPosting.evaluate(doc, XPathConstants.NODE);
- if (scriptNode == null) {
- return null;
- }
- return mapper.readValue(scriptNode.getTextContent(), JsonNode.class);
- }
-
- public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filterParams) {
- java.util.Iterator<Entry<String, JsonNode>> iter = filterParams.fields();
- while (iter.hasNext()) {
- Entry<String, JsonNode> entry = iter.next();
- String key = entry.getKey();
- JsonNode node = entry.getValue();
- LabelledJsonPointer labelP =
- new LabelledJsonPointer(key, JsonPointer.valueOf(node.asText()));
- expressions.add(labelP);
- }
- }
-
- class LabelledJsonPointer {
-
- String label;
- JsonPointer pointer;
-
- public LabelledJsonPointer(String label, JsonPointer pointer) {
- this.label = label;
- this.pointer = pointer;
- }
-
- @Override
- public String toString() {
- return label + " => " + pointer.toString();
- }
- }
-
- @Override
- public boolean needsDOM() {
- return true;
- }
-}
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
deleted file mode 100644
index 7468a55c..00000000
--- a/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.parse.filter;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import java.net.MalformedURLException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.filtering.URLFilters;
-import org.apache.stormcrawler.parse.Outlink;
-import org.apache.stormcrawler.parse.ParseData;
-import org.apache.stormcrawler.parse.ParseResult;
-import org.apache.stormcrawler.util.MetadataTransfer;
-import org.apache.stormcrawler.util.URLUtil;
-import org.jetbrains.annotations.NotNull;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.DocumentFragment;
-
-/**
- * ParseFilter to extract additional links with Xpath can be configured with e.g.
- *
- * <pre>{@code
- * {
- * "class": "org.apache.stormcrawler.parse.filter.LinkParseFilter",
- * "name": "LinkParseFilter",
- * "params": {
- * "pattern": "//IMG/@src",
- * "pattern2": "//VIDEO/SOURCE/@src"
- * }
- * }
- *
- * }</pre>
- */
-public class LinkParseFilter extends XPathFilter {
-
- private static final Logger LOG = LoggerFactory.getLogger(LinkParseFilter.class);
-
- private MetadataTransfer metadataTransfer;
-
- private URLFilters urlFilters;
-
- @Override
- public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {
-
- ParseData parseData = parse.getOrCreate(url);
- Metadata metadata = parseData.getMetadata();
-
- Map<String, Outlink> dedup = new HashMap<>();
-
- for (Outlink o : parse.getOutlinks()) {
- dedup.put(o.getTargetURL(), o);
- }
-
- java.net.URL sourceUrl;
- try {
- sourceUrl = URLUtil.toURL(url);
- } catch (MalformedURLException e1) {
- // we would have known by now as previous components check whether
- // the URL is valid
- LOG.error("Malformed URL on {}", url);
- return;
- }
-
- // applies the XPATH expression in the order in which they are produced
- for (List<LabelledExpression> leList : expressions.values()) {
- for (LabelledExpression le : leList) {
- try {
- List<String> values = le.evaluate(doc);
- if (values == null || values.isEmpty()) {
- continue;
- }
- for (String target : values) {
- // resolve URL
- target = URLUtil.resolveUrl(sourceUrl, target).toExternalForm();
-
- // apply filtering
- target = urlFilters.filter(sourceUrl, metadata, target);
- if (target == null) {
- continue;
- }
-
- // check whether we already have this link
- if (dedup.containsKey(target)) {
- continue;
- }
-
- // create outlink
- Outlink ol = new Outlink(target);
-
- // get the metadata for the outlink from the parent one
- Metadata metadataOutlink =
- metadataTransfer.getMetaForOutlink(target, url, metadata);
-
- ol.setMetadata(metadataOutlink);
- dedup.put(ol.getTargetURL(), ol);
- }
- } catch (Exception e) {
- LOG.error("Error evaluating {}: {}", le.key, e);
- }
- }
- }
-
- parse.setOutlinks(new ArrayList<>(dedup.values()));
- }
-
- @Override
- public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filterParams) {
- super.configure(stormConf, filterParams);
- this.metadataTransfer = MetadataTransfer.getInstance(stormConf);
- this.urlFilters = URLFilters.fromConf(stormConf);
- }
-}
diff --git a/core/src/main/java/org/apache/stormcrawler/util/RobotsTags.java b/core/src/main/java/org/apache/stormcrawler/util/RobotsTags.java
index 31c7238c..76782016 100644
--- a/core/src/main/java/org/apache/stormcrawler/util/RobotsTags.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/RobotsTags.java
@@ -17,16 +17,7 @@
package org.apache.stormcrawler.util;
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpression;
-import javax.xml.xpath.XPathExpressionException;
-import javax.xml.xpath.XPathFactory;
import org.apache.stormcrawler.Metadata;
-import org.w3c.dom.DocumentFragment;
-import org.w3c.dom.NamedNodeMap;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
/**
* Normalises the robots instructions provided by the HTML meta tags or the HTTP X-Robots-Tag
@@ -52,17 +43,6 @@ public class RobotsTags {
private boolean noCache = false;
- private static final XPathExpression expression;
-
- static {
- XPath xpath = XPathFactory.newInstance().newXPath();
- try {
- expression = xpath.compile("/HTML/*/META");
- } catch (XPathExpressionException e) {
- throw new RuntimeException(e);
- }
- }
-
/** Get the values from the fetch metadata. */
public RobotsTags(Metadata metadata, String protocolMDprefix) {
// HTTP headers
@@ -80,46 +60,6 @@ public class RobotsTags {
public RobotsTags() {}
- // set the values based on the meta tags
- // HTML tags
- // <meta name="robots" content="noarchive, nofollow"/>
- // called by the parser bolts
- public void extractMetaTags(DocumentFragment doc) throws XPathExpressionException {
- NodeList nodes = (NodeList) expression.evaluate(doc, XPathConstants.NODESET);
- if (nodes == null) {
- return;
- }
- int numNodes = nodes.getLength();
- for (int i = 0; i < numNodes; i++) {
- Node n = (Node) nodes.item(i);
- // iterate on the attributes
- // and check that it has name=robots and content
- // whatever the case is
- boolean isRobots = false;
- String content = null;
- NamedNodeMap attrs = n.getAttributes();
- for (int att = 0; att < attrs.getLength(); att++) {
- Node keyval = attrs.item(att);
- if ("name".equalsIgnoreCase(keyval.getNodeName())
- && "robots".equalsIgnoreCase(keyval.getNodeValue())) {
- isRobots = true;
- continue;
- }
- if ("content".equalsIgnoreCase(keyval.getNodeName())) {
- content = keyval.getNodeValue();
- continue;
- }
- }
-
- if (isRobots && content != null) {
- // got a value - split it
- String[] vals = content.split(" *, *");
- parseValues(vals);
- return;
- }
- }
- }
-
/** Extracts meta tags based on the value of the content attribute. */
public void extractMetaTags(String content) {
if (content == null) {
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java
index c4a525bc..1ffbfae3 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java
@@ -64,16 +64,4 @@ class XPathFilterTest extends ParsingTester {
Assertions.assertEquals("", scripts[0].trim());
Assertions.assertTrue(scripts[1].contains("_paq"));
}
-
- @Test
- void testLDJsonExtraction() throws IOException {
- prepareParserBolt("test.parsefilters.json");
- parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html");
- Assertions.assertEquals(1, output.getEmitted().size());
- List<Object> parsedTuple = output.getEmitted().get(0);
- Metadata metadata = (Metadata) parsedTuple.get(2);
- Assertions.assertNotNull(metadata);
- String[] scripts = metadata.getValues("streetAddress");
- Assertions.assertNotNull(scripts);
- }
}
diff --git a/core/src/test/resources/test.parsefilters.json b/core/src/test/resources/test.parsefilters.json
index d32c5b52..ceac8bde 100644
--- a/core/src/test/resources/test.parsefilters.json
+++ b/core/src/test/resources/test.parsefilters.json
@@ -11,13 +11,6 @@
"title": "//TITLE"
}
},
- {
- "class": "org.apache.stormcrawler.parse.filter.LDJsonParseFilter",
- "name": "LDJsonParseFilter",
- "params": {
- "streetAddress": "/location/streetAddress"
- }
- },
{
"class": "org.apache.stormcrawler.parse.filter.CommaSeparatedToMultivaluedMetadata",
"name": "CommaSeparatedToMultivaluedMetadata",
diff --git a/docs/src/main/asciidoc/internals.adoc b/docs/src/main/asciidoc/internals.adoc
index e98fd99d..f7628749 100644
--- a/docs/src/main/asciidoc/internals.adoc
+++ b/docs/src/main/asciidoc/internals.adoc
@@ -214,12 +214,12 @@ The archetype includes a default link:https://github.com/apache/stormcrawler/blo
* **CommaSeparatedToMultivaluedMetadata** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java[CommaSeparatedToMultivaluedMetadata] rewrites single metadata values containing comma-separated entries into multiple values for the same key, useful for keyword tags. The number of values produced from a single entry is bounded by the optional `maxTokens` parameter (128 by default), so that a pa [...]
* **DebugParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/DebugParseFilter.java[DebugParseFilter] dumps an XML representation of the DOM structure to a temporary file.
* **DomainParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java[DomainParseFilter] stores the domain or host name in the metadata for later indexing.
-* **LDJsonParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/LDJsonParseFilter.java[LDJsonParseFilter] extracts data from JSON-LD representations.
-* **LinkParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java[LinkParseFilter] extracts outlinks from documents using XPath expressions defined in the configuration.
* **MD5SignatureParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/MD5SignatureParseFilter.java[MD5SignatureParseFilter] generates an MD5 signature of a document based on the binary content, text, or URL (as a last resort). It can be combined with content filtering to exclude boilerplate text.
* **MimeTypeNormalization** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/MimeTypeNormalization.java[MimeTypeNormalization] converts server-reported or inferred mime-type values into human-readable values such as _pdf_, _html_, or _image_ and stores them in the metadata, useful for indexing and filtering search results.
* **XPathFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java[XPathFilter] allows extraction of data using XPath expressions and storing them in the Metadata object.
+The JSoupParserBolt also runs JSoupFilters, which work directly on the JSoup document and do not require a DOM. Extracting data from JSON-LD and outlinks with XPath expressions is done with the JSoupFilters link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/jsoup/LDJsonParseFilter.java[LDJsonParseFilter] and link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java[LinkParseFil [...]
+
You can also implement custom ParseFilters to extend the capabilities of the parsing pipeline. For example, you might create a filter to enrich a document's metadata with additional information, such as language detection, sentiment analysis, named entity recognition, or custom tags extracted from the content. Custom filters can also modify or remove outlinks, normalize text, or integrate external data sources, allowing you to tailor the crawler to your specific processing or indexing re [...]
By implementing the ParseFilter interface and configuring the filter in the JSON file, your custom logic will be seamlessly executed within the parsing bolt.
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
index 6f6d7be6..566ff753 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
@@ -70,6 +70,11 @@ public class JSONResourceWrapper extends ParseFilter {
refresher.getDelegate().filter(url, content, doc, parse);
}
+ @Override
+ public boolean needsDOM() {
+ return refresher.getDelegate().needsDOM();
+ }
+
@Override
public void cleanup() {
if (refresher != null) {
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapperTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapperTest.java
new file mode 100644
index 00000000..0b988576
--- /dev/null
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapperTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.opensearch.parse.filter;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.stormcrawler.JSONResource;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.jetbrains.annotations.NotNull;
+import org.junit.jupiter.api.Test;
+import org.w3c.dom.DocumentFragment;
+
+class JSONResourceWrapperTest {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ /** Minimal ParseFilter + JSONResource implementation. */
+ public static class StubParseFilter extends ParseFilter implements JSONResource {
+
+ @Override
+ public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode params) {}
+
+ @Override
+ public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {}
+
+ @Override
+ public String getResourceFile() {
+ return "stub.json";
+ }
+
+ @Override
+ public void loadJSONResources(InputStream inputStream) {}
+ }
+
+ /** ParseFilter which requires a DOM. */
+ public static class DOMStubParseFilter extends StubParseFilter {
+ @Override
+ public boolean needsDOM() {
+ return true;
+ }
+ }
+
+ private static boolean wrapperNeedsDOM(Class<?> delegateClass) {
+ ObjectNode delegate = MAPPER.createObjectNode();
+ delegate.put("class", delegateClass.getName());
+ delegate.set("params", MAPPER.createObjectNode());
+ ObjectNode params = MAPPER.createObjectNode();
+ params.set("delegate", delegate);
+
+ JSONResourceWrapper wrapper = new JSONResourceWrapper();
+ wrapper.configure(new HashMap<>(), params);
+ try {
+ return wrapper.needsDOM();
+ } finally {
+ wrapper.cleanup();
+ }
+ }
+
+ @Test
+ void needsDOMIsDelegated() {
+ assertTrue(wrapperNeedsDOM(DOMStubParseFilter.class));
+ assertFalse(wrapperNeedsDOM(StubParseFilter.class));
+ }
+}