You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by jn...@apache.org on 2024/03/28 15:31:23 UTC
(incubator-stormcrawler) 01/01: Merge from main
This is an automated email from the ASF dual-hosted git repository.
jnioche pushed a commit to branch 990
in repository https://gitbox.apache.org/repos/asf/incubator-stormcrawler.git
commit 2a5197fb559cb3855ea409e385e0619192c39804
Merge: 82b40dba bdc34cbc
Author: Julien Nioche <ju...@digitalpebble.com>
AuthorDate: Thu Mar 28 15:31:03 2024 +0000
Merge from main
Signed-off-by: Julien Nioche <ju...@digitalpebble.com>
.github/workflows/code_coverage.yml | 29 ++
.github/workflows/maven.yml | 6 +-
DISCLAIMER | 10 +
NOTICE | 4 +-
README.md | 36 +-
THIRD-PARTY.properties | 4 +
THIRD-PARTY.txt | 546 +++++++++++++++++++++
archetype/pom.xml | 6 +-
.../META-INF/maven/archetype-metadata.xml | 71 +--
.../main/resources/archetype-resources/README.md | 2 +-
.../archetype-resources/crawler-conf.yaml | 58 ++-
.../resources/archetype-resources/crawler.flux | 20 +-
.../src/main/resources/archetype-resources/pom.xml | 24 +-
.../src/main/java/CrawlTopology.java | 24 +-
.../src/main/resources/jsoupfilters.json | 6 +-
.../src/main/resources/parsefilters.json | 8 +-
.../src/main/resources/urlfilters.json | 18 +-
core/pom.xml | 52 +-
.../stormcrawler/protocol/Protocol.java | 40 --
.../selenium/DelegatorRemoteDriverProtocol.java | 81 ---
.../protocol/selenium/RemoteDriverProtocol.java | 87 ----
.../apache}/stormcrawler/ConfigurableTopology.java | 6 +-
.../apache}/stormcrawler/Constants.java | 2 +-
.../apache}/stormcrawler/JSONResource.java | 2 +-
.../apache}/stormcrawler/Metadata.java | 20 +-
.../apache}/stormcrawler/bolt/FeedParserBolt.java | 28 +-
.../apache}/stormcrawler/bolt/FetcherBolt.java | 108 ++--
.../apache}/stormcrawler/bolt/JSoupParserBolt.java | 48 +-
.../stormcrawler/bolt/SimpleFetcherBolt.java | 45 +-
.../stormcrawler/bolt/SiteMapParserBolt.java | 28 +-
.../stormcrawler/bolt/StatusEmitterBolt.java | 26 +-
.../apache}/stormcrawler/bolt/URLFilterBolt.java | 12 +-
.../stormcrawler/bolt/URLPartitionerBolt.java | 8 +-
.../apache}/stormcrawler/filtering/URLFilter.java | 6 +-
.../apache}/stormcrawler/filtering/URLFilters.java | 84 +++-
.../filtering/basic/BasicURLFilter.java | 6 +-
.../filtering/basic/BasicURLNormalizer.java | 7 +-
.../filtering/basic/SelfURLFilter.java | 6 +-
.../filtering/depth/MaxDepthFilter.java | 8 +-
.../stormcrawler/filtering/host/HostURLFilter.java | 6 +-
.../filtering/metadata/MetadataFilter.java | 6 +-
.../filtering/regex/FastURLFilter.java | 10 +-
.../stormcrawler/filtering/regex/RegexRule.java | 2 +-
.../filtering/regex/RegexURLFilter.java | 2 +-
.../filtering/regex/RegexURLFilterBase.java | 6 +-
.../filtering/regex/RegexURLNormalizer.java | 6 +-
.../filtering/robots/RobotsFilter.java | 14 +-
.../filtering/sitemap/SitemapFilter.java | 10 +-
.../stormcrawler/indexing/AbstractIndexerBolt.java | 143 ++++--
.../stormcrawler/indexing/DummyIndexer.java | 8 +-
.../stormcrawler/indexing/StdOutIndexer.java | 8 +-
.../stormcrawler/jsoup/LDJsonParseFilter.java | 12 +-
.../stormcrawler/jsoup/LinkParseFilter.java | 20 +-
.../apache}/stormcrawler/jsoup/XPathFilter.java | 12 +-
.../parse/DocumentFragmentBuilder.java | 2 +-
.../apache}/stormcrawler/parse/JSoupFilter.java | 7 +-
.../apache}/stormcrawler/parse/JSoupFilters.java | 10 +-
.../apache}/stormcrawler/parse/Outlink.java | 4 +-
.../apache}/stormcrawler/parse/ParseData.java | 4 +-
.../apache}/stormcrawler/parse/ParseFilter.java | 8 +-
.../apache}/stormcrawler/parse/ParseFilters.java | 8 +-
.../apache}/stormcrawler/parse/ParseResult.java | 4 +-
.../apache}/stormcrawler/parse/TextExtractor.java | 4 +-
.../parse/filter/CollectionTagger.java | 10 +-
.../CommaSeparatedToMultivaluedMetadata.java | 8 +-
.../parse/filter/DebugParseFilter.java | 6 +-
.../parse/filter/DomainParseFilter.java | 12 +-
.../parse/filter/LDJsonParseFilter.java | 10 +-
.../stormcrawler/parse/filter/LinkParseFilter.java | 20 +-
.../parse/filter/MD5SignatureParseFilter.java | 10 +-
.../parse/filter/MimeTypeNormalization.java | 8 +-
.../stormcrawler/parse/filter/XPathFilter.java | 10 +-
.../persistence/AbstractQueryingSpout.java | 9 +-
.../persistence/AbstractStatusUpdaterBolt.java | 10 +-
.../persistence/AdaptiveScheduler.java | 22 +-
.../stormcrawler/persistence/DefaultScheduler.java | 14 +-
.../persistence/EmptyQueueListener.java | 2 +-
.../persistence/MemoryStatusUpdater.java | 6 +-
.../stormcrawler/persistence/Scheduler.java | 8 +-
.../apache}/stormcrawler/persistence/Status.java | 2 +-
.../persistence/StdOutStatusUpdater.java | 4 +-
.../persistence/urlbuffer/AbstractURLBuffer.java | 8 +-
.../persistence/urlbuffer/PriorityURLBuffer.java | 4 +-
.../persistence/urlbuffer/SchedulingURLBuffer.java | 4 +-
.../persistence/urlbuffer/SimpleURLBuffer.java | 2 +-
.../persistence/urlbuffer/URLBuffer.java | 12 +-
.../protocol/AbstractHttpProtocol.java | 118 +----
.../stormcrawler/protocol/DelegatorProtocol.java | 159 ++++--
.../apache}/stormcrawler/protocol/HttpHeaders.java | 2 +-
.../protocol/HttpRobotRulesParser.java | 87 +++-
.../org/apache/stormcrawler/protocol/Protocol.java | 156 ++++++
.../stormcrawler/protocol/ProtocolFactory.java | 17 +-
.../stormcrawler/protocol/ProtocolResponse.java | 10 +-
.../apache}/stormcrawler/protocol/RobotRules.java | 2 +-
.../stormcrawler/protocol/RobotRulesParser.java | 84 +++-
.../stormcrawler/protocol/file/FileProtocol.java | 16 +-
.../stormcrawler/protocol/file/FileResponse.java | 8 +-
.../protocol/httpclient/HttpProtocol.java | 25 +-
.../protocol/okhttp/DNSResolutionListener.java | 2 +-
.../stormcrawler/protocol/okhttp/HttpProtocol.java | 24 +-
.../protocol/selenium/NavigationFilter.java | 8 +-
.../protocol/selenium/NavigationFilters.java | 14 +-
.../protocol/selenium/RemoteDriverProtocol.java | 131 +++++
.../protocol/selenium/SeleniumProtocol.java | 26 +-
.../stormcrawler/proxy/MultiProxyManager.java | 8 +-
.../apache}/stormcrawler/proxy/ProxyManager.java | 4 +-
.../apache}/stormcrawler/proxy/SCProxy.java | 14 +-
.../stormcrawler/proxy/SingleProxyManager.java | 13 +-
.../apache}/stormcrawler/spout/FileSpout.java | 32 +-
.../apache}/stormcrawler/spout/MemorySpout.java | 10 +-
.../stormcrawler/util/AbstractConfigurable.java | 2 +-
.../stormcrawler/util/CharsetIdentification.java | 6 +-
.../stormcrawler/util/CollectionMetric.java | 2 +-
.../apache}/stormcrawler/util/ConfUtils.java | 83 +++-
.../apache}/stormcrawler/util/Configurable.java | 2 +-
.../stormcrawler/util/ConfigurableHelper.java | 2 +-
.../apache}/stormcrawler/util/CookieConverter.java | 2 +-
.../stormcrawler/util/InitialisationUtil.java | 2 +-
.../stormcrawler/util/MetadataTransfer.java | 30 +-
.../stormcrawler/util/PerSecondReducer.java | 2 +-
.../apache}/stormcrawler/util/RefreshTag.java | 2 +-
.../apache}/stormcrawler/util/RobotsTags.java | 4 +-
.../apache}/stormcrawler/util/StringTabScheme.java | 4 +-
.../apache}/stormcrawler/util/URLPartitioner.java | 28 +-
.../stormcrawler/util/URLStreamGrouping.java | 8 +-
.../apache}/stormcrawler/util/URLUtil.java | 2 +-
core/src/main/resources/crawler-default.yaml | 97 +++-
.../apache/stormcrawler/MetadataTest.java} | 26 +-
.../stormcrawler/TestMetadataSerialization.java | 2 +-
.../apache}/stormcrawler/TestOutputCollector.java | 2 +-
.../apache}/stormcrawler/TestUtil.java | 2 +-
.../stormcrawler/bolt/AbstractFetcherBoltTest.java | 12 +-
.../stormcrawler/bolt/FeedParserBoltTest.java | 16 +-
.../apache}/stormcrawler/bolt/FetcherBoltTest.java | 2 +-
.../stormcrawler/bolt/JSoupParserBoltTest.java | 16 +-
.../stormcrawler/bolt/SimpleFetcherBoltTest.java | 2 +-
.../stormcrawler/bolt/SiteMapParserBoltTest.java | 95 ++--
.../stormcrawler/filtering/BasicURLFilterTest.java | 6 +-
.../filtering/BasicURLNormalizerTest.java | 6 +-
.../stormcrawler/filtering/FastURLFilterTest.java | 6 +-
.../stormcrawler/filtering/HostURLFilterTest.java | 6 +-
.../stormcrawler/filtering/MaxDepthFilterTest.java | 8 +-
.../stormcrawler/filtering/MetadataFilterTest.java | 6 +-
.../stormcrawler/filtering/RegexFilterTest.java | 6 +-
.../ClassInheritingFomAbstractAndInterface.java | 6 +-
.../ClassInheritingFromAbstractClassOnly.java | 4 +-
.../ClassInheritingFromOpenClass.java | 4 +-
.../ClassWithoutValidConstructor.java | 4 +-
.../initialisation/FinalClassToInitialize.java | 2 +-
.../helper/initialisation/SimpleOpenClass.java | 2 +-
.../helper/initialisation/base/AbstractClass.java | 2 +-
.../helper/initialisation/base/ITestInterface.java | 2 +-
.../OpenClassWithAbstractClassAndInterface.java | 2 +-
.../stormcrawler/indexer/BasicIndexingTest.java | 27 +-
.../apache}/stormcrawler/indexer/DummyIndexer.java | 6 +-
.../stormcrawler/indexer/IndexerTester.java | 10 +-
.../apache}/stormcrawler/json/JsoupFilterTest.java | 10 +-
.../stormcrawler/jsoup/JSoupFiltersTest.java | 10 +-
.../stormcrawler/parse/DuplicateLinksTest.java | 10 +-
.../apache}/stormcrawler/parse/ParsingTester.java | 8 +-
.../stormcrawler/parse/StackOverflowTest.java | 8 +-
.../stormcrawler/parse/TextExtractorTest.java | 2 +-
.../parse/filter/CSVMetadataFilterTest.java | 8 +-
.../parse/filter/CollectionTaggerTest.java | 4 +-
.../parse/filter/SubDocumentsFilterTest.java | 8 +-
.../parse/filter/SubDocumentsParseFilter.java | 8 +-
.../stormcrawler/parse/filter/XPathFilterTest.java | 8 +-
.../persistence/AdaptiveSchedulerTest.java | 19 +-
.../persistence/DefaultSchedulerTest.java | 4 +-
.../stormcrawler/persistence/URLBufferTest.java | 10 +-
.../protocol/AbstractProtocolTest.java | 96 ++++
.../protocol/DelegationProtocolTest.java | 41 +-
.../stormcrawler/protocol/DummyProtocol.java} | 28 +-
.../stormcrawler/protocol/HttpHeadersTest.java | 2 +-
.../protocol/HttpRobotRulesParserTest.java | 282 +++++++++++
.../protocol/selenium/ProtocolTest.java | 166 +++++++
.../stormcrawler/proxy/MultiProxyManagerTest.java | 2 +-
.../apache}/stormcrawler/proxy/SCProxyTest.java | 2 +-
.../stormcrawler/proxy/SingleProxyManagerTest.java | 2 +-
.../apache/stormcrawler/util/ConfUtilsTest.java | 64 +++
.../stormcrawler/util/CookieConverterTest.java | 2 +-
.../stormcrawler/util/InitialisationUtilTest.java | 6 +-
.../stormcrawler/util/MetadataTransferTest.java | 61 ++-
.../apache}/stormcrawler/util/RefreshTagTest.java | 2 +-
.../apache}/stormcrawler/util/RobotsTagsTest.java | 4 +-
core/src/test/resources/basicurlnormalizer.json | 4 +-
core/src/test/resources/delegator-conf.yaml | 21 +-
core/src/test/resources/test.jsoupfilters.json | 8 +-
core/src/test/resources/test.parsefilters.json | 8 +-
core/src/test/resources/test.subdocfilter.json | 6 +-
.../test/resources/tripadvisor.sitemap.index.xml | 22 +
core/src/test/resources/tripadvisor.sitemap.xml.gz | Bin 0 -> 1537978 bytes
external/aws/README.md | 2 +-
external/aws/pom.xml | 8 +-
.../aws/bolt/CloudSearchConstants.java | 2 +-
.../aws/bolt/CloudSearchIndexerBolt.java | 12 +-
.../stormcrawler/aws/bolt/CloudSearchUtils.java | 2 +-
.../stormcrawler/aws/s3/AbstractS3CacheBolt.java | 4 +-
.../stormcrawler/aws/s3/S3CacheChecker.java | 6 +-
.../apache}/stormcrawler/aws/s3/S3Cacher.java | 6 +-
.../stormcrawler/aws/s3/S3ContentCacher.java | 4 +-
external/elasticsearch/README.md | 20 +-
external/elasticsearch/archetype/pom.xml | 4 +-
.../META-INF/maven/archetype-metadata.xml | 30 +-
.../archetype-resources/crawler-conf.yaml | 58 ++-
.../resources/archetype-resources/es-conf.yaml | 2 +-
.../resources/archetype-resources/es-crawler.flux | 22 +-
.../archetype-resources/es-injection.flux | 8 +-
.../archetype-resources/kibana/importKibana.sh | 8 +-
.../src/main/resources/archetype-resources/pom.xml | 24 +-
.../src/main/java/ESCrawlTopology.java | 36 +-
.../src/main/resources/jsoupfilters.json | 6 +-
.../src/main/resources/parsefilters.json | 8 +-
.../src/main/resources/urlfilters.json | 18 +-
external/elasticsearch/pom.xml | 9 +-
.../BulkItemResponseToFailedFlag.java | 2 +-
.../elasticsearch/ElasticSearchConnection.java | 4 +-
.../elasticsearch/bolt/DeletionBolt.java | 23 +-
.../elasticsearch/bolt/IndexerBolt.java | 20 +-
.../filtering/JSONURLFilterWrapper.java | 14 +-
.../elasticsearch/metrics/MetricsConsumer.java | 8 +-
.../elasticsearch/metrics/StatusMetricsBolt.java | 6 +-
.../parse/filter/JSONResourceWrapper.java | 14 +-
.../elasticsearch/persistence/AbstractSpout.java | 10 +-
.../persistence/AggregationSpout.java | 8 +-
.../elasticsearch/persistence/CollapsingSpout.java | 4 +-
.../elasticsearch/persistence/HybridSpout.java | 6 +-
.../elasticsearch/persistence/ScrollSpout.java | 10 +-
.../persistence/StatusUpdaterBolt.java | 50 +-
.../elasticsearch/bolt/IndexerBoltTest.java | 12 +-
.../elasticsearch/bolt/StatusBoltTest.java | 14 +-
external/langid/pom.xml | 6 +-
.../stormcrawler/parse/filter/LanguageID.java | 12 +-
external/opensearch/OS_IndexInit.sh | 23 -
external/opensearch/README.md | 17 +-
external/opensearch/archetype/pom.xml | 4 +-
.../META-INF/archetype-post-generate.groovy | 5 +-
.../META-INF/maven/archetype-metadata.xml | 32 +-
.../resources/archetype-resources/OS_IndexInit.sh | 25 +
.../main/resources/archetype-resources/README.md | 11 +-
.../archetype-resources/crawler-conf.yaml | 58 ++-
.../resources/archetype-resources/crawler.flux | 22 +-
.../dashboards/importDashboards.sh | 8 +-
.../resources/archetype-resources/injection.flux | 8 +-
.../archetype-resources/opensearch-conf.yaml | 12 +-
.../src/main/resources/archetype-resources/pom.xml | 24 +-
.../src/main/resources/indexer.mapping | 0
.../src/main/resources/jsoupfilters.json | 6 +-
.../src/main/resources/metrics.mapping | 0
.../src/main/resources/parsefilters.json | 8 +-
.../src/main/resources/queues.mapping | 0
.../src/main/resources/status.mapping | 0
.../src/main/resources/urlfilters.json | 18 +-
external/opensearch/opensearch-conf.yaml | 12 +-
external/opensearch/pom.xml | 24 +-
.../stormcrawler/opensearch/bolt/DeletionBolt.java | 94 ----
.../opensearch/BulkItemResponseToFailedFlag.java | 6 +-
.../apache}/stormcrawler/opensearch/Constants.java | 2 +-
.../stormcrawler/opensearch/IndexCreation.java | 2 +-
.../opensearch/OpenSearchConnection.java | 90 ++--
.../stormcrawler/opensearch/bolt/DeletionBolt.java | 308 ++++++++++++
.../stormcrawler/opensearch/bolt/IndexerBolt.java | 36 +-
.../opensearch/filtering/JSONURLFilterWrapper.java | 14 +-
.../opensearch/metrics/MetricsConsumer.java | 10 +-
.../opensearch/metrics/StatusMetricsBolt.java | 10 +-
.../parse/filter/JSONResourceWrapper.java | 14 +-
.../opensearch/persistence/AbstractSpout.java | 16 +-
.../opensearch/persistence/AggregationSpout.java | 12 +-
.../opensearch/persistence/HybridSpout.java | 10 +-
.../opensearch/persistence/QueueBolt.java | 12 +-
.../opensearch/persistence/StatusUpdaterBolt.java | 58 ++-
.../opensearch/bolt/AbstractOpenSearchTest.java | 46 ++
.../opensearch/bolt/IndexerBoltTest.java | 30 +-
.../opensearch/bolt/StatusBoltTest.java | 38 +-
.../src/{main => test}/resources/indexer.mapping | 0
.../src/{main => test}/resources/metrics.mapping | 0
.../src/{main => test}/resources/status.mapping | 0
external/pom.xml | 23 +-
external/solr/README.md | 2 +-
external/solr/pom.xml | 14 +-
external/solr/solr-conf.yaml | 2 +-
.../apache}/stormcrawler/solr/SeedInjector.java | 10 +-
.../apache}/stormcrawler/solr/SolrConnection.java | 4 +-
.../stormcrawler/solr/SolrCrawlTopology.java | 26 +-
.../stormcrawler/solr/bolt/DeletionBolt.java | 86 ++++
.../stormcrawler/solr/bolt/IndexerBolt.java | 13 +-
.../stormcrawler/solr/metrics/MetricsConsumer.java | 6 +-
.../stormcrawler/solr/persistence/SolrSpout.java | 11 +-
.../solr/persistence/StatusUpdaterBolt.java | 17 +-
external/sql/pom.xml | 6 +-
external/sql/sql-conf.yaml | 2 +-
.../apache}/stormcrawler/sql/Constants.java | 2 +-
.../apache}/stormcrawler/sql/IndexerBolt.java | 12 +-
.../apache}/stormcrawler/sql/SQLSpout.java | 10 +-
.../apache}/stormcrawler/sql/SQLUtil.java | 2 +-
.../stormcrawler/sql/StatusUpdaterBolt.java | 33 +-
.../stormcrawler/sql/metrics/MetricsConsumer.java | 18 +-
external/tika/README.md | 4 +-
external/tika/pom.xml | 12 +-
.../apache}/stormcrawler/tika/DOMBuilder.java | 2 +-
.../apache}/stormcrawler/tika/ParserBolt.java | 38 +-
.../apache}/stormcrawler/tika/RedirectionBolt.java | 4 +-
.../stormcrawler/tika/XMLCharacterRecognizer.java | 2 +-
.../apache}/stormcrawler/tika/ParserBoltTest.java | 16 +-
external/urlfrontier/README.md | 2 +-
external/urlfrontier/pom.xml | 9 +-
.../stormcrawler/urlfrontier/Constants.java | 2 +-
.../urlfrontier/ManagedChannelUtil.java | 4 +-
.../apache}/stormcrawler/urlfrontier/Spout.java | 10 +-
.../urlfrontier/StatusUpdaterBolt.java | 14 +-
.../urlfrontier/StatusUpdaterBoltTest.java | 16 +-
.../urlfrontier/URLFrontierContainer.java | 2 +-
.../urlfrontier/URLFrontierContainerConfig.java | 2 +-
external/warc/README.md | 43 +-
external/warc/pom.xml | 20 +-
.../warc/FileTimeSizeRotationPolicy.java | 2 +-
.../apache}/stormcrawler/warc/GzipHdfsBolt.java | 2 +-
.../stormcrawler/warc/WARCFileNameFormat.java | 2 +-
.../apache}/stormcrawler/warc/WARCHdfsBolt.java | 6 +-
.../stormcrawler/warc/WARCRecordFormat.java | 20 +-
.../stormcrawler/warc/WARCRequestRecordFormat.java | 8 +-
.../apache}/stormcrawler/warc/WARCSpout.java | 65 ++-
.../stormcrawler/warc/WARCHdfsBoltTest.java | 10 +-
.../stormcrawler/warc/WARCRecordFormatTest.java | 8 +-
.../apache/stormcrawler/warc/WARCSpoutTest.java | 70 +++
external/warc/src/test/resources/test.warc.gz | Bin 0 -> 301243 bytes
.../src/test/resources/unparsable-date.warc.gz | Bin 0 -> 938 bytes
external/warc/src/test/resources/warc.inputs | 2 +
pom.xml | 264 ++++++++--
329 files changed, 4971 insertions(+), 2219 deletions(-)
diff --cc THIRD-PARTY.txt
index 00000000,8f2ea7b5..98cd4b9c
mode 000000,100644..100644
--- a/THIRD-PARTY.txt
+++ b/THIRD-PARTY.txt
@@@ -1,0 -1,547 +1,546 @@@
+
+ List of third-party dependencies grouped by their license type.
+
+
+ 3-Clause BSD License
+
+ * Kryo (com.esotericsoftware:kryo:5.6.0 - https://github.com/EsotericSoftware/kryo/kryo)
+ * MinLog (com.esotericsoftware:minlog:1.3.1 - https://github.com/EsotericSoftware/minlog)
+ * ReflectASM (com.esotericsoftware:reflectasm:1.11.9 - https://github.com/EsotericSoftware/reflectasm)
+
+ Apache License
+
+ * Log4j Implemented Over SLF4J (org.slf4j:log4j-over-slf4j:1.7.36 - http://www.slf4j.org)
+
+ Apache License, Version 2.0
+
+ * aggs-matrix-stats (org.opensearch.plugin:aggs-matrix-stats-client:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * Apache Avro (org.apache.avro:avro:1.11.3 - https://avro.apache.org)
+ * Apache Commons BeanUtils (commons-beanutils:commons-beanutils:1.9.4 - https://commons.apache.org/proper/commons-beanutils/)
+ * Apache Commons CLI (commons-cli:commons-cli:1.6.0 - https://commons.apache.org/proper/commons-cli/)
+ * Apache Commons Codec (commons-codec:commons-codec:1.11 - http://commons.apache.org/proper/commons-codec/)
+ * Apache Commons Codec (commons-codec:commons-codec:1.15 - https://commons.apache.org/proper/commons-codec/)
+ * Apache Commons Codec (commons-codec:commons-codec:1.16.0 - https://commons.apache.org/proper/commons-codec/)
+ * Apache Commons Collections (commons-collections:commons-collections:3.2.2 - http://commons.apache.org/collections/)
+ * Apache Commons Collections (org.apache.commons:commons-collections4:4.4 - https://commons.apache.org/proper/commons-collections/)
+ * Apache Commons Compress (org.apache.commons:commons-compress:1.21 - https://commons.apache.org/proper/commons-compress/)
+ * Apache Commons Compress (org.apache.commons:commons-compress:1.24.0 - https://commons.apache.org/proper/commons-compress/)
+ * Apache Commons Configuration (org.apache.commons:commons-configuration2:2.8.0 - https://commons.apache.org/proper/commons-configuration/)
+ * Apache Commons Crypto (org.apache.commons:commons-crypto:1.1.0 - https://commons.apache.org/proper/commons-crypto/)
+ * Apache Commons CSV (org.apache.commons:commons-csv:1.10.0 - https://commons.apache.org/proper/commons-csv/)
+ * Apache Commons Exec (org.apache.commons:commons-exec:1.3 - http://commons.apache.org/proper/commons-exec/)
+ * Apache Commons IO (commons-io:commons-io:2.11.0 - https://commons.apache.org/proper/commons-io/)
+ * Apache Commons Lang (org.apache.commons:commons-lang3:3.12.0 - https://commons.apache.org/proper/commons-lang/)
+ * Apache Commons Lang (org.apache.commons:commons-lang3:3.13.0 - https://commons.apache.org/proper/commons-lang/)
+ * Apache Commons Logging (commons-logging:commons-logging:1.2 - http://commons.apache.org/proper/commons-logging/)
+ * Apache Commons Math (org.apache.commons:commons-math3:3.6.1 - http://commons.apache.org/proper/commons-math/)
+ * Apache Commons Net (commons-net:commons-net:3.9.0 - https://commons.apache.org/proper/commons-net/)
+ * Apache Commons Text (org.apache.commons:commons-text:1.10.0 - https://commons.apache.org/proper/commons-text)
+ * Apache FontBox (org.apache.pdfbox:fontbox:2.0.29 - http://pdfbox.apache.org/)
+ * Apache Hadoop Annotations (org.apache.hadoop:hadoop-annotations:3.3.6 - no url defined)
+ * Apache Hadoop Auth (org.apache.hadoop:hadoop-auth:3.3.6 - no url defined)
+ * Apache Hadoop Client Aggregator (org.apache.hadoop:hadoop-client:3.3.6 - no url defined)
+ * Apache Hadoop Common (org.apache.hadoop:hadoop-common:3.3.6 - no url defined)
+ * Apache Hadoop HDFS (org.apache.hadoop:hadoop-hdfs:3.3.6 - no url defined)
+ * Apache Hadoop HDFS Client (org.apache.hadoop:hadoop-hdfs-client:3.3.6 - no url defined)
+ * Apache Hadoop MapReduce Common (org.apache.hadoop:hadoop-mapreduce-client-common:3.3.6 - no url defined)
+ * Apache Hadoop MapReduce Core (org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6 - no url defined)
+ * Apache Hadoop MapReduce JobClient (org.apache.hadoop:hadoop-mapreduce-client-jobclient:3.3.6 - no url defined)
+ * Apache Hadoop shaded Guava (org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1 - https://www.apache.org/hadoop-thirdparty/hadoop-shaded-guava/)
+ * Apache Hadoop shaded Protobuf 3.7 (org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.1.1 - https://www.apache.org/hadoop-thirdparty/hadoop-shaded-protobuf_3_7/)
+ * Apache Hadoop YARN API (org.apache.hadoop:hadoop-yarn-api:3.3.6 - no url defined)
+ * Apache Hadoop YARN Client (org.apache.hadoop:hadoop-yarn-client:3.3.6 - no url defined)
+ * Apache Hadoop YARN Common (org.apache.hadoop:hadoop-yarn-common:3.3.6 - no url defined)
+ * Apache HBase - Annotations (org.apache.hbase:hbase-annotations:2.5.6-hadoop3 - https://hbase.apache.org/hbase-annotations)
+ * Apache HBase - Client (org.apache.hbase:hbase-client:2.5.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-client)
+ * Apache HBase - Common (org.apache.hbase:hbase-common:2.5.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-common)
+ * Apache HBase - Hadoop Compatibility (org.apache.hbase:hbase-hadoop-compat:2.5.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-hadoop-compat)
+ * Apache HBase - Hadoop Two Compatibility (org.apache.hbase:hbase-hadoop2-compat:2.5.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-hadoop2-compat)
+ * Apache HBase - Logging (org.apache.hbase:hbase-logging:2.5.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-logging)
+ * Apache HBase - Metrics API (org.apache.hbase:hbase-metrics-api:2.5.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-metrics-api)
+ * Apache HBase - Metrics Implementation (org.apache.hbase:hbase-metrics:2.5.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-metrics)
+ * Apache HBase Patched and Relocated (Shaded) Protobuf (org.apache.hbase.thirdparty:hbase-shaded-protobuf:4.1.5 - https://hbase.apache.org/hbase-shaded-protobuf)
+ * Apache HBase - Protocol (org.apache.hbase:hbase-protocol:2.5.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-protocol)
+ * Apache HBase Relocated (Shaded) GSON Libs (org.apache.hbase.thirdparty:hbase-shaded-gson:4.1.5 - https://hbase.apache.org/hbase-shaded-gson)
+ * Apache HBase Relocated (Shaded) Netty Libs (org.apache.hbase.thirdparty:hbase-shaded-netty:4.1.5 - https://hbase.apache.org/hbase-shaded-netty)
+ * Apache HBase Relocated (Shaded) Third-party Miscellaneous Libs (org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:4.1.5 - https://hbase.apache.org/hbase-shaded-miscellaneous)
+ * Apache HBase - Shaded Protocol (org.apache.hbase:hbase-protocol-shaded:2.5.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-protocol-shaded)
+ * Apache HBase Unsafe Wrapper (org.apache.hbase.thirdparty:hbase-unsafe:4.1.5 - https://hbase.apache.org/hbase-unsafe)
+ * Apache HttpAsyncClient (org.apache.httpcomponents:httpasyncclient:4.1.4 - http://hc.apache.org/httpcomponents-asyncclient)
+ * Apache HttpAsyncClient (org.apache.httpcomponents:httpasyncclient:4.1.5 - http://hc.apache.org/httpcomponents-asyncclient)
+ * Apache HttpClient (org.apache.httpcomponents:httpclient:4.5.10 - http://hc.apache.org/httpcomponents-client)
+ * Apache HttpClient (org.apache.httpcomponents:httpclient:4.5.14 - http://hc.apache.org/httpcomponents-client-ga)
+ * Apache HttpClient Mime (org.apache.httpcomponents:httpmime:4.5.14 - http://hc.apache.org/httpcomponents-client-ga)
+ * Apache HttpCore (org.apache.httpcomponents:httpcore:4.4.12 - http://hc.apache.org/httpcomponents-core-ga)
+ * Apache HttpCore (org.apache.httpcomponents:httpcore:4.4.16 - http://hc.apache.org/httpcomponents-core-ga)
+ * Apache HttpCore NIO (org.apache.httpcomponents:httpcore-nio:4.4.12 - http://hc.apache.org/httpcomponents-core-ga)
+ * Apache HttpCore NIO (org.apache.httpcomponents:httpcore-nio:4.4.16 - http://hc.apache.org/httpcomponents-core-ga)
+ * Apache James :: Mime4j :: Core (org.apache.james:apache-mime4j-core:0.8.9 - http://james.apache.org/mime4j/apache-mime4j-core)
+ * Apache James :: Mime4j :: DOM (org.apache.james:apache-mime4j-dom:0.8.9 - http://james.apache.org/mime4j/apache-mime4j-dom)
+ * Apache JempBox (org.apache.pdfbox:jempbox:1.8.17 - http://www.apache.org/pdfbox-parent/jempbox/)
+ * Apache Log4j API (org.apache.logging.log4j:log4j-api:2.23.0 - https://logging.apache.org/log4j/2.x/log4j/log4j-api/)
+ * Apache Log4j Core (org.apache.logging.log4j:log4j-core:2.23.0 - https://logging.apache.org/log4j/2.x/log4j/log4j-core/)
+ * Apache Log4j JUL Adapter (org.apache.logging.log4j:log4j-jul:2.21.0 - https://logging.apache.org/log4j/2.x/log4j/log4j-jul/)
+ * Apache Log4j SLF4J Binding (org.apache.logging.log4j:log4j-slf4j-impl:2.23.0 - https://logging.apache.org/log4j/2.x/log4j/log4j-slf4j-impl/)
+ * Apache Lucene (module: backward-codecs) (org.apache.lucene:lucene-backward-codecs:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: common) (org.apache.lucene:lucene-analysis-common:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: core) (org.apache.lucene:lucene-core:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: grouping) (org.apache.lucene:lucene-grouping:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: highlighter) (org.apache.lucene:lucene-highlighter:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: join) (org.apache.lucene:lucene-join:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: memory) (org.apache.lucene:lucene-memory:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: misc) (org.apache.lucene:lucene-misc:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: queries) (org.apache.lucene:lucene-queries:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: queryparser) (org.apache.lucene:lucene-queryparser:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: sandbox) (org.apache.lucene:lucene-sandbox:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: spatial3d) (org.apache.lucene:lucene-spatial3d:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: spatial-extras) (org.apache.lucene:lucene-spatial-extras:9.9.2 - https://lucene.apache.org/)
+ * Apache Lucene (module: suggest) (org.apache.lucene:lucene-suggest:9.9.2 - https://lucene.apache.org/)
+ * Apache PDFBox (org.apache.pdfbox:pdfbox:2.0.29 - https://www.apache.org/pdfbox-parent/pdfbox/)
+ * Apache PDFBox tools (org.apache.pdfbox:pdfbox-tools:2.0.29 - https://www.apache.org/pdfbox-parent/pdfbox-tools/)
+ * Apache POI (org.apache.poi:poi-ooxml-lite:5.2.3 - https://poi.apache.org/)
+ * Apache POI (org.apache.poi:poi-scratchpad:5.2.3 - https://poi.apache.org/)
+ * Apache POI - API based on OPC and OOXML schemas (org.apache.poi:poi-ooxml:5.2.3 - https://poi.apache.org/)
+ * Apache POI - Common (org.apache.poi:poi:5.2.3 - https://poi.apache.org/)
+ * Apache Solr (module: api) (org.apache.solr:solr-api:9.5.0 - https://solr.apache.org/)
+ * Apache Solr (module: solrj) (org.apache.solr:solr-solrj:9.5.0 - https://solr.apache.org/)
+ * Apache Solr (module: solrj-streaming) (org.apache.solr:solr-solrj-streaming:9.5.0 - https://solr.apache.org/)
+ * Apache Solr (module: solrj-zookeeper) (org.apache.solr:solr-solrj-zookeeper:9.5.0 - https://solr.apache.org/)
+ * Apache Tika Apple parser module (org.apache.tika:tika-parser-apple-module:2.9.1 - https://tika.apache.org/tika-parser-apple-module/)
+ * Apache Tika audiovideo parser module (org.apache.tika:tika-parser-audiovideo-module:2.9.1 - https://tika.apache.org/tika-parser-audiovideo-module/)
+ * Apache Tika cad parser module (org.apache.tika:tika-parser-cad-module:2.9.1 - https://tika.apache.org/tika-parser-cad-module/)
+ * Apache Tika code parser module (org.apache.tika:tika-parser-code-module:2.9.1 - https://tika.apache.org/tika-parser-code-module/)
+ * Apache Tika core (org.apache.tika:tika-core:2.9.1 - https://tika.apache.org/)
+ * Apache Tika crypto parser module (org.apache.tika:tika-parser-crypto-module:2.9.1 - https://tika.apache.org/tika-parser-crypto-module/)
+ * Apache Tika digest commons (org.apache.tika:tika-parser-digest-commons:2.9.1 - https://tika.apache.org/tika-parser-digest-commons/)
+ * Apache Tika font parser module (org.apache.tika:tika-parser-font-module:2.9.1 - https://tika.apache.org/tika-parser-font-module/)
+ * Apache Tika html parser module (org.apache.tika:tika-parser-html-module:2.9.1 - https://tika.apache.org/tika-parser-html-module/)
+ * Apache Tika image parser module (org.apache.tika:tika-parser-image-module:2.9.1 - https://tika.apache.org/tika-parser-image-module/)
+ * Apache Tika mail commons (org.apache.tika:tika-parser-mail-commons:2.9.1 - https://tika.apache.org/tika-parser-mail-commons/)
+ * Apache Tika mail parser module (org.apache.tika:tika-parser-mail-module:2.9.1 - https://tika.apache.org/tika-parser-mail-module/)
+ * Apache Tika Microsoft parser module (org.apache.tika:tika-parser-microsoft-module:2.9.1 - https://tika.apache.org/tika-parser-microsoft-module/)
+ * Apache Tika miscellaneous office format parser module (org.apache.tika:tika-parser-miscoffice-module:2.9.1 - https://tika.apache.org/tika-parser-miscoffice-module/)
+ * Apache Tika news parser module (org.apache.tika:tika-parser-news-module:2.9.1 - https://tika.apache.org/tika-parser-news-module/)
+ * Apache Tika OCR parser module (org.apache.tika:tika-parser-ocr-module:2.9.1 - https://tika.apache.org/tika-parser-ocr-module/)
+ * Apache Tika package parser module (org.apache.tika:tika-parser-pkg-module:2.9.1 - https://tika.apache.org/tika-parser-pkg-module/)
+ * Apache Tika PDF parser module (org.apache.tika:tika-parser-pdf-module:2.9.1 - https://tika.apache.org/tika-parser-pdf-module/)
+ * Apache Tika plugin for Ogg, Vorbis and FLAC (org.gagravarr:vorbis-java-tika:0.8 - https://github.com/Gagravarr/VorbisJava)
+ * Apache Tika standard parser package (org.apache.tika:tika-parsers-standard-package:2.9.1 - https://tika.apache.org/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/)
+ * Apache Tika text parser module (org.apache.tika:tika-parser-text-module:2.9.1 - https://tika.apache.org/tika-parser-text-module/)
+ * Apache Tika WARC parser module (org.apache.tika:tika-parser-webarchive-module:2.9.1 - https://tika.apache.org/tika-parser-webarchive-module/)
+ * Apache Tika XML parser module (org.apache.tika:tika-parser-xml-module:2.9.1 - https://tika.apache.org/tika-parser-xml-module/)
+ * Apache Tika XMP commons (org.apache.tika:tika-parser-xmp-commons:2.9.1 - https://tika.apache.org/tika-parser-xmp-commons/)
+ * Apache Tika ZIP commons (org.apache.tika:tika-parser-zip-commons:2.9.1 - https://tika.apache.org/tika-parser-zip-commons/)
+ * Apache XmpBox (org.apache.pdfbox:xmpbox:2.0.29 - https://www.apache.org/pdfbox-parent/xmpbox/)
+ * Apache Yetus - Audience Annotations (org.apache.yetus:audience-annotations:0.5.0 - https://yetus.apache.org/audience-annotations)
+ * Apache ZooKeeper - Jute (org.apache.zookeeper:zookeeper-jute:3.6.3 - http://zookeeper.apache.org/zookeeper-jute)
+ * Apache ZooKeeper - Jute (org.apache.zookeeper:zookeeper-jute:3.9.1 - http://zookeeper.apache.org/zookeeper-jute)
+ * Apache ZooKeeper - Server (org.apache.zookeeper:zookeeper:3.6.3 - http://zookeeper.apache.org/zookeeper)
+ * Apache ZooKeeper - Server (org.apache.zookeeper:zookeeper:3.9.1 - http://zookeeper.apache.org/zookeeper)
+ * AutoService (com.google.auto.service:auto-service-annotations:1.1.1 - https://github.com/google/auto/tree/main/service)
+ * AWS Java SDK for Amazon CloudSearch (com.amazonaws:aws-java-sdk-cloudsearch:1.12.663 - https://aws.amazon.com/sdkforjava)
+ * AWS Java SDK for Amazon S3 (com.amazonaws:aws-java-sdk-s3:1.12.663 - https://aws.amazon.com/sdkforjava)
+ * AWS Java SDK for AWS KMS (com.amazonaws:aws-java-sdk-kms:1.12.663 - https://aws.amazon.com/sdkforjava)
+ * AWS SDK for Java - Core (com.amazonaws:aws-java-sdk-core:1.12.663 - https://aws.amazon.com/sdkforjava)
+ * Byte Buddy (without dependencies) (net.bytebuddy:byte-buddy:1.14.11 - https://bytebuddy.net/byte-buddy)
+ * Caffeine cache (com.github.ben-manes.caffeine:caffeine:3.1.8 - https://github.com/ben-manes/caffeine)
+ * com.drewnoakes:metadata-extractor (com.drewnoakes:metadata-extractor:2.18.0 - https://drewnoakes.com/code/exif/)
+ * Commons Daemon (commons-daemon:commons-daemon:1.0.13 - http://commons.apache.org/daemon/)
+ * Commons Lang (commons-lang:commons-lang:2.6 - http://commons.apache.org/lang/)
+ * Commons Logging (commons-logging:commons-logging:1.1.3 - http://commons.apache.org/proper/commons-logging/)
+ * Commons Math (org.apache.commons:commons-math3:3.1.1 - http://commons.apache.org/math/)
+ * compiler (com.github.spullara.mustache.java:compiler:0.9.10 - http://github.com/spullara/mustache.java)
+ * compiler (com.github.spullara.mustache.java:compiler:0.9.6 - http://github.com/spullara/mustache.java)
+ * Crawler-commons (com.github.crawler-commons:crawler-commons:1.4 - https://github.com/crawler-commons/crawler-commons)
+ * Curator Client (org.apache.curator:curator-client:5.2.0 - http://curator.apache.org/curator-client)
+ * Curator Framework (org.apache.curator:curator-framework:5.2.0 - http://curator.apache.org/curator-framework)
+ * Curator Recipes (org.apache.curator:curator-recipes:5.2.0 - http://curator.apache.org/curator-recipes)
+ * error-prone annotations (com.google.errorprone:error_prone_annotations:2.14.0 - https://errorprone.info/error_prone_annotations)
+ * error-prone annotations (com.google.errorprone:error_prone_annotations:2.21.1 - https://errorprone.info/error_prone_annotations)
+ * Failsafe (dev.failsafe:failsafe:3.3.2 - https://failsafe.dev/failsafe)
+ * FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/)
+ * Google Android Annotations Library (com.google.android:annotations:4.1.1.4 - http://source.android.com/)
+ * Graphite Integration for Metrics (io.dropwizard.metrics:metrics-graphite:3.2.6 - http://metrics.dropwizard.io/metrics-graphite/)
+ * Gson (com.google.code.gson:gson:2.9.0 - https://github.com/google/gson/gson)
+ * Guava: Google Core Libraries for Java (com.google.guava:guava:18.0 - http://code.google.com/p/guava-libraries/guava)
+ * Guava: Google Core Libraries for Java (com.google.guava:guava:31.1-android - https://github.com/google/guava)
+ * Guava: Google Core Libraries for Java (com.google.guava:guava:32.1.3-jre - https://github.com/google/guava)
+ * Guava: Google Core Libraries for Java (com.google.guava:guava:33.0.0-jre - https://github.com/google/guava)
+ * Guava InternalFutureFailureAccess and InternalFutures (com.google.guava:failureaccess:1.0.1 - https://github.com/google/guava/failureaccess)
+ * Guava InternalFutureFailureAccess and InternalFutures (com.google.guava:failureaccess:1.0.2 - https://github.com/google/guava/failureaccess)
+ * Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture)
+ * HPPC Collections (com.carrotsearch:hppc:0.8.1 - http://labs.carrotsearch.com/hppc.html/hppc)
+ * IntelliJ IDEA Annotations (com.intellij:annotations:12.0 - http://www.jetbrains.org)
+ * io.grpc:grpc-api (io.grpc:grpc-api:1.50.2 - https://github.com/grpc/grpc-java)
+ * io.grpc:grpc-context (io.grpc:grpc-context:1.50.2 - https://github.com/grpc/grpc-java)
+ * io.grpc:grpc-core (io.grpc:grpc-core:1.50.2 - https://github.com/grpc/grpc-java)
+ * io.grpc:grpc-netty-shaded (io.grpc:grpc-netty-shaded:1.50.2 - https://github.com/grpc/grpc-java)
+ * io.grpc:grpc-protobuf (io.grpc:grpc-protobuf:1.50.2 - https://github.com/grpc/grpc-java)
+ * io.grpc:grpc-protobuf-lite (io.grpc:grpc-protobuf-lite:1.50.2 - https://github.com/grpc/grpc-java)
+ * io.grpc:grpc-stub (io.grpc:grpc-stub:1.50.2 - https://github.com/grpc/grpc-java)
+ * J2ObjC Annotations (com.google.j2objc:j2objc-annotations:1.3 - https://github.com/google/j2objc/)
+ * J2ObjC Annotations (com.google.j2objc:j2objc-annotations:2.8 - https://github.com/google/j2objc/)
+ * Jackcess (com.healthmarketscience.jackcess:jackcess:4.0.5 - https://jackcess.sourceforge.io)
+ * Jackcess Encrypt (com.healthmarketscience.jackcess:jackcess-encrypt:4.0.2 - http://jackcessencrypt.sf.net)
+ * Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.15.2 - https://github.com/FasterXML/jackson)
+ * Jackson-core (com.fasterxml.jackson.core:jackson-core:2.10.4 - https://github.com/FasterXML/jackson-core)
+ * Jackson-core (com.fasterxml.jackson.core:jackson-core:2.15.2 - https://github.com/FasterXML/jackson-core)
+ * Jackson-core (com.fasterxml.jackson.core:jackson-core:2.16.1 - https://github.com/FasterXML/jackson-core)
+ * jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.15.2 - https://github.com/FasterXML/jackson)
+ * Jackson dataformat: CBOR (com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.10.4 - http://github.com/FasterXML/jackson-dataformats-binary)
+ * Jackson dataformat: CBOR (com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.6 - http://github.com/FasterXML/jackson-dataformats-binary)
+ * Jackson dataformat: CBOR (com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.16.1 - https://github.com/FasterXML/jackson-dataformats-binary)
+ * Jackson dataformat: Smile (com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.10.4 - http://github.com/FasterXML/jackson-dataformats-binary)
+ * Jackson dataformat: Smile (com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.16.1 - https://github.com/FasterXML/jackson-dataformats-binary)
+ * Jackson-dataformat-YAML (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.10.4 - https://github.com/FasterXML/jackson-dataformats-text)
+ * Jackson-dataformat-YAML (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.16.1 - https://github.com/FasterXML/jackson-dataformats-text)
+ * Jackson-JAXRS-base (com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.12.7 - http://github.com/FasterXML/jackson-jaxrs-providers/jackson-jaxrs-base)
+ * Jackson-JAXRS-JSON (com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.7 - http://github.com/FasterXML/jackson-jaxrs-providers/jackson-jaxrs-json-provider)
+ * Jackson module: JAXB Annotations (com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.7 - https://github.com/FasterXML/jackson-modules-base)
+ * java-libpst (com.pff:java-libpst:0.9.3 - https://github.com/rjohnsondev/java-libpst)
+ * JCIP Annotations under Apache License (com.github.stephenc.jcip:jcip-annotations:1.0-1 - http://stephenc.github.com/jcip-annotations)
+ * JCL 1.2 implemented over SLF4J (org.slf4j:jcl-over-slf4j:2.0.10 - http://www.slf4j.org)
+ * JCL 1.2 implemented over SLF4J (org.slf4j:jcl-over-slf4j:2.0.9 - http://www.slf4j.org)
+ * JetBrains Java Annotations (org.jetbrains:annotations:24.1.0 - https://github.com/JetBrains/java-annotations)
+ * Jettison (org.codehaus.jettison:jettison:1.1 - no url defined)
+ * JMES Path Query library (com.amazonaws:jmespath-java:1.12.663 - https://aws.amazon.com/sdkforjava)
+ * Joda-Time (joda-time:joda-time:2.10.10 - https://www.joda.org/joda-time/)
+ * Joda-Time (joda-time:joda-time:2.12.2 - https://www.joda.org/joda-time/)
+ * Joda-Time (joda-time:joda-time:2.8.1 - http://www.joda.org/joda-time/)
+ * jsonic (net.arnx:jsonic:1.2.11 - http://jsonic.sourceforge.jp/)
+ * JVM Integration for Metrics (io.dropwizard.metrics:metrics-jvm:3.2.6 - http://metrics.dropwizard.io/metrics-jvm/)
+ * jwarc (org.netpreserve:jwarc:0.28.3 - https://github.com/iipc/jwarc)
+ * jwarc (org.netpreserve:jwarc:0.29.0 - https://github.com/iipc/jwarc)
+ * Kerb Simple Kdc (org.apache.kerby:kerb-simplekdc:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-simplekdc)
+ * Kerby ASN1 Project (org.apache.kerby:kerby-asn1:1.0.1 - http://directory.apache.org/kerby/kerby-common/kerby-asn1)
+ * Kerby Config (org.apache.kerby:kerby-config:1.0.1 - http://directory.apache.org/kerby/kerby-common/kerby-config)
+ * Kerby-kerb Admin (org.apache.kerby:kerb-admin:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-admin)
+ * Kerby-kerb Client (org.apache.kerby:kerb-client:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-client)
+ * Kerby-kerb Common (org.apache.kerby:kerb-common:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-common)
+ * Kerby-kerb core (org.apache.kerby:kerb-core:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-core)
+ * Kerby-kerb Crypto (org.apache.kerby:kerb-crypto:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-crypto)
+ * Kerby-kerb Identity (org.apache.kerby:kerb-identity:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-identity)
+ * Kerby-kerb Server (org.apache.kerby:kerb-server:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-server)
+ * Kerby-kerb Util (org.apache.kerby:kerb-util:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-util)
+ * Kerby PKIX Project (org.apache.kerby:kerby-pkix:1.0.1 - http://directory.apache.org/kerby/kerby-pkix)
+ * Kerby Util (org.apache.kerby:kerby-util:1.0.1 - http://directory.apache.org/kerby/kerby-common/kerby-util)
+ * Kerby XDR Project (org.apache.kerby:kerby-xdr:1.0.1 - http://directory.apache.org/kerby/kerby-common/kerby-xdr)
+ * Kotlin Stdlib (org.jetbrains.kotlin:kotlin-stdlib:1.8.21 - https://kotlinlang.org/)
+ * Kotlin Stdlib Common (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 - https://kotlinlang.org/)
+ * Kotlin Stdlib Jdk7 (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.8.21 - https://kotlinlang.org/)
+ * Kotlin Stdlib Jdk8 (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.8.21 - https://kotlinlang.org/)
+ * lang-mustache (org.opensearch.plugin:lang-mustache-client:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * language-detector (com.optimaize.languagedetector:language-detector:0.6 - https://github.com/optimaize/language-detector)
+ * Lucene Common Analyzers (org.apache.lucene:lucene-analyzers-common:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-analyzers-common)
+ * Lucene Core (org.apache.lucene:lucene-core:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-core)
+ * Lucene Grouping (org.apache.lucene:lucene-grouping:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-grouping)
+ * Lucene Highlighter (org.apache.lucene:lucene-highlighter:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-highlighter)
+ * Lucene Join (org.apache.lucene:lucene-join:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-join)
+ * Lucene Memory (org.apache.lucene:lucene-backward-codecs:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-backward-codecs)
+ * Lucene Memory (org.apache.lucene:lucene-memory:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-memory)
+ * Lucene Miscellaneous (org.apache.lucene:lucene-misc:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-misc)
+ * Lucene Queries (org.apache.lucene:lucene-queries:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-queries)
+ * Lucene QueryParsers (org.apache.lucene:lucene-queryparser:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-queryparser)
+ * Lucene Sandbox (org.apache.lucene:lucene-sandbox:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-sandbox)
+ * Lucene Spatial 3D (org.apache.lucene:lucene-spatial3d:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-spatial3d)
+ * Lucene Suggest (org.apache.lucene:lucene-suggest:8.11.1 - https://lucene.apache.org/lucene-parent/lucene-suggest)
+ * LZ4 and xxHash (org.lz4:lz4-java:1.8.0 - https://github.com/lz4/lz4-java)
+ * mapper-extras (org.opensearch.plugin:mapper-extras-client:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * Metrics Core (io.dropwizard.metrics:metrics-core:3.2.6 - http://metrics.dropwizard.io/metrics-core/)
+ * Netty/All-in-One (io.netty:netty-all:4.1.89.Final - https://netty.io/netty-all/)
+ * Netty/Buffer (io.netty:netty-buffer:4.1.89.Final - https://netty.io/netty-buffer/)
+ * Netty/Buffer (io.netty:netty-buffer:4.1.94.Final - https://netty.io/netty-buffer/)
+ * Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.89.Final - https://netty.io/netty-codec-dns/)
+ * Netty/Codec/HAProxy (io.netty:netty-codec-haproxy:4.1.89.Final - https://netty.io/netty-codec-haproxy/)
+ * Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.89.Final - https://netty.io/netty-codec-http/)
+ * Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.89.Final - https://netty.io/netty-codec-http2/)
+ * Netty/Codec/Memcache (io.netty:netty-codec-memcache:4.1.89.Final - https://netty.io/netty-codec-memcache/)
+ * Netty/Codec/MQTT (io.netty:netty-codec-mqtt:4.1.89.Final - https://netty.io/netty-codec-mqtt/)
+ * Netty/Codec/Redis (io.netty:netty-codec-redis:4.1.89.Final - https://netty.io/netty-codec-redis/)
+ * Netty/Codec/SMTP (io.netty:netty-codec-smtp:4.1.89.Final - https://netty.io/netty-codec-smtp/)
+ * Netty/Codec/Socks (io.netty:netty-codec-socks:4.1.89.Final - https://netty.io/netty-codec-socks/)
+ * Netty/Codec/Stomp (io.netty:netty-codec-stomp:4.1.89.Final - https://netty.io/netty-codec-stomp/)
+ * Netty/Codec/XML (io.netty:netty-codec-xml:4.1.89.Final - https://netty.io/netty-codec-xml/)
+ * Netty/Codec (io.netty:netty-codec:4.1.89.Final - https://netty.io/netty-codec/)
+ * Netty/Codec (io.netty:netty-codec:4.1.94.Final - https://netty.io/netty-codec/)
+ * Netty/Common (io.netty:netty-common:4.1.89.Final - https://netty.io/netty-common/)
+ * Netty/Common (io.netty:netty-common:4.1.94.Final - https://netty.io/netty-common/)
+ * Netty/Handler/Proxy (io.netty:netty-handler-proxy:4.1.89.Final - https://netty.io/netty-handler-proxy/)
+ * Netty/Handler/Ssl/Ocsp (io.netty:netty-handler-ssl-ocsp:4.1.89.Final - https://netty.io/netty-handler-ssl-ocsp/)
+ * Netty/Handler (io.netty:netty-handler:4.1.89.Final - https://netty.io/netty-handler/)
+ * Netty/Handler (io.netty:netty-handler:4.1.94.Final - https://netty.io/netty-handler/)
+ * Netty/Resolver/DNS/Classes/MacOS (io.netty:netty-resolver-dns-classes-macos:4.1.89.Final - https://netty.io/netty-resolver-dns-classes-macos/)
+ * Netty/Resolver/DNS/Native/MacOS (io.netty:netty-resolver-dns-native-macos:4.1.89.Final - https://netty.io/netty-resolver-dns-native-macos/)
+ * Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.89.Final - https://netty.io/netty-resolver-dns/)
+ * Netty/Resolver (io.netty:netty-resolver:4.1.89.Final - https://netty.io/netty-resolver/)
+ * Netty/Resolver (io.netty:netty-resolver:4.1.94.Final - https://netty.io/netty-resolver/)
+ * Netty/TomcatNative [BoringSSL - Static] (io.netty:netty-tcnative-boringssl-static:2.0.61.Final - https://github.com/netty/netty-tcnative/netty-tcnative-boringssl-static/)
+ * Netty/TomcatNative [OpenSSL - Classes] (io.netty:netty-tcnative-classes:2.0.61.Final - https://github.com/netty/netty-tcnative/netty-tcnative-classes/)
+ * Netty/Transport/Classes/Epoll (io.netty:netty-transport-classes-epoll:4.1.89.Final - https://netty.io/netty-transport-classes-epoll/)
+ * Netty/Transport/Classes/Epoll (io.netty:netty-transport-classes-epoll:4.1.94.Final - https://netty.io/netty-transport-classes-epoll/)
+ * Netty/Transport/Classes/KQueue (io.netty:netty-transport-classes-kqueue:4.1.89.Final - https://netty.io/netty-transport-classes-kqueue/)
+ * Netty/Transport/Native/Epoll (io.netty:netty-transport-native-epoll:4.1.63.Final - https://netty.io/netty-transport-native-epoll/)
+ * Netty/Transport/Native/Epoll (io.netty:netty-transport-native-epoll:4.1.89.Final - https://netty.io/netty-transport-native-epoll/)
+ * Netty/Transport/Native/Epoll (io.netty:netty-transport-native-epoll:4.1.94.Final - https://netty.io/netty-transport-native-epoll/)
+ * Netty/Transport/Native/KQueue (io.netty:netty-transport-native-kqueue:4.1.89.Final - https://netty.io/netty-transport-native-kqueue/)
+ * Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.89.Final - https://netty.io/netty-transport-native-unix-common/)
+ * Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.94.Final - https://netty.io/netty-transport-native-unix-common/)
+ * Netty/Transport/RXTX (io.netty:netty-transport-rxtx:4.1.89.Final - https://netty.io/netty-transport-rxtx/)
+ * Netty/Transport/SCTP (io.netty:netty-transport-sctp:4.1.89.Final - https://netty.io/netty-transport-sctp/)
+ * Netty/Transport/UDT (io.netty:netty-transport-udt:4.1.89.Final - https://netty.io/netty-transport-udt/)
+ * Netty/Transport (io.netty:netty-transport:4.1.89.Final - https://netty.io/netty-transport/)
+ * Netty/Transport (io.netty:netty-transport:4.1.94.Final - https://netty.io/netty-transport/)
+ * Netty (io.netty:netty:3.10.6.Final - http://netty.io/)
+ * Nimbus JOSE+JWT (com.nimbusds:nimbus-jose-jwt:9.8.1 - https://bitbucket.org/connect2id/nimbus-jose-jwt)
+ * Non-Blocking Reactive Foundation for the JVM (io.projectreactor:reactor-core:3.5.14 - https://github.com/reactor/reactor-core)
+ * Objenesis (org.objenesis:objenesis:3.3 - http://objenesis.org/objenesis)
+ * Ogg and Vorbis for Java, Core (org.gagravarr:vorbis-java-core:0.8 - https://github.com/Gagravarr/VorbisJava)
+ * okhttp (com.squareup.okhttp3:okhttp:4.12.0 - https://square.github.io/okhttp/)
+ * okhttp-brotli (com.squareup.okhttp3:okhttp-brotli:4.12.0 - https://square.github.io/okhttp/)
+ * okio (com.squareup.okio:okio:3.6.0 - https://github.com/square/okio/)
+ * okio (com.squareup.okio:okio-jvm:3.6.0 - https://github.com/square/okio/)
+ * opensearch-cli (org.opensearch:opensearch-cli:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * opensearch-common (org.opensearch:opensearch-common:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * opensearch-compress (org.opensearch:opensearch-compress:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * opensearch-core (org.opensearch:opensearch-core:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * opensearch-geo (org.opensearch:opensearch-geo:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * opensearch-secure-sm (org.opensearch:opensearch-secure-sm:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * opensearch-telemetry (org.opensearch:opensearch-telemetry:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * opensearch-x-content (org.opensearch:opensearch-x-content:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-api:1.35.0 - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-api-events:1.35.0-alpha - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-context:1.35.0 - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-exporter-logging:1.35.0 - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-extension-incubator:1.35.0-alpha - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-sdk:1.35.0 - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-sdk-common:1.35.0 - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.35.0 - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.35.0 - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-sdk-logs:1.35.0 - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-sdk-metrics:1.35.0 - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-sdk-trace:1.35.0 - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Java (io.opentelemetry:opentelemetry-semconv:1.15.0-alpha - https://github.com/open-telemetry/opentelemetry-java)
+ * OpenTelemetry Semantic Conventions Java (io.opentelemetry.semconv:opentelemetry-semconv:1.23.1-alpha - https://github.com/open-telemetry/semantic-conventions-java)
+ * org.seleniumhq.selenium:selenium-api (org.seleniumhq.selenium:selenium-api:4.18.1 - https://selenium.dev/)
+ * org.seleniumhq.selenium:selenium-http (org.seleniumhq.selenium:selenium-http:4.18.1 - https://selenium.dev/)
+ * org.seleniumhq.selenium:selenium-json (org.seleniumhq.selenium:selenium-json:4.18.1 - https://selenium.dev/)
+ * org.seleniumhq.selenium:selenium-manager (org.seleniumhq.selenium:selenium-manager:4.18.1 - https://selenium.dev/)
+ * org.seleniumhq.selenium:selenium-os (org.seleniumhq.selenium:selenium-os:4.18.1 - https://selenium.dev/)
+ * org.seleniumhq.selenium:selenium-remote-driver (org.seleniumhq.selenium:selenium-remote-driver:4.18.1 - https://selenium.dev/)
+ * org.seleniumhq.selenium:selenium-support (org.seleniumhq.selenium:selenium-support:4.18.1 - https://selenium.dev/)
+ * parent-join (org.opensearch.plugin:parent-join-client:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * parso (com.epam:parso:2.0.14 - https://github.com/epam/parso)
+ * PDFBox JBIG2 ImageIO plugin (org.apache.pdfbox:jbig2-imageio:3.0.4 - https://www.apache.org/jbig2-imageio/)
+ * perfmark:perfmark-api (io.perfmark:perfmark-api:0.25.0 - https://github.com/perfmark/perfmark)
+ * proto-google-common-protos (com.google.api.grpc:proto-google-common-protos:2.9.0 - https://github.com/googleapis/java-iam/proto-google-common-protos)
+ * rank-eval (org.opensearch.plugin:rank-eval-client:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * rest (org.elasticsearch.client:elasticsearch-rest-client:7.17.7 - https://github.com/elastic/elasticsearch)
+ * rest (org.opensearch.client:opensearch-rest-client:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * rest-high-level (org.opensearch.client:opensearch-rest-high-level-client:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * rome (com.rometools:rome:2.1.0 - http://rometools.com/rome)
+ * rome-utils (com.rometools:rome-utils:2.1.0 - http://rometools.com/rome-utils)
+ * server (org.opensearch:opensearch:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * Shaded Deps for Storm Client (org.apache.storm:storm-shaded-deps:2.6.1 - https://storm.apache.org/storm-shaded-deps)
+ * SnakeYAML (org.yaml:snakeyaml:2.2 - https://bitbucket.org/snakeyaml/snakeyaml)
+ * snappy-java (org.xerial.snappy:snappy-java:1.1.8.2 - https://github.com/xerial/snappy-java)
+ * sniffer (org.elasticsearch.client:elasticsearch-rest-client-sniffer:7.17.7 - https://github.com/elastic/elasticsearch)
+ * sniffer (org.opensearch.client:opensearch-rest-client-sniffer:2.12.0 - https://github.com/opensearch-project/OpenSearch.git)
+ * SparseBitSet (com.zaxxer:SparseBitSet:1.2 - https://github.com/brettwooldridge/SparseBitSet)
+ * storm-autocreds (org.apache.storm:storm-autocreds:2.6.1 - https://storm.apache.org/external/storm-autocreds)
+ * Storm Client (org.apache.storm:storm-client:2.6.1 - https://storm.apache.org/storm-client)
- * storm-crawler-core (com.digitalpebble.stormcrawler:storm-crawler-core:2.12-SNAPSHOT - https://github.com/DigitalPebble/storm-crawler/tree/master/core)
+ * storm-hdfs (org.apache.storm:storm-hdfs:2.6.1 - https://storm.apache.org/external/storm-hdfs)
+ * swagger-annotations-jakarta (io.swagger.core.v3:swagger-annotations-jakarta:2.2.17 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations-jakarta)
+ * TagSoup (org.ccil.cowan.tagsoup:tagsoup:1.2.1 - http://home.ccil.org/~cowan/XML/tagsoup/)
+ * T-Digest (com.tdunning:t-digest:3.2 - https://github.com/tdunning/t-digest)
+ * Token provider (org.apache.kerby:token-provider:1.0.1 - http://directory.apache.org/kerby/kerby-provider/token-provider)
+ * urlfrontier-API (com.github.crawler-commons:urlfrontier-API:2.3.1 - https://github.com/crawler-commons/url-frontier/urlfrontier-API)
+ * Woodstox (com.fasterxml.woodstox:woodstox-core:5.4.0 - https://github.com/FasterXML/woodstox)
+ * Xerces2-j (xerces:xercesImpl:2.12.2 - https://xerces.apache.org/xerces2-j/)
+ * XmlBeans (org.apache.xmlbeans:xmlbeans:5.1.1 - https://xmlbeans.apache.org/)
+
+ Apache License, Version 2.0, Eclipse Public License - Version 1.0
+
+ * Jetty :: Asynchronous HTTP Client (org.eclipse.jetty:jetty-client:9.4.51.v20230217 - https://eclipse.org/jetty/jetty-client)
+ * Jetty :: Http Utility (org.eclipse.jetty:jetty-http:9.4.51.v20230217 - https://eclipse.org/jetty/jetty-http)
+ * Jetty :: IO Utility (org.eclipse.jetty:jetty-io:9.4.51.v20230217 - https://eclipse.org/jetty/jetty-io)
+ * Jetty :: Security (org.eclipse.jetty:jetty-security:9.4.51.v20230217 - https://eclipse.org/jetty/jetty-security)
+ * Jetty :: Server Core (org.eclipse.jetty:jetty-server:9.4.51.v20230217 - https://eclipse.org/jetty/jetty-server)
+ * Jetty :: Servlet Handling (org.eclipse.jetty:jetty-servlet:9.4.51.v20230217 - https://eclipse.org/jetty/jetty-servlet)
+ * Jetty :: Utilities :: Ajax(JSON) (org.eclipse.jetty:jetty-util-ajax:9.4.51.v20230217 - https://eclipse.org/jetty/jetty-util-ajax)
+ * Jetty :: Utilities (org.eclipse.jetty:jetty-util:9.4.51.v20230217 - https://eclipse.org/jetty/jetty-util)
+ * Jetty :: Webapp Application Support (org.eclipse.jetty:jetty-webapp:9.4.51.v20230217 - https://eclipse.org/jetty/jetty-webapp)
+ * Jetty :: Websocket :: API (org.eclipse.jetty.websocket:websocket-api:9.4.51.v20230217 - https://eclipse.org/jetty/websocket-parent/websocket-api)
+ * Jetty :: Websocket :: Client (org.eclipse.jetty.websocket:websocket-client:9.4.51.v20230217 - https://eclipse.org/jetty/websocket-parent/websocket-client)
+ * Jetty :: Websocket :: Common (org.eclipse.jetty.websocket:websocket-common:9.4.51.v20230217 - https://eclipse.org/jetty/websocket-parent/websocket-common)
+ * Jetty :: XML utilities (org.eclipse.jetty:jetty-xml:9.4.51.v20230217 - https://eclipse.org/jetty/jetty-xml)
+
+ Apache License, Version 2.0, Eclipse Public License - Version 2.0
+
+ * Jetty :: ALPN :: Client (org.eclipse.jetty:jetty-alpn-client:10.0.19 - https://eclipse.dev/jetty/jetty-alpn-parent/jetty-alpn-client)
+ * Jetty :: ALPN :: JDK9 Client Implementation (org.eclipse.jetty:jetty-alpn-java-client:10.0.19 - https://eclipse.dev/jetty/jetty-alpn-parent/jetty-alpn-java-client)
+ * Jetty :: Asynchronous HTTP Client (org.eclipse.jetty:jetty-client:10.0.19 - https://eclipse.dev/jetty/jetty-client)
+ * Jetty :: HTTP2 :: Client (org.eclipse.jetty.http2:http2-client:10.0.19 - https://eclipse.dev/jetty/http2-parent/http2-client)
+ * Jetty :: HTTP2 :: Common (org.eclipse.jetty.http2:http2-common:10.0.19 - https://eclipse.dev/jetty/http2-parent/http2-common)
+ * Jetty :: HTTP2 :: HPACK (org.eclipse.jetty.http2:http2-hpack:10.0.19 - https://eclipse.dev/jetty/http2-parent/http2-hpack)
+ * Jetty :: HTTP2 :: HTTP Client Transport (org.eclipse.jetty.http2:http2-http-client-transport:10.0.19 - https://eclipse.dev/jetty/http2-parent/http2-http-client-transport)
+ * Jetty :: Http Utility (org.eclipse.jetty:jetty-http:10.0.19 - https://eclipse.dev/jetty/jetty-http)
+ * Jetty :: IO Utility (org.eclipse.jetty:jetty-io:10.0.19 - https://eclipse.dev/jetty/jetty-io)
+ * Jetty :: Utilities (org.eclipse.jetty:jetty-util:10.0.19 - https://eclipse.dev/jetty/jetty-util)
+
+ Apache License, Version 2.0, LGPL-2.1-or-later
+
+ * Java Native Access (net.java.dev.jna:jna:5.10.0 - https://github.com/java-native-access/jna)
+ * Java Native Access (net.java.dev.jna:jna:5.13.0 - https://github.com/java-native-access/jna)
+
+ Bouncy Castle Licence
+
+ * Bouncy Castle ASN.1 Extension and Utility APIs (org.bouncycastle:bcutil-jdk18on:1.76 - https://www.bouncycastle.org/java.html)
+ * Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (org.bouncycastle:bcpkix-jdk18on:1.76 - https://www.bouncycastle.org/java.html)
+ * Bouncy Castle Provider (org.bouncycastle:bcprov-jdk18on:1.76 - https://www.bouncycastle.org/java.html)
+ * Bouncy Castle S/MIME API (org.bouncycastle:bcmail-jdk18on:1.76 - https://www.bouncycastle.org/java.html)
+
+ BSD-2-Clause, Public Domain, per Creative Commons CC0
+
+ * HdrHistogram (org.hdrhistogram:HdrHistogram:2.1.12 - http://hdrhistogram.github.io/HdrHistogram/)
+
+ BSD 2-Clause License
+
+ * dnsjava (dnsjava:dnsjava:2.1.7 - http://www.dnsjava.org)
+ * zstd-jni (com.github.luben:zstd-jni:1.5.5-5 - https://github.com/luben/zstd-jni)
+
+ BSD 3-Clause License
+
+ * Adobe XMPCore (com.adobe.xmp:xmpcore:6.1.11 - https://www.adobe.com/devnet/xmp/library/eula-xmp-library-java.html)
+ * asm (org.ow2.asm:asm:9.6 - http://asm.ow2.io/)
+ * leveldbjni-all (org.fusesource.leveldbjni:leveldbjni-all:1.8 - http://leveldbjni.fusesource.org/leveldbjni-all)
+ * Protocol Buffer Java API (com.google.protobuf:protobuf-java:2.5.0 - http://code.google.com/p/protobuf)
+ * Protocol Buffers [Core] (com.google.protobuf:protobuf-java:3.21.7 - https://developers.google.com/protocol-buffers/protobuf-java/)
+ * Protocol Buffers [Core] (com.google.protobuf:protobuf-java:3.22.3 - https://developers.google.com/protocol-buffers/protobuf-java/)
+
+ BSD 3-clause License w/nuclear disclaimer
+
+ * Java Advanced Imaging Image I/O Tools API core (standalone) (com.github.jai-imageio:jai-imageio-core:1.4.0 - https://github.com/jai-imageio/jai-imageio-core)
+
+ BSD License
+
+ * curvesapi (com.github.virtuald:curvesapi:1.07 - https://github.com/virtuald/curvesapi)
+ * JLine Bundle (org.jline:jline:3.9.0 - http://nexus.sonatype.org/oss-repository-hosting.html/jline-parent/jline)
+ * JMatIO (org.tallison:jmatio:1.5 - https://github.com/tballison/jmatio)
+ * JZlib (com.jcraft:jzlib:1.1.3 - http://www.jcraft.com/jzlib/)
+ * Stax2 API (org.codehaus.woodstox:stax2-api:4.2.1 - http://github.com/FasterXML/stax2-api)
+
+ CDDL, v1.0, LGPL, v2.1 or later
+
+ * JHighlight (org.codelibs:jhighlight:1.1.0 - https://github.com/codelibs/jhighlight)
+
+ CDDL/GPLv2+CE
+
+ * JavaBeans Activation Framework API jar (javax.activation:javax.activation-api:1.2.0 - http://java.net/all/javax.activation-api/)
+
+ Common Development and Distribution License
+
+ * Expression Language 3.0 (org.glassfish:javax.el:3.0.1-b12 - http://uel.java.net)
+ * Java Servlet API (javax.servlet:javax.servlet-api:3.1.0 - http://servlet-spec.java.net)
+ * javax.annotation API (javax.annotation:javax.annotation-api:1.3.2 - http://jcp.org/en/jsr/detail?id=250)
+ * jsr311-api (javax.ws.rs:jsr311-api:1.1.1 - https://jsr311.dev.java.net)
+
+ Common Development and Distribution License (CDDL) v1.1, The GNU General Public License (GPL), Version 2, With Classpath Exception
+
+ * jaxb-api (javax.xml.bind:jaxb-api:2.3.0 - https://github.com/javaee/jaxb-spec/jaxb-api)
+ * JAXB RI (com.sun.xml.bind:jaxb-impl:2.2.3-1 - http://jaxb.java.net/)
+ * jersey-client (com.sun.jersey:jersey-client:1.19.4 - https://jersey.java.net/jersey-client/)
+ * jersey-core (com.sun.jersey:jersey-core:1.19.4 - https://jersey.java.net/jersey-core/)
+ * jersey-json (com.github.pjfanning:jersey-json:1.20 - https://github.com/pjfanning/jersey-1.x)
+ * jersey-server (com.sun.jersey:jersey-server:1.19.4 - https://jersey.java.net/jersey-server/)
+ * jersey-servlet (com.sun.jersey:jersey-servlet:1.19.4 - https://jersey.java.net/jersey-servlet/)
+ * jsp-api (javax.servlet.jsp:jsp-api:2.1 - no url defined)
+
+ Eclipse Distribution License, Version 1.0
+
+ * istack common utility code runtime (com.sun.istack:istack-commons-runtime:3.0.12 - https://projects.eclipse.org/projects/ee4j/istack-commons/istack-commons-runtime)
+ * jakarta.xml.bind-api (jakarta.xml.bind:jakarta.xml.bind-api:2.3.2 - https://github.com/eclipse-ee4j/jaxb-api/jakarta.xml.bind-api)
+ * JavaBeans Activation Framework (com.sun.activation:jakarta.activation:1.2.1 - https://github.com/eclipse-ee4j/jaf/jakarta.activation)
+ * JavaBeans Activation Framework API jar (jakarta.activation:jakarta.activation-api:1.2.1 - https://github.com/eclipse-ee4j/jaf/jakarta.activation-api)
+ * JAXB Runtime (org.glassfish.jaxb:jaxb-runtime:2.3.6 - https://eclipse-ee4j.github.io/jaxb-ri/)
+ * TXW2 Runtime (org.glassfish.jaxb:txw2:2.3.6 - https://eclipse-ee4j.github.io/jaxb-ri/)
+
+ Eclipse Public License, Version 2.0, GPL-2.0-with-classpath-exception
+
+ * Jakarta RESTful WS API (jakarta.ws.rs:jakarta.ws.rs-api:3.1.0 - https://github.com/eclipse-ee4j/jaxrs-api)
+
+ Eclipse Public License, Version 2.0, The GNU General Public License (GPL), Version 2, With Classpath Exception
+
+ * Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:1.3.5 - https://projects.eclipse.org/projects/ee4j.ca)
+
+ Elastic License 2.0
+
+ * rest-high-level (org.elasticsearch.client:elasticsearch-rest-high-level-client:7.17.7 - https://github.com/elastic/elasticsearch)
+
+ Elastic License 2.0, Server Side Public License, v 1
+
+ * aggs-matrix-stats (org.elasticsearch.plugin:aggs-matrix-stats-client:7.17.7 - https://github.com/elastic/elasticsearch)
+ * elasticsearch-cli (org.elasticsearch:elasticsearch-cli:7.17.7 - https://github.com/elastic/elasticsearch)
+ * elasticsearch-core (org.elasticsearch:elasticsearch-core:7.17.7 - https://github.com/elastic/elasticsearch)
+ * elasticsearch-geo (org.elasticsearch:elasticsearch-geo:7.17.7 - https://github.com/elastic/elasticsearch)
+ * elasticsearch-lz4 (org.elasticsearch:elasticsearch-lz4:7.17.7 - https://github.com/elastic/elasticsearch)
+ * elasticsearch-plugin-classloader (org.elasticsearch:elasticsearch-plugin-classloader:7.17.7 - https://github.com/elastic/elasticsearch)
+ * elasticsearch-secure-sm (org.elasticsearch:elasticsearch-secure-sm:7.17.7 - https://github.com/elastic/elasticsearch)
+ * elasticsearch-x-content (org.elasticsearch:elasticsearch-x-content:7.17.7 - https://github.com/elastic/elasticsearch)
+ * lang-mustache (org.elasticsearch.plugin:lang-mustache-client:7.17.7 - https://github.com/elastic/elasticsearch)
+ * mapper-extras (org.elasticsearch.plugin:mapper-extras-client:7.17.7 - https://github.com/elastic/elasticsearch)
+ * parent-join (org.elasticsearch.plugin:parent-join-client:7.17.7 - https://github.com/elastic/elasticsearch)
+ * rank-eval (org.elasticsearch.plugin:rank-eval-client:7.17.7 - https://github.com/elastic/elasticsearch)
+ * server (org.elasticsearch:elasticsearch:7.17.7 - https://github.com/elastic/elasticsearch)
+
+ GENERAL PUBLIC LICENSE, version 3 (GPL-3.0), GNU LESSER GENERAL PUBLIC LICENSE, version 3 (LGPL-3.0), Mozilla Public License Version 1.1
+
+ * juniversalchardet (com.github.albfernandez:juniversalchardet:2.4.0 - https://github.com/albfernandez/juniversalchardet)
+
+ MIT-0
+
+ * reactive-streams (org.reactivestreams:reactive-streams:1.0.4 - http://www.reactive-streams.org/)
+
+ MIT License
+
+ * Animal Sniffer Annotations (org.codehaus.mojo:animal-sniffer-annotations:1.21 - https://www.mojohaus.org/animal-sniffer/animal-sniffer-annotations)
+ * Checker Qual (org.checkerframework:checker-qual:3.37.0 - https://checkerframework.org/)
+ * dd-plist (com.googlecode.plist:dd-plist:1.27 - http://www.github.com/3breadt/dd-plist)
+ * JCodings (org.jruby.jcodings:jcodings:1.0.55 - http://nexus.sonatype.org/oss-repository-hosting.html/jcodings)
+ * Joni (org.jruby.joni:joni:2.1.31 - http://nexus.sonatype.org/oss-repository-hosting.html/joni)
+ * JOpt Simple (net.sf.jopt-simple:jopt-simple:5.0.2 - http://pholser.github.io/jopt-simple)
+ * JOpt Simple (net.sf.jopt-simple:jopt-simple:5.0.4 - http://jopt-simple.github.io/jopt-simple)
+ * jsoup Java HTML Parser (org.jsoup:jsoup:1.17.2 - https://jsoup.org/)
+ * org.brotli:dec (org.brotli:dec:0.1.2 - http://brotli.org/dec)
+ * semver4j (org.semver4j:semver4j:5.2.2 - https://github.com/semver4j/semver4j)
+ * SLF4J API Module (org.slf4j:slf4j-api:1.7.36 - http://www.slf4j.org)
+ * SLF4J API Module (org.slf4j:slf4j-api:1.7.6 - http://www.slf4j.org)
+ * SLF4J API Module (org.slf4j:slf4j-api:2.0.10 - http://www.slf4j.org)
+ * xsoup (us.codecraft:xsoup:0.3.7 - https://github.com/code4craft/xsoup/)
+
+ Public Domain
+
+ * XZ for Java (org.tukaani:xz:1.9 - https://tukaani.org/xz/java.html)
+
+ Public Domain, per Creative Commons CC0
+
+ * HdrHistogram (org.hdrhistogram:HdrHistogram:2.1.9 - http://hdrhistogram.github.io/HdrHistogram/)
+
+ Revised BSD
+
+ * JSch (com.jcraft:jsch:0.1.55 - http://www.jcraft.com/jsch/)
+
+ Similar to Apache License but with the acknowledgment clause removed
+
+ * JDOM (org.jdom:jdom2:2.0.6.1 - http://www.jdom.org)
+
+ The Go license
+
+ * re2j (com.google.re2j:re2j:1.1 - http://github.com/google/re2j)
+
+ Unicode/ICU License
+
+ * icu4j (com.ibm.icu:icu4j:74.2 - https://icu.unicode.org/main/icu4j/)
+
+ UnRar License
+
+ * Java Unrar (com.github.junrar:junrar:7.5.5 - https://github.com/junrar/junrar)
diff --cc external/opensearch/archetype/src/main/resources/archetype-resources/injection.flux
index 56556426,060c1052..2f66da4e
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/injection.flux
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/injection.flux
@@@ -28,13 -28,9 +28,13 @@@ bolts
parallelism: 1
- id: "status"
- className: "com.digitalpebble.stormcrawler.opensearch.persistence.StatusUpdaterBolt"
+ className: "org.apache.stormcrawler.opensearch.persistence.StatusUpdaterBolt"
parallelism: 1
+ - id: "queues"
+ className: "com.digitalpebble.stormcrawler.opensearch.persistence.QueueBolt"
+ parallelism: 1
+
streams:
- from: "filespout"
to: "filter"
@@@ -49,13 -45,6 +49,13 @@@
streamId: "status"
type: CUSTOM
customClass:
- className: "com.digitalpebble.stormcrawler.util.URLStreamGrouping"
+ className: "org.apache.stormcrawler.util.URLStreamGrouping"
constructorArgs:
- "byDomain"
+
+ - from: "status"
+ to: "queues"
+ grouping:
+ type: FIELDS
+ args: ["key"]
+ streamId: "queue"
diff --cc external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml
index 731d0827,f37b8d5a..d4145481
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml
@@@ -13,13 -21,9 +21,13 @@@ config
opensearch.indexer.concurrentRequests: 1
# MetricsConsumer
- opensearch.metrics.addresses: "http://localhost:9200"
+ # opensearch.metrics.addresses: "http://localhost:9200"
opensearch.metrics.index.name: "metrics"
+ # Queues index
+ opensearch.queues.addresses: "http://localhost:9200"
+ opensearch.queues.index.name: "queues"
+
# Spout and persistence bolt
opensearch.status.addresses: "http://localhost:9200"
opensearch.status.index.name: "status"
diff --cc external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/queues.mapping
index 7c880b3c,00000000..7c880b3c
mode 100644,000000..100644
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/queues.mapping
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/queues.mapping
diff --cc external/opensearch/opensearch-conf.yaml
index 3be058aa,f37b8d5a..2279ec89
--- a/external/opensearch/opensearch-conf.yaml
+++ b/external/opensearch/opensearch-conf.yaml
@@@ -13,12 -21,8 +21,12 @@@ config
opensearch.indexer.concurrentRequests: 1
# MetricsConsumer
- opensearch.metrics.addresses: "http://localhost:9200"
+ # opensearch.metrics.addresses: "http://localhost:9200"
opensearch.metrics.index.name: "metrics"
+
+ # Queues index
+ opensearch.queues.addresses: "http://localhost:9200"
+ opensearch.queues.index.name: "queues"
# Spout and persistence bolt
opensearch.status.addresses: "http://localhost:9200"
diff --cc external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
index dd4e0ff1,00000000..b866c1bf
mode 100644,000000..100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
@@@ -1,124 -1,0 +1,122 @@@
+/**
+ * Licensed to DigitalPebble Ltd under one or more contributor license agreements. See the NOTICE
+ * file distributed with this work for additional information regarding copyright ownership.
+ * DigitalPebble licenses this file to You under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy of the
+ * License at
+ *
+ * <p>http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * <p>Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
- package com.digitalpebble.stormcrawler.opensearch.persistence;
++package org.apache.stormcrawler.opensearch.persistence;
+
- import com.digitalpebble.stormcrawler.opensearch.IndexCreation;
- import com.digitalpebble.stormcrawler.opensearch.OpenSearchConnection;
- import com.digitalpebble.stormcrawler.util.ConfUtils;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseRichBolt;
+import org.apache.storm.tuple.Tuple;
++import org.apache.stormcrawler.opensearch.IndexCreation;
++import org.apache.stormcrawler.opensearch.OpenSearchConnection;
++import org.apache.stormcrawler.util.ConfUtils;
+import org.joda.time.Instant;
+import org.opensearch.action.index.IndexRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Sends information about the queues into an OpenSearch index. This has to be connected to a status
+ * updater bolt.
+ */
+public class QueueBolt extends BaseRichBolt {
+
+ private static final Logger LOG = LoggerFactory.getLogger(QueueBolt.class);
+
+ private static final String OSBoltType = "queues";
+
+ static final String ESIndexNameParamName =
- com.digitalpebble.stormcrawler.opensearch.Constants.PARAMPREFIX
- + OSBoltType
- + ".index.name";
++ org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".index.name";
+
+ private OutputCollector _collector;
+
+ private String indexName;
+
+ private OpenSearchConnection connection;
+
+ private Cache<String, String> knownQueue;
+
+ public QueueBolt() {}
+
+ /** Sets the index name instead of taking it from the configuration. * */
+ public QueueBolt(String indexName) {
+ this.indexName = indexName;
+ }
+
+ @Override
+ public void prepare(
+ Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
+ _collector = collector;
+ if (indexName == null) {
+ indexName = ConfUtils.getString(conf, QueueBolt.ESIndexNameParamName, OSBoltType);
+ }
+ try {
+ connection = OpenSearchConnection.getConnection(conf, OSBoltType);
+ } catch (Exception e1) {
+ LOG.error("Can't connect to OpenSearch", e1);
+ throw new RuntimeException(e1);
+ }
+ try {
+ IndexCreation.checkOrCreateIndex(connection.getClient(), indexName, OSBoltType, LOG);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ knownQueue = Caffeine.newBuilder().maximumSize(10000).build();
+ }
+
+ @Override
+ public void cleanup() {
+ if (connection != null) connection.close();
+ }
+
+ @Override
+ public void execute(Tuple tuple) {
+
+ final String key = tuple.getStringByField("key");
+
+ // check whether this key is already known
+ if (knownQueue.getIfPresent(key) != null) {
+ _collector.ack(tuple);
+ return;
+ }
+
+ final String docID = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key);
+
+ final HashMap<String, String> fields = new HashMap<>();
+ fields.put("key", key);
+ fields.put("lastUpdated", Instant.now().toString());
+
+ final IndexRequest indexRequest =
+ new IndexRequest(indexName).source(fields).id(docID).create(true);
+
+ knownQueue.put(key, docID);
+
+ connection.addToProcessor(indexRequest);
+
+ // ack no matter what
+ _collector.ack(tuple);
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ // nothing to do here - this bolt is the last of a topology
+ }
+}
diff --cc external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
index 74b08e57,abdf5574..0185a75a
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
@@@ -45,7 -37,16 +37,17 @@@ import org.apache.storm.metric.api.Mult
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
+ import org.apache.stormcrawler.Metadata;
+ import org.apache.stormcrawler.opensearch.BulkItemResponseToFailedFlag;
+ import org.apache.stormcrawler.opensearch.Constants;
+ import org.apache.stormcrawler.opensearch.IndexCreation;
+ import org.apache.stormcrawler.opensearch.OpenSearchConnection;
+ import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt;
+ import org.apache.stormcrawler.persistence.Status;
+ import org.apache.stormcrawler.util.ConfUtils;
+ import org.apache.stormcrawler.util.PerSecondReducer;
+ import org.apache.stormcrawler.util.URLPartitioner;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.opensearch.action.DocWriteRequest;
@@@ -230,12 -244,6 +245,11 @@@ public class StatusUpdaterBolt extends
partitionKey = "_DEFAULT_";
}
+ // send a tuple on the queue stream in case a bolt
+ // wants to handle it
+ super._collector.emit(
- com.digitalpebble.stormcrawler.Constants.QUEUE_STREAM_NAME,
- new Values(partitionKey));
++ org.apache.stormcrawler.Constants.QUEUE_STREAM_NAME, new Values(partitionKey));
+
// store routing key in metadata?
if (StringUtils.isNotBlank(fieldNameForRoutingKey) && routingFieldNameInMetadata) {
builder.field(fieldNameForRoutingKey, partitionKey);