You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by hu...@apache.org on 2024/03/28 13:47:29 UTC
(incubator-stormcrawler) 01/01: Changed package names to org.apache + fixed references to DigitalPebble where possible (#1165)
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-stormcrawler.git
commit bdc34cbc7d85a08ba4a524b0e96deeee5e827928
Author: Julien Nioche <ju...@digitalpebble.com>
AuthorDate: Thu Mar 28 11:30:35 2024 +0000
Changed package names to org.apache + fixed references to DigitalPebble where possible (#1165)
Signed-off-by: Julien Nioche <ju...@digitalpebble.com>
---
README.md | 16 ++++----
THIRD-PARTY.txt | 3 +-
archetype/pom.xml | 2 +-
.../archetype-resources/crawler-conf.yaml | 8 ++--
.../resources/archetype-resources/crawler.flux | 20 ++++-----
.../src/main/resources/archetype-resources/pom.xml | 6 +--
.../src/main/java/CrawlTopology.java | 24 +++++------
.../src/main/resources/jsoupfilters.json | 6 +--
.../src/main/resources/parsefilters.json | 8 ++--
.../src/main/resources/urlfilters.json | 18 ++++----
core/pom.xml | 6 +--
.../apache}/stormcrawler/ConfigurableTopology.java | 6 +--
.../apache}/stormcrawler/Constants.java | 2 +-
.../apache}/stormcrawler/JSONResource.java | 2 +-
.../apache}/stormcrawler/Metadata.java | 2 +-
.../apache}/stormcrawler/bolt/FeedParserBolt.java | 28 ++++++-------
.../apache}/stormcrawler/bolt/FetcherBolt.java | 30 +++++++-------
.../apache}/stormcrawler/bolt/JSoupParserBolt.java | 48 +++++++++++-----------
.../stormcrawler/bolt/SimpleFetcherBolt.java | 42 +++++++++----------
.../stormcrawler/bolt/SiteMapParserBolt.java | 28 ++++++-------
.../stormcrawler/bolt/StatusEmitterBolt.java | 26 ++++++------
.../apache}/stormcrawler/bolt/URLFilterBolt.java | 12 +++---
.../stormcrawler/bolt/URLPartitionerBolt.java | 8 ++--
.../apache}/stormcrawler/filtering/URLFilter.java | 6 +--
.../apache}/stormcrawler/filtering/URLFilters.java | 10 ++---
.../filtering/basic/BasicURLFilter.java | 6 +--
.../filtering/basic/BasicURLNormalizer.java | 6 +--
.../filtering/basic/SelfURLFilter.java | 6 +--
.../filtering/depth/MaxDepthFilter.java | 8 ++--
.../stormcrawler/filtering/host/HostURLFilter.java | 6 +--
.../filtering/metadata/MetadataFilter.java | 6 +--
.../filtering/regex/FastURLFilter.java | 10 ++---
.../stormcrawler/filtering/regex/RegexRule.java | 2 +-
.../filtering/regex/RegexURLFilter.java | 2 +-
.../filtering/regex/RegexURLFilterBase.java | 6 +--
.../filtering/regex/RegexURLNormalizer.java | 6 +--
.../filtering/robots/RobotsFilter.java | 14 +++----
.../filtering/sitemap/SitemapFilter.java | 10 ++---
.../stormcrawler/indexing/AbstractIndexerBolt.java | 12 +++---
.../stormcrawler/indexing/DummyIndexer.java | 8 ++--
.../stormcrawler/indexing/StdOutIndexer.java | 8 ++--
.../stormcrawler/jsoup/LDJsonParseFilter.java | 12 +++---
.../stormcrawler/jsoup/LinkParseFilter.java | 20 ++++-----
.../apache}/stormcrawler/jsoup/XPathFilter.java | 12 +++---
.../parse/DocumentFragmentBuilder.java | 2 +-
.../apache}/stormcrawler/parse/JSoupFilter.java | 7 ++--
.../apache}/stormcrawler/parse/JSoupFilters.java | 10 ++---
.../apache}/stormcrawler/parse/Outlink.java | 4 +-
.../apache}/stormcrawler/parse/ParseData.java | 4 +-
.../apache}/stormcrawler/parse/ParseFilter.java | 8 ++--
.../apache}/stormcrawler/parse/ParseFilters.java | 8 ++--
.../apache}/stormcrawler/parse/ParseResult.java | 4 +-
.../apache}/stormcrawler/parse/TextExtractor.java | 4 +-
.../parse/filter/CollectionTagger.java | 10 ++---
.../CommaSeparatedToMultivaluedMetadata.java | 8 ++--
.../parse/filter/DebugParseFilter.java | 6 +--
.../parse/filter/DomainParseFilter.java | 12 +++---
.../parse/filter/LDJsonParseFilter.java | 10 ++---
.../stormcrawler/parse/filter/LinkParseFilter.java | 20 ++++-----
.../parse/filter/MD5SignatureParseFilter.java | 10 ++---
.../parse/filter/MimeTypeNormalization.java | 8 ++--
.../stormcrawler/parse/filter/XPathFilter.java | 10 ++---
.../persistence/AbstractQueryingSpout.java | 8 ++--
.../persistence/AbstractStatusUpdaterBolt.java | 10 ++---
.../persistence/AdaptiveScheduler.java | 22 +++++-----
.../stormcrawler/persistence/DefaultScheduler.java | 14 +++----
.../persistence/EmptyQueueListener.java | 2 +-
.../persistence/MemoryStatusUpdater.java | 6 +--
.../stormcrawler/persistence/Scheduler.java | 8 ++--
.../apache}/stormcrawler/persistence/Status.java | 2 +-
.../persistence/StdOutStatusUpdater.java | 4 +-
.../persistence/urlbuffer/AbstractURLBuffer.java | 8 ++--
.../persistence/urlbuffer/PriorityURLBuffer.java | 4 +-
.../persistence/urlbuffer/SchedulingURLBuffer.java | 4 +-
.../persistence/urlbuffer/SimpleURLBuffer.java | 2 +-
.../persistence/urlbuffer/URLBuffer.java | 12 +++---
.../protocol/AbstractHttpProtocol.java | 12 +++---
.../stormcrawler/protocol/DelegatorProtocol.java | 12 +++---
.../apache}/stormcrawler/protocol/HttpHeaders.java | 2 +-
.../protocol/HttpRobotRulesParser.java | 8 ++--
.../apache}/stormcrawler/protocol/Protocol.java | 8 ++--
.../stormcrawler/protocol/ProtocolFactory.java | 10 ++---
.../stormcrawler/protocol/ProtocolResponse.java | 4 +-
.../apache}/stormcrawler/protocol/RobotRules.java | 2 +-
.../stormcrawler/protocol/RobotRulesParser.java | 4 +-
.../stormcrawler/protocol/file/FileProtocol.java | 12 +++---
.../stormcrawler/protocol/file/FileResponse.java | 8 ++--
.../protocol/httpclient/HttpProtocol.java | 24 +++++------
.../protocol/okhttp/DNSResolutionListener.java | 2 +-
.../stormcrawler/protocol/okhttp/HttpProtocol.java | 24 +++++------
.../protocol/selenium/NavigationFilter.java | 8 ++--
.../protocol/selenium/NavigationFilters.java | 12 +++---
.../protocol/selenium/RemoteDriverProtocol.java | 6 +--
.../protocol/selenium/SeleniumProtocol.java | 10 ++---
.../stormcrawler/proxy/MultiProxyManager.java | 8 ++--
.../apache}/stormcrawler/proxy/ProxyManager.java | 4 +-
.../apache}/stormcrawler/proxy/SCProxy.java | 2 +-
.../stormcrawler/proxy/SingleProxyManager.java | 6 +--
.../apache}/stormcrawler/spout/FileSpout.java | 8 ++--
.../apache}/stormcrawler/spout/MemorySpout.java | 10 ++---
.../stormcrawler/util/AbstractConfigurable.java | 2 +-
.../stormcrawler/util/CharsetIdentification.java | 6 +--
.../stormcrawler/util/CollectionMetric.java | 2 +-
.../apache}/stormcrawler/util/ConfUtils.java | 2 +-
.../apache}/stormcrawler/util/Configurable.java | 2 +-
.../stormcrawler/util/ConfigurableHelper.java | 2 +-
.../apache}/stormcrawler/util/CookieConverter.java | 2 +-
.../stormcrawler/util/InitialisationUtil.java | 2 +-
.../stormcrawler/util/MetadataTransfer.java | 6 +--
.../stormcrawler/util/PerSecondReducer.java | 2 +-
.../apache}/stormcrawler/util/RefreshTag.java | 2 +-
.../apache}/stormcrawler/util/RobotsTags.java | 4 +-
.../apache}/stormcrawler/util/StringTabScheme.java | 4 +-
.../apache}/stormcrawler/util/URLPartitioner.java | 6 +--
.../stormcrawler/util/URLStreamGrouping.java | 8 ++--
.../apache}/stormcrawler/util/URLUtil.java | 2 +-
core/src/main/resources/crawler-default.yaml | 10 ++---
.../apache}/stormcrawler/MetadataTest.java | 2 +-
.../stormcrawler/TestMetadataSerialization.java | 2 +-
.../apache}/stormcrawler/TestOutputCollector.java | 2 +-
.../apache}/stormcrawler/TestUtil.java | 2 +-
.../stormcrawler/bolt/AbstractFetcherBoltTest.java | 8 ++--
.../stormcrawler/bolt/FeedParserBoltTest.java | 16 ++++----
.../apache}/stormcrawler/bolt/FetcherBoltTest.java | 2 +-
.../stormcrawler/bolt/JSoupParserBoltTest.java | 16 ++++----
.../stormcrawler/bolt/SimpleFetcherBoltTest.java | 2 +-
.../stormcrawler/bolt/SiteMapParserBoltTest.java | 12 +++---
.../stormcrawler/filtering/BasicURLFilterTest.java | 6 +--
.../filtering/BasicURLNormalizerTest.java | 6 +--
.../stormcrawler/filtering/FastURLFilterTest.java | 6 +--
.../stormcrawler/filtering/HostURLFilterTest.java | 6 +--
.../stormcrawler/filtering/MaxDepthFilterTest.java | 8 ++--
.../stormcrawler/filtering/MetadataFilterTest.java | 6 +--
.../stormcrawler/filtering/RegexFilterTest.java | 6 +--
.../ClassInheritingFomAbstractAndInterface.java | 6 +--
.../ClassInheritingFromAbstractClassOnly.java | 4 +-
.../ClassInheritingFromOpenClass.java | 4 +-
.../ClassWithoutValidConstructor.java | 4 +-
.../initialisation/FinalClassToInitialize.java | 2 +-
.../helper/initialisation/SimpleOpenClass.java | 2 +-
.../helper/initialisation/base/AbstractClass.java | 2 +-
.../helper/initialisation/base/ITestInterface.java | 2 +-
.../OpenClassWithAbstractClassAndInterface.java | 2 +-
.../stormcrawler/indexer/BasicIndexingTest.java | 6 +--
.../apache}/stormcrawler/indexer/DummyIndexer.java | 6 +--
.../stormcrawler/indexer/IndexerTester.java | 10 ++---
.../apache}/stormcrawler/json/JsoupFilterTest.java | 10 ++---
.../stormcrawler/jsoup/JSoupFiltersTest.java | 10 ++---
.../stormcrawler/parse/DuplicateLinksTest.java | 10 ++---
.../apache}/stormcrawler/parse/ParsingTester.java | 8 ++--
.../stormcrawler/parse/StackOverflowTest.java | 8 ++--
.../stormcrawler/parse/TextExtractorTest.java | 2 +-
.../parse/filter/CSVMetadataFilterTest.java | 8 ++--
.../parse/filter/CollectionTaggerTest.java | 4 +-
.../parse/filter/SubDocumentsFilterTest.java | 8 ++--
.../parse/filter/SubDocumentsParseFilter.java | 8 ++--
.../stormcrawler/parse/filter/XPathFilterTest.java | 8 ++--
.../persistence/AdaptiveSchedulerTest.java | 6 +--
.../persistence/DefaultSchedulerTest.java | 4 +-
.../stormcrawler/persistence/URLBufferTest.java | 10 ++---
.../protocol/AbstractProtocolTest.java | 2 +-
.../protocol/DelegationProtocolTest.java | 14 +++----
.../stormcrawler/protocol/DummyProtocol.java | 4 +-
.../stormcrawler/protocol/HttpHeadersTest.java | 2 +-
.../protocol/HttpRobotRulesParserTest.java | 2 +-
.../protocol/selenium/ProtocolTest.java | 8 ++--
.../stormcrawler/proxy/MultiProxyManagerTest.java | 2 +-
.../apache}/stormcrawler/proxy/SCProxyTest.java | 2 +-
.../stormcrawler/proxy/SingleProxyManagerTest.java | 2 +-
.../apache}/stormcrawler/util/ConfUtilsTest.java | 2 +-
.../stormcrawler/util/CookieConverterTest.java | 2 +-
.../stormcrawler/util/InitialisationUtilTest.java | 6 +--
.../stormcrawler/util/MetadataTransferTest.java | 4 +-
.../apache}/stormcrawler/util/RefreshTagTest.java | 2 +-
.../apache}/stormcrawler/util/RobotsTagsTest.java | 4 +-
core/src/test/resources/basicurlnormalizer.json | 4 +-
core/src/test/resources/delegator-conf.yaml | 10 ++---
core/src/test/resources/test.jsoupfilters.json | 8 ++--
core/src/test/resources/test.parsefilters.json | 8 ++--
core/src/test/resources/test.subdocfilter.json | 6 +--
external/aws/README.md | 2 +-
external/aws/pom.xml | 4 +-
.../aws/bolt/CloudSearchConstants.java | 2 +-
.../aws/bolt/CloudSearchIndexerBolt.java | 12 +++---
.../stormcrawler/aws/bolt/CloudSearchUtils.java | 2 +-
.../stormcrawler/aws/s3/AbstractS3CacheBolt.java | 4 +-
.../stormcrawler/aws/s3/S3CacheChecker.java | 6 +--
.../apache}/stormcrawler/aws/s3/S3Cacher.java | 6 +--
.../stormcrawler/aws/s3/S3ContentCacher.java | 4 +-
external/elasticsearch/README.md | 12 +++---
external/elasticsearch/archetype/pom.xml | 2 +-
.../archetype-resources/crawler-conf.yaml | 8 ++--
.../resources/archetype-resources/es-conf.yaml | 2 +-
.../resources/archetype-resources/es-crawler.flux | 22 +++++-----
.../archetype-resources/es-injection.flux | 8 ++--
.../src/main/resources/archetype-resources/pom.xml | 6 +--
.../src/main/java/ESCrawlTopology.java | 36 ++++++++--------
.../src/main/resources/jsoupfilters.json | 6 +--
.../src/main/resources/parsefilters.json | 8 ++--
.../src/main/resources/urlfilters.json | 18 ++++----
external/elasticsearch/pom.xml | 6 +--
.../BulkItemResponseToFailedFlag.java | 2 +-
.../elasticsearch/ElasticSearchConnection.java | 4 +-
.../elasticsearch/bolt/DeletionBolt.java | 8 ++--
.../elasticsearch/bolt/IndexerBolt.java | 20 ++++-----
.../filtering/JSONURLFilterWrapper.java | 14 +++----
.../elasticsearch/metrics/MetricsConsumer.java | 8 ++--
.../elasticsearch/metrics/StatusMetricsBolt.java | 6 +--
.../parse/filter/JSONResourceWrapper.java | 14 +++----
.../elasticsearch/persistence/AbstractSpout.java | 10 ++---
.../persistence/AggregationSpout.java | 6 +--
.../elasticsearch/persistence/CollapsingSpout.java | 4 +-
.../elasticsearch/persistence/HybridSpout.java | 6 +--
.../elasticsearch/persistence/ScrollSpout.java | 10 ++---
.../persistence/StatusUpdaterBolt.java | 20 ++++-----
.../elasticsearch/bolt/IndexerBoltTest.java | 12 +++---
.../elasticsearch/bolt/StatusBoltTest.java | 14 +++----
external/langid/pom.xml | 4 +-
.../stormcrawler/parse/filter/LanguageID.java | 12 +++---
external/opensearch/README.md | 12 +++---
external/opensearch/archetype/pom.xml | 2 +-
.../archetype-resources/crawler-conf.yaml | 8 ++--
.../resources/archetype-resources/crawler.flux | 22 +++++-----
.../resources/archetype-resources/injection.flux | 8 ++--
.../archetype-resources/opensearch-conf.yaml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 6 +--
.../src/main/resources/jsoupfilters.json | 6 +--
.../src/main/resources/parsefilters.json | 8 ++--
.../src/main/resources/urlfilters.json | 18 ++++----
external/opensearch/opensearch-conf.yaml | 2 +-
external/opensearch/pom.xml | 6 +--
.../opensearch/BulkItemResponseToFailedFlag.java | 2 +-
.../apache}/stormcrawler/opensearch/Constants.java | 2 +-
.../stormcrawler/opensearch/IndexCreation.java | 2 +-
.../opensearch/OpenSearchConnection.java | 4 +-
.../stormcrawler/opensearch/bolt/DeletionBolt.java | 10 ++---
.../stormcrawler/opensearch/bolt/IndexerBolt.java | 34 +++++++--------
.../opensearch/filtering/JSONURLFilterWrapper.java | 14 +++----
.../opensearch/metrics/MetricsConsumer.java | 10 ++---
.../opensearch/metrics/StatusMetricsBolt.java | 8 ++--
.../parse/filter/JSONResourceWrapper.java | 14 +++----
.../opensearch/persistence/AbstractSpout.java | 16 ++++----
.../opensearch/persistence/AggregationSpout.java | 8 ++--
.../opensearch/persistence/HybridSpout.java | 8 ++--
.../opensearch/persistence/StatusUpdaterBolt.java | 24 +++++------
.../opensearch/bolt/AbstractOpenSearchTest.java | 2 +-
.../opensearch/bolt/IndexerBoltTest.java | 12 +++---
.../opensearch/bolt/StatusBoltTest.java | 14 +++----
external/pom.xml | 4 +-
external/solr/README.md | 2 +-
external/solr/pom.xml | 6 +--
external/solr/solr-conf.yaml | 2 +-
.../apache}/stormcrawler/solr/SeedInjector.java | 10 ++---
.../apache}/stormcrawler/solr/SolrConnection.java | 4 +-
.../stormcrawler/solr/SolrCrawlTopology.java | 24 +++++------
.../stormcrawler/solr/bolt/DeletionBolt.java | 4 +-
.../stormcrawler/solr/bolt/IndexerBolt.java | 12 +++---
.../stormcrawler/solr/metrics/MetricsConsumer.java | 6 +--
.../stormcrawler/solr/persistence/SolrSpout.java | 10 ++---
.../solr/persistence/StatusUpdaterBolt.java | 16 ++++----
external/sql/pom.xml | 4 +-
external/sql/sql-conf.yaml | 2 +-
.../apache}/stormcrawler/sql/Constants.java | 2 +-
.../apache}/stormcrawler/sql/IndexerBolt.java | 12 +++---
.../apache}/stormcrawler/sql/SQLSpout.java | 10 ++---
.../apache}/stormcrawler/sql/SQLUtil.java | 2 +-
.../stormcrawler/sql/StatusUpdaterBolt.java | 12 +++---
.../stormcrawler/sql/metrics/MetricsConsumer.java | 8 ++--
external/tika/README.md | 4 +-
external/tika/pom.xml | 6 +--
.../apache}/stormcrawler/tika/DOMBuilder.java | 2 +-
.../apache}/stormcrawler/tika/ParserBolt.java | 38 ++++++++---------
.../apache}/stormcrawler/tika/RedirectionBolt.java | 4 +-
.../stormcrawler/tika/XMLCharacterRecognizer.java | 2 +-
.../apache}/stormcrawler/tika/ParserBoltTest.java | 16 ++++----
external/urlfrontier/README.md | 2 +-
external/urlfrontier/pom.xml | 6 +--
.../stormcrawler/urlfrontier/Constants.java | 2 +-
.../urlfrontier/ManagedChannelUtil.java | 4 +-
.../apache}/stormcrawler/urlfrontier/Spout.java | 10 ++---
.../urlfrontier/StatusUpdaterBolt.java | 14 +++----
.../urlfrontier/StatusUpdaterBoltTest.java | 16 ++++----
.../urlfrontier/URLFrontierContainer.java | 2 +-
.../urlfrontier/URLFrontierContainerConfig.java | 2 +-
external/warc/README.md | 12 +++---
external/warc/pom.xml | 8 ++--
.../warc/FileTimeSizeRotationPolicy.java | 2 +-
.../apache}/stormcrawler/warc/GzipHdfsBolt.java | 2 +-
.../stormcrawler/warc/WARCFileNameFormat.java | 2 +-
.../apache}/stormcrawler/warc/WARCHdfsBolt.java | 6 +--
.../stormcrawler/warc/WARCRecordFormat.java | 18 ++++----
.../stormcrawler/warc/WARCRequestRecordFormat.java | 8 ++--
.../apache}/stormcrawler/warc/WARCSpout.java | 18 ++++----
.../stormcrawler/warc/WARCHdfsBoltTest.java | 10 ++---
.../stormcrawler/warc/WARCRecordFormatTest.java | 8 ++--
.../apache}/stormcrawler/warc/WARCSpoutTest.java | 6 +--
pom.xml | 36 +++++++---------
297 files changed, 1215 insertions(+), 1241 deletions(-)
diff --git a/README.md b/README.md
index 2339aa73..183a05b8 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,10 @@
[](http://stormcrawler.net/)
=============
-[](http://www.apache.org/licenses/LICENSE-2.0)
-
-[](https://javadoc.io/doc/com.digitalpebble.stormcrawler/storm-crawler-core/)
-[](https://coveralls.io/github/DigitalPebble/storm-crawler?branch=master)
+[](http://www.apache.org/licenses/LICENSE-2.0)
+
+[](https://javadoc.io/doc/org.apache.stormcrawler/storm-crawler-core/)
+[](https://coveralls.io/github/apache/incubator-stormcrawler?branch=master)
StormCrawler is an open source collection of resources for building low-latency, scalable web crawlers on [Apache Storm](http://storm.apache.org/). It is provided under [Apache License](http://www.apache.org/licenses/LICENSE-2.0) and is written mostly in Java.
@@ -14,7 +14,7 @@ NOTE: These instructions assume that you have [Apache Maven](https://maven.apach
StormCrawler requires Java 11 or above.
-The version of Apache Storm to install must match the one defined in the pom.xml file of your topology. The major version of StormCrawler mirrors the one from Apache Storm, i.e whereas StormCrawler 1.x used Storm 1.2.3, the current version now requires Storm 2.6.1. Our [Ansible-Storm](https://github.com/DigitalPebble/ansible-storm) repository contains resources to install Apache Storm using Ansible. Alternatively, our [stormCrawler-docker](https://github.com/DigitalPebble/stormcrawler-do [...]
+The version of Apache Storm to install must match the one defined in the pom.xml file of your topology. The major version of StormCrawler mirrors the one from Apache Storm, i.e whereas StormCrawler 1.x used Storm 1.2.3, the current version now requires Storm 2.6.1. DigitalPebble's [Ansible-Storm](https://github.com/DigitalPebble/ansible-storm) repository contains resources to install Apache Storm using Ansible. Alternatively, this [stormCrawler-docker](https://github.com/DigitalPebble/st [...]
Once Storm is installed, the easiest way to get started is to generate a brand new StormCrawler project using \:
@@ -24,13 +24,13 @@ You'll be asked to enter a groupId (e.g. com.mycompany.crawler), an artefactId (
This will not only create a fully formed project containing a POM with the dependency above but also the default resource files, a default CrawlTopology class and a configuration file. Enter the directory you just created (should be the same as the artefactId you specified earlier) and follow the instructions on the README file.
-Alternatively if you can't or don't want to use the Maven archetype above, you can simply copy the files from [archetype-resources](https://github.com/DigitalPebble/storm-crawler/tree/master/archetype/src/main/resources/archetype-resources).
+Alternatively if you can't or don't want to use the Maven archetype above, you can simply copy the files from [archetype-resources](https://github.com/apache/incubator-stormcrawler/tree/master/archetype/src/main/resources/archetype-resources).
-Have a look at the code of the [CrawlTopology class](https://github.com/DigitalPebble/storm-crawler/blob/master/archetype/src/main/resources/archetype-resources/src/main/java/CrawlTopology.java), the [crawler-conf.yaml](https://github.com/DigitalPebble/storm-crawler/blob/master/archetype/src/main/resources/archetype-resources/crawler-conf.yaml) file as well as the files in [src/main/resources/](https://github.com/DigitalPebble/storm-crawler/tree/master/archetype/src/main/resources/archet [...]
+Have a look at the code of the [CrawlTopology class](https://github.com/apache/incubator-stormcrawler/blob/master/archetype/src/main/resources/archetype-resources/src/main/java/CrawlTopology.java), the [crawler-conf.yaml](https://github.com/apache/incubator-stormcrawler/blob/master/archetype/src/main/resources/archetype-resources/crawler-conf.yaml) file as well as the files in [src/main/resources/](https://github.com/apache/incubator-stormcrawler/tree/master/archetype/src/main/resources/ [...]
## Getting help
-The [WIKI](https://github.com/DigitalPebble/storm-crawler/wiki) is a good place to start your investigations but if you are stuck please use the tag [stormcrawler](http://stackoverflow.com/questions/tagged/stormcrawler) on StackOverflow or ask a question in the [discussions](https://github.com/DigitalPebble/storm-crawler/discussions) section. Alternatively, you can join our [Discord channel](https://discord.com/invite/C62MHusNnG).
+The [WIKI](https://github.com/apache/incubator-stormcrawler/wiki) is a good place to start your investigations but if you are stuck please use the tag [stormcrawler](http://stackoverflow.com/questions/tagged/stormcrawler) on StackOverflow or ask a question in the [discussions](https://github.com/apache/incubator-stormcrawler/discussions) section. Alternatively, you can join our [Discord channel](https://discord.com/invite/C62MHusNnG).
[DigitalPebble Ltd](http://digitalpebble.com) provide commercial support and consulting for StormCrawler.
diff --git a/THIRD-PARTY.txt b/THIRD-PARTY.txt
index 2102e77d..8f2ea7b5 100644
--- a/THIRD-PARTY.txt
+++ b/THIRD-PARTY.txt
@@ -211,8 +211,6 @@ List of third-party dependencies grouped by their license type.
* JVM Integration for Metrics (io.dropwizard.metrics:metrics-jvm:3.2.6 - http://metrics.dropwizard.io/metrics-jvm/)
* jwarc (org.netpreserve:jwarc:0.28.3 - https://github.com/iipc/jwarc)
* jwarc (org.netpreserve:jwarc:0.29.0 - https://github.com/iipc/jwarc)
- * kafka-avro-serializer (io.confluent:kafka-avro-serializer:1.0 - no url defined)
- * kafka-schema-registry-client (io.confluent:kafka-schema-registry-client:1.0 - no url defined)
* Kerb Simple Kdc (org.apache.kerby:kerb-simplekdc:1.0.1 - http://directory.apache.org/kerby/kerby-kerb/kerb-simplekdc)
* Kerby ASN1 Project (org.apache.kerby:kerby-asn1:1.0.1 - http://directory.apache.org/kerby/kerby-common/kerby-asn1)
* Kerby Config (org.apache.kerby:kerby-config:1.0.1 - http://directory.apache.org/kerby/kerby-common/kerby-config)
@@ -350,6 +348,7 @@ List of third-party dependencies grouped by their license type.
* SparseBitSet (com.zaxxer:SparseBitSet:1.2 - https://github.com/brettwooldridge/SparseBitSet)
* storm-autocreds (org.apache.storm:storm-autocreds:2.6.1 - https://storm.apache.org/external/storm-autocreds)
* Storm Client (org.apache.storm:storm-client:2.6.1 - https://storm.apache.org/storm-client)
+ * storm-crawler-core (com.digitalpebble.stormcrawler:storm-crawler-core:2.12-SNAPSHOT - https://github.com/DigitalPebble/storm-crawler/tree/master/core)
* storm-hdfs (org.apache.storm:storm-hdfs:2.6.1 - https://storm.apache.org/external/storm-hdfs)
* swagger-annotations-jakarta (io.swagger.core.v3:swagger-annotations-jakarta:2.2.17 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations-jakarta)
* TagSoup (org.ccil.cowan.tagsoup:tagsoup:1.2.1 - http://home.ccil.org/~cowan/XML/tagsoup/)
diff --git a/archetype/pom.xml b/archetype/pom.xml
index 570c771d..ab683fb1 100644
--- a/archetype/pom.xml
+++ b/archetype/pom.xml
@@ -1,7 +1,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler</artifactId>
<version>2.12-SNAPSHOT</version>
</parent>
diff --git a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml
index 2ce3e1b3..5e1f0bdd 100644
--- a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml
+++ b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml
@@ -17,8 +17,8 @@ config:
# mandatory when using Flux
topology.kryo.register:
- - com.digitalpebble.stormcrawler.Metadata
- - com.digitalpebble.stormcrawler.persistence.Status
+ - org.apache.stormcrawler.Metadata
+ - org.apache.stormcrawler.persistence.Status
# Lists the metadata to transfer to outlinks
# Used by Fetcher and SiteMapParser for redirections,
@@ -54,8 +54,8 @@ config:
# Finally, an email so that they can get in touch with you
http.agent.email: "${http-agent-email}"
- http.protocol.implementation: "com.digitalpebble.stormcrawler.protocol.okhttp.HttpProtocol"
- https.protocol.implementation: "com.digitalpebble.stormcrawler.protocol.okhttp.HttpProtocol"
+ http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
+ https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
# The maximum number of bytes for returned HTTP response bodies.
# The fetched page will be trimmed to 65KB in this case
diff --git a/archetype/src/main/resources/archetype-resources/crawler.flux b/archetype/src/main/resources/archetype-resources/crawler.flux
index 2b4f2a02..f9a301e6 100644
--- a/archetype/src/main/resources/archetype-resources/crawler.flux
+++ b/archetype/src/main/resources/archetype-resources/crawler.flux
@@ -11,36 +11,36 @@ includes:
spouts:
- id: "spout"
- className: "com.digitalpebble.stormcrawler.urlfrontier.Spout"
+ className: "org.apache.stormcrawler.urlfrontier.Spout"
parallelism: 1
bolts:
- id: "partitioner"
- className: "com.digitalpebble.stormcrawler.bolt.URLPartitionerBolt"
+ className: "org.apache.stormcrawler.bolt.URLPartitionerBolt"
parallelism: 1
- id: "fetcher"
- className: "com.digitalpebble.stormcrawler.bolt.FetcherBolt"
+ className: "org.apache.stormcrawler.bolt.FetcherBolt"
parallelism: 1
- id: "sitemap"
- className: "com.digitalpebble.stormcrawler.bolt.SiteMapParserBolt"
+ className: "org.apache.stormcrawler.bolt.SiteMapParserBolt"
parallelism: 1
- id: "feed"
- className: "com.digitalpebble.stormcrawler.bolt.FeedParserBolt"
+ className: "org.apache.stormcrawler.bolt.FeedParserBolt"
parallelism: 1
- id: "parse"
- className: "com.digitalpebble.stormcrawler.bolt.JSoupParserBolt"
+ className: "org.apache.stormcrawler.bolt.JSoupParserBolt"
parallelism: 1
- id: "shunt"
- className: "com.digitalpebble.stormcrawler.tika.RedirectionBolt"
+ className: "org.apache.stormcrawler.tika.RedirectionBolt"
parallelism: 1
- id: "tika"
- className: "com.digitalpebble.stormcrawler.tika.ParserBolt"
+ className: "org.apache.stormcrawler.tika.ParserBolt"
parallelism: 1
- id: "index"
- className: "com.digitalpebble.stormcrawler.indexing.StdOutIndexer"
+ className: "org.apache.stormcrawler.indexing.StdOutIndexer"
parallelism: 1
- id: "status"
- className: "com.digitalpebble.stormcrawler.urlfrontier.StatusUpdaterBolt"
+ className: "org.apache.stormcrawler.urlfrontier.StatusUpdaterBolt"
parallelism: 1
streams:
diff --git a/archetype/src/main/resources/archetype-resources/pom.xml b/archetype/src/main/resources/archetype-resources/pom.xml
index 43ed2b31..22694441 100644
--- a/archetype/src/main/resources/archetype-resources/pom.xml
+++ b/archetype/src/main/resources/archetype-resources/pom.xml
@@ -97,7 +97,7 @@
<dependencies>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-core</artifactId>
<version>${stormcrawler.version}</version>
</dependency>
@@ -113,12 +113,12 @@
<version>${storm.version}</version>
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-tika</artifactId>
<version>${stormcrawler.version}</version>
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-urlfrontier</artifactId>
<version>${stormcrawler.version}</version>
</dependency>
diff --git a/archetype/src/main/resources/archetype-resources/src/main/java/CrawlTopology.java b/archetype/src/main/resources/archetype-resources/src/main/java/CrawlTopology.java
index e5948765..44368c0e 100644
--- a/archetype/src/main/resources/archetype-resources/src/main/java/CrawlTopology.java
+++ b/archetype/src/main/resources/archetype-resources/src/main/java/CrawlTopology.java
@@ -17,18 +17,18 @@ package ${package};
* limitations under the License.
*/
-import com.digitalpebble.stormcrawler.*;
-import com.digitalpebble.stormcrawler.bolt.FetcherBolt;
-import com.digitalpebble.stormcrawler.bolt.JSoupParserBolt;
-import com.digitalpebble.stormcrawler.bolt.SiteMapParserBolt;
-import com.digitalpebble.stormcrawler.bolt.URLPartitionerBolt;
-import com.digitalpebble.stormcrawler.bolt.FeedParserBolt;
-import com.digitalpebble.stormcrawler.indexing.StdOutIndexer;
-import com.digitalpebble.stormcrawler.persistence.StdOutStatusUpdater;
-import com.digitalpebble.stormcrawler.urlfrontier.Spout;
-import com.digitalpebble.stormcrawler.urlfrontier.StatusUpdaterBolt;
-import com.digitalpebble.stormcrawler.tika.ParserBolt;
-import com.digitalpebble.stormcrawler.tika.RedirectionBolt;
+import org.apache.stormcrawler.*;
+import org.apache.stormcrawler.bolt.FetcherBolt;
+import org.apache.stormcrawler.bolt.JSoupParserBolt;
+import org.apache.stormcrawler.bolt.SiteMapParserBolt;
+import org.apache.stormcrawler.bolt.URLPartitionerBolt;
+import org.apache.stormcrawler.bolt.FeedParserBolt;
+import org.apache.stormcrawler.indexing.StdOutIndexer;
+import org.apache.stormcrawler.persistence.StdOutStatusUpdater;
+import org.apache.stormcrawler.urlfrontier.Spout;
+import org.apache.stormcrawler.urlfrontier.StatusUpdaterBolt;
+import org.apache.stormcrawler.tika.ParserBolt;
+import org.apache.stormcrawler.tika.RedirectionBolt;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
diff --git a/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json b/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json
index c48b4134..4d87d8d5 100644
--- a/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json
+++ b/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json
@@ -1,7 +1,7 @@
{
- "com.digitalpebble.stormcrawler.parse.JSoupFilters": [
+ "org.apache.stormcrawler.parse.JSoupFilters": [
{
- "class": "com.digitalpebble.stormcrawler.jsoup.XPathFilter",
+ "class": "org.apache.stormcrawler.jsoup.XPathFilter",
"name": "XPathFilter",
"params": {
"canonical": "//*[@rel=\"canonical\"]/@href",
@@ -17,7 +17,7 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.jsoup.LinkParseFilter",
+ "class": "org.apache.stormcrawler.jsoup.LinkParseFilter",
"name": "LinkParseFilter",
"params": {
"pattern": "//FRAME/@src"
diff --git a/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json b/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json
index 04eaa895..5d525830 100644
--- a/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json
+++ b/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json
@@ -1,7 +1,7 @@
{
- "com.digitalpebble.stormcrawler.parse.ParseFilters": [
+ "org.apache.stormcrawler.parse.ParseFilters": [
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.DomainParseFilter",
+ "class": "org.apache.stormcrawler.parse.filter.DomainParseFilter",
"name": "DomainParseFilter",
"params": {
"key": "domain",
@@ -9,11 +9,11 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.MimeTypeNormalization",
+ "class": "org.apache.stormcrawler.parse.filter.MimeTypeNormalization",
"name": "MimeTypeNormalization"
},
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.CommaSeparatedToMultivaluedMetadata",
+ "class": "org.apache.stormcrawler.parse.filter.CommaSeparatedToMultivaluedMetadata",
"name": "CommaSeparatedToMultivaluedMetadata",
"params": {
"keys": ["parse.keywords"]
diff --git a/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json b/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json
index ec6740ef..6098631b 100644
--- a/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json
+++ b/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json
@@ -1,7 +1,7 @@
{
- "com.digitalpebble.stormcrawler.filtering.URLFilters": [
+ "org.apache.stormcrawler.filtering.URLFilters": [
{
- "class": "com.digitalpebble.stormcrawler.filtering.basic.BasicURLFilter",
+ "class": "org.apache.stormcrawler.filtering.basic.BasicURLFilter",
"name": "BasicURLFilter",
"params": {
"maxPathRepetition": 3,
@@ -9,14 +9,14 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.depth.MaxDepthFilter",
+ "class": "org.apache.stormcrawler.filtering.depth.MaxDepthFilter",
"name": "MaxDepthFilter",
"params": {
"maxDepth": -1
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.basic.BasicURLNormalizer",
+ "class": "org.apache.stormcrawler.filtering.basic.BasicURLNormalizer",
"name": "BasicURLNormalizer",
"params": {
"removeAnchorPart": true,
@@ -27,7 +27,7 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.host.HostURLFilter",
+ "class": "org.apache.stormcrawler.filtering.host.HostURLFilter",
"name": "HostURLFilter",
"params": {
"ignoreOutsideHost": false,
@@ -35,25 +35,25 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.regex.RegexURLNormalizer",
+ "class": "org.apache.stormcrawler.filtering.regex.RegexURLNormalizer",
"name": "RegexURLNormalizer",
"params": {
"regexNormalizerFile": "default-regex-normalizers.xml"
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.regex.RegexURLFilter",
+ "class": "org.apache.stormcrawler.filtering.regex.RegexURLFilter",
"name": "RegexURLFilter",
"params": {
"regexFilterFile": "default-regex-filters.txt"
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.basic.SelfURLFilter",
+ "class": "org.apache.stormcrawler.filtering.basic.SelfURLFilter",
"name": "SelfURLFilter"
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.sitemap.SitemapFilter",
+ "class": "org.apache.stormcrawler.filtering.sitemap.SitemapFilter",
"name": "SitemapFilter"
}
]
diff --git a/core/pom.xml b/core/pom.xml
index 6cf7f45d..d5050bb1 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler</artifactId>
<version>2.12-SNAPSHOT</version>
</parent>
@@ -12,8 +12,8 @@
<packaging>jar</packaging>
<name>storm-crawler-core</name>
- <url>https://github.com/DigitalPebble/storm-crawler/tree/master/core</url>
- <description>Storm-Crawler core Java API.</description>
+ <url>https://github.com/apache/incubator-stormcrawler/storm-crawler/tree/master/core</url>
+ <description>StormCrawler core Java API.</description>
<properties>
<!-- dependency versions -->
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/ConfigurableTopology.java b/core/src/main/java/org/apache/stormcrawler/ConfigurableTopology.java
similarity index 95%
rename from core/src/main/java/com/digitalpebble/stormcrawler/ConfigurableTopology.java
rename to core/src/main/java/org/apache/stormcrawler/ConfigurableTopology.java
index 78db483f..50a3fb76 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/ConfigurableTopology.java
+++ b/core/src/main/java/org/apache/stormcrawler/ConfigurableTopology.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler;
+package org.apache.stormcrawler;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Collections;
@@ -27,6 +25,8 @@ import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.ConfUtils;
public abstract class ConfigurableTopology {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/Constants.java b/core/src/main/java/org/apache/stormcrawler/Constants.java
similarity index 98%
rename from core/src/main/java/com/digitalpebble/stormcrawler/Constants.java
rename to core/src/main/java/org/apache/stormcrawler/Constants.java
index 21d4f790..ef47a149 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/Constants.java
+++ b/core/src/main/java/org/apache/stormcrawler/Constants.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler;
+package org.apache.stormcrawler;
public class Constants {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/JSONResource.java b/core/src/main/java/org/apache/stormcrawler/JSONResource.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/JSONResource.java
rename to core/src/main/java/org/apache/stormcrawler/JSONResource.java
index 2a85f9cb..3c60b900 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/JSONResource.java
+++ b/core/src/main/java/org/apache/stormcrawler/JSONResource.java
@@ -1,4 +1,4 @@
-package com.digitalpebble.stormcrawler;
+package org.apache.stormcrawler;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/Metadata.java b/core/src/main/java/org/apache/stormcrawler/Metadata.java
similarity index 99%
rename from core/src/main/java/com/digitalpebble/stormcrawler/Metadata.java
rename to core/src/main/java/org/apache/stormcrawler/Metadata.java
index bb616fb2..2996dbdd 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/Metadata.java
+++ b/core/src/main/java/org/apache/stormcrawler/Metadata.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler;
+package org.apache.stormcrawler;
import com.esotericsoftware.kryo.serializers.DefaultArraySerializers.StringArraySerializer;
import com.esotericsoftware.kryo.serializers.DefaultSerializers.StringSerializer;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/FeedParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java
similarity index 93%
rename from core/src/main/java/com/digitalpebble/stormcrawler/bolt/FeedParserBolt.java
rename to core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java
index 35264712..1c9fb1b6 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/FeedParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java
@@ -12,20 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
-
-import static com.digitalpebble.stormcrawler.Constants.StatusStreamName;
-
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.Outlink;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseFilters;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
+package org.apache.stormcrawler.bolt;
+
+import static org.apache.stormcrawler.Constants.StatusStreamName;
+
import com.google.common.primitives.Bytes;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEntry;
@@ -46,6 +36,16 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.Outlink;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseFilters;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/FetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/bolt/FetcherBolt.java
rename to core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
index 299d7475..3f66e120 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/FetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
@@ -12,18 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
-
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.Protocol;
-import com.digitalpebble.stormcrawler.protocol.ProtocolFactory;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
-import com.digitalpebble.stormcrawler.protocol.RobotRules;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.PerSecondReducer;
+package org.apache.stormcrawler.bolt;
+
import crawlercommons.domains.PaidLevelDomain;
import crawlercommons.robots.BaseRobotRules;
import java.io.File;
@@ -58,6 +48,16 @@ import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils;
import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.Protocol;
+import org.apache.stormcrawler.protocol.ProtocolFactory;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.protocol.RobotRules;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.PerSecondReducer;
import org.slf4j.LoggerFactory;
/**
@@ -578,7 +578,7 @@ public class FetcherBolt extends StatusEmitterBolt {
// pass the info about denied by robots
metadata.setValue(Constants.STATUS_ERROR_CAUSE, "robots.txt");
collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
fit.t,
new Values(fit.url, metadata, Status.ERROR));
// no need to wait next time as we won't request from
@@ -606,7 +606,7 @@ public class FetcherBolt extends StatusEmitterBolt {
// pass the info about crawl delay
metadata.setValue(Constants.STATUS_ERROR_CAUSE, "crawl_delay");
collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
fit.t,
new Values(fit.url, metadata, Status.ERROR));
// no need to wait next time as we won't request
@@ -973,7 +973,7 @@ public class FetcherBolt extends StatusEmitterBolt {
// Report to status stream and ack
metadata.setValue(Constants.STATUS_ERROR_CAUSE, "malformed URL");
collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
input,
new Values(urlString, metadata, Status.ERROR));
collector.ack(input);
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
similarity index 93%
rename from core/src/main/java/com/digitalpebble/stormcrawler/bolt/JSoupParserBolt.java
rename to core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
index 479d2787..905984c3 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/JSoupParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
@@ -12,29 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
-
-import static com.digitalpebble.stormcrawler.Constants.StatusStreamName;
-
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.DocumentFragmentBuilder;
-import com.digitalpebble.stormcrawler.parse.JSoupFilter;
-import com.digitalpebble.stormcrawler.parse.JSoupFilters;
-import com.digitalpebble.stormcrawler.parse.Outlink;
-import com.digitalpebble.stormcrawler.parse.ParseData;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseFilters;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
-import com.digitalpebble.stormcrawler.parse.TextExtractor;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
-import com.digitalpebble.stormcrawler.util.CharsetIdentification;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.RefreshTag;
-import com.digitalpebble.stormcrawler.util.RobotsTags;
-import com.digitalpebble.stormcrawler.util.URLUtil;
+package org.apache.stormcrawler.bolt;
+
+import static org.apache.stormcrawler.Constants.StatusStreamName;
+
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -54,6 +35,25 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.DocumentFragmentBuilder;
+import org.apache.stormcrawler.parse.JSoupFilter;
+import org.apache.stormcrawler.parse.JSoupFilters;
+import org.apache.stormcrawler.parse.Outlink;
+import org.apache.stormcrawler.parse.ParseData;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseFilters;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.parse.TextExtractor;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.util.CharsetIdentification;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.RefreshTag;
+import org.apache.stormcrawler.util.RobotsTags;
+import org.apache.stormcrawler.util.URLUtil;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.detect.Detector;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -347,7 +347,7 @@ public class JSoupParserBolt extends StatusEmitterBolt {
// Mark URL as redirected
collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
tuple,
new Values(url, metadata, Status.REDIRECTION));
collector.ack(tuple);
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/SimpleFetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
similarity index 94%
rename from core/src/main/java/com/digitalpebble/stormcrawler/bolt/SimpleFetcherBolt.java
rename to core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
index 1ba73b3e..929a2687 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/SimpleFetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
@@ -12,18 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
-
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.Protocol;
-import com.digitalpebble.stormcrawler.protocol.ProtocolFactory;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
-import com.digitalpebble.stormcrawler.protocol.RobotRules;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.PerSecondReducer;
+package org.apache.stormcrawler.bolt;
+
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import crawlercommons.domains.PaidLevelDomain;
@@ -50,6 +40,16 @@ import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.Protocol;
+import org.apache.stormcrawler.protocol.ProtocolFactory;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.protocol.RobotRules;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.PerSecondReducer;
import org.slf4j.LoggerFactory;
/**
@@ -266,7 +266,7 @@ public class SimpleFetcherBolt extends StatusEmitterBolt {
// Report to status stream and ack
metadata.setValue(Constants.STATUS_ERROR_CAUSE, "malformed URL");
collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
input,
new Values(urlString, metadata, Status.ERROR));
collector.ack(input);
@@ -339,7 +339,7 @@ public class SimpleFetcherBolt extends StatusEmitterBolt {
// Report to status stream and ack
collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
input,
new Values(urlString, metadata, Status.ERROR));
collector.ack(input);
@@ -397,7 +397,7 @@ public class SimpleFetcherBolt extends StatusEmitterBolt {
url);
metadata.setValue(Constants.STATUS_ERROR_CAUSE, "crawl_delay");
collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
input,
new Values(urlString, metadata, Status.ERROR));
collector.ack(input);
@@ -479,7 +479,7 @@ public class SimpleFetcherBolt extends StatusEmitterBolt {
// rescheduled
// but do not try to parse or index
collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
input,
values4status);
} else {
@@ -505,15 +505,11 @@ public class SimpleFetcherBolt extends StatusEmitterBolt {
}
// Mark URL as redirected
collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
- input,
- values4status);
+ org.apache.stormcrawler.Constants.StatusStreamName, input, values4status);
} else {
// Error
collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
- input,
- values4status);
+ org.apache.stormcrawler.Constants.StatusStreamName, input, values4status);
}
} catch (Exception exece) {
@@ -545,7 +541,7 @@ public class SimpleFetcherBolt extends StatusEmitterBolt {
metadata.setValue("fetch.exception", message);
collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
input,
new Values(urlString, metadata, Status.FETCH_ERROR));
}
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/SiteMapParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
similarity index 95%
rename from core/src/main/java/com/digitalpebble/stormcrawler/bolt/SiteMapParserBolt.java
rename to core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
index fc2e7137..62b23dee 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/SiteMapParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
@@ -12,20 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
-
-import static com.digitalpebble.stormcrawler.Constants.StatusStreamName;
-
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.Outlink;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseFilters;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
-import com.digitalpebble.stormcrawler.persistence.DefaultScheduler;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
+package org.apache.stormcrawler.bolt;
+
+import static org.apache.stormcrawler.Constants.StatusStreamName;
+
import com.google.common.primitives.Bytes;
import crawlercommons.sitemaps.AbstractSiteMap;
import crawlercommons.sitemaps.Namespace;
@@ -56,6 +46,16 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.Outlink;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseFilters;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.persistence.DefaultScheduler;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.LoggerFactory;
/**
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/StatusEmitterBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/StatusEmitterBolt.java
similarity index 83%
rename from core/src/main/java/com/digitalpebble/stormcrawler/bolt/StatusEmitterBolt.java
rename to core/src/main/java/org/apache/stormcrawler/bolt/StatusEmitterBolt.java
index 721915d6..5db24419 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/StatusEmitterBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/StatusEmitterBolt.java
@@ -12,15 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
-
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilters;
-import com.digitalpebble.stormcrawler.parse.Outlink;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.MetadataTransfer;
-import com.digitalpebble.stormcrawler.util.URLUtil;
+package org.apache.stormcrawler.bolt;
+
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Map;
@@ -31,6 +24,13 @@ import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilters;
+import org.apache.stormcrawler.parse.Outlink;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.MetadataTransfer;
+import org.apache.stormcrawler.util.URLUtil;
/**
* Provides common functionalities for Bolts which emit tuples to the status stream, e.g. Fetchers,
@@ -54,15 +54,13 @@ public abstract class StatusEmitterBolt extends BaseRichBolt {
metadataTransfer = MetadataTransfer.getInstance(stormConf);
allowRedirs =
ConfUtils.getBoolean(
- stormConf,
- com.digitalpebble.stormcrawler.Constants.AllowRedirParamName,
- true);
+ stormConf, org.apache.stormcrawler.Constants.AllowRedirParamName, true);
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declareStream(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
new Fields("url", "metadata", "status"));
}
@@ -77,7 +75,7 @@ public abstract class StatusEmitterBolt extends BaseRichBolt {
if (ol == null) return;
collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
t,
new Values(ol.getTargetURL(), ol.getMetadata(), Status.DISCOVERED));
}
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/URLFilterBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/URLFilterBolt.java
similarity index 91%
rename from core/src/main/java/com/digitalpebble/stormcrawler/bolt/URLFilterBolt.java
rename to core/src/main/java/org/apache/stormcrawler/bolt/URLFilterBolt.java
index eea5a165..9bd4cad4 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/URLFilterBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/URLFilterBolt.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
+package org.apache.stormcrawler.bolt;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilters;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.MetadataTransfer;
import java.io.IOException;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
@@ -29,6 +25,10 @@ import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilters;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.MetadataTransfer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,7 +44,7 @@ public class URLFilterBolt extends BaseRichBolt {
private final boolean discoveredOnly;
- private static final String _s = com.digitalpebble.stormcrawler.Constants.StatusStreamName;
+ private static final String _s = org.apache.stormcrawler.Constants.StatusStreamName;
/**
* Relies on the file defined in urlfilters.config.file and applied to all tuples regardless of
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/URLPartitionerBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/bolt/URLPartitionerBolt.java
rename to core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java
index b4b06b35..d2d86aad 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/bolt/URLPartitionerBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java
@@ -12,11 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
+package org.apache.stormcrawler.bolt;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import crawlercommons.domains.PaidLevelDomain;
import java.net.InetAddress;
import java.net.MalformedURLException;
@@ -33,6 +30,9 @@ import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/URLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/URLFilter.java
similarity index 91%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/URLFilter.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/URLFilter.java
index 0f420393..c4cc5977 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/URLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/URLFilter.java
@@ -12,11 +12,11 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering;
+package org.apache.stormcrawler.filtering;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.AbstractConfigurable;
import java.net.URL;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.AbstractConfigurable;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/URLFilters.java b/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java
similarity index 96%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/URLFilters.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java
index a89cf849..dd3f12cb 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/URLFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering;
+package org.apache.stormcrawler.filtering;
-import com.digitalpebble.stormcrawler.JSONResource;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.Configurable;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
@@ -35,6 +31,10 @@ import org.apache.commons.cli.ParseException;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.Config;
import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.JSONResource;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.Configurable;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/basic/BasicURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLFilter.java
similarity index 94%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/basic/BasicURLFilter.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLFilter.java
index 7f502c3a..a1a64477 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/basic/BasicURLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLFilter.java
@@ -12,14 +12,14 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.basic;
+package org.apache.stormcrawler.filtering.basic;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
import com.fasterxml.jackson.databind.JsonNode;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilter;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java
similarity index 98%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/basic/BasicURLNormalizer.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java
index fccec6c6..189fa062 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/basic/BasicURLNormalizer.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.basic;
+package org.apache.stormcrawler.filtering.basic;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import java.net.IDN;
@@ -36,6 +34,8 @@ import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilter;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/basic/SelfURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/SelfURLFilter.java
similarity index 90%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/basic/SelfURLFilter.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/basic/SelfURLFilter.java
index f62ad7ea..83ba55a1 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/basic/SelfURLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/SelfURLFilter.java
@@ -12,13 +12,13 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.basic;
+package org.apache.stormcrawler.filtering.basic;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
import com.fasterxml.jackson.databind.JsonNode;
import java.net.URL;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilter;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/depth/MaxDepthFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/depth/MaxDepthFilter.java
similarity index 92%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/depth/MaxDepthFilter.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/depth/MaxDepthFilter.java
index bde5c121..0c260587 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/depth/MaxDepthFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/depth/MaxDepthFilter.java
@@ -12,15 +12,15 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.depth;
+package org.apache.stormcrawler.filtering.depth;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
-import com.digitalpebble.stormcrawler.util.MetadataTransfer;
import com.fasterxml.jackson.databind.JsonNode;
import java.net.URL;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilter;
+import org.apache.stormcrawler.util.MetadataTransfer;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/host/HostURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
similarity index 96%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/host/HostURLFilter.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
index eb22efdd..944db1fe 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/host/HostURLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
@@ -12,15 +12,15 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.host;
+package org.apache.stormcrawler.filtering.host;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
import com.fasterxml.jackson.databind.JsonNode;
import crawlercommons.domains.PaidLevelDomain;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilter;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/metadata/MetadataFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/metadata/MetadataFilter.java
similarity index 94%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/metadata/MetadataFilter.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/metadata/MetadataFilter.java
index 8d9ab876..bfb64bef 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/metadata/MetadataFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/metadata/MetadataFilter.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.metadata;
+package org.apache.stormcrawler.filtering.metadata;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
import com.fasterxml.jackson.databind.JsonNode;
import java.net.URL;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilter;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/FastURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/FastURLFilter.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java
index 8d0f0bb5..366814d1 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/FastURLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.regex;
+package org.apache.stormcrawler.filtering.regex;
-import com.digitalpebble.stormcrawler.JSONResource;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
@@ -33,6 +29,10 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
+import org.apache.stormcrawler.JSONResource;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilter;
+import org.apache.stormcrawler.util.ConfUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/RegexRule.java b/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexRule.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/RegexRule.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexRule.java
index 28e73ad0..2c392ab5 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/RegexRule.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexRule.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.regex;
+package org.apache.stormcrawler.filtering.regex;
/** A generic regular expression rule. Borrowed from Apache Nutch 1.9. */
public abstract class RegexRule {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/RegexURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLFilter.java
similarity index 96%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/RegexURLFilter.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLFilter.java
index dbd4b5ef..a0c16345 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/RegexURLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLFilter.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.regex;
+package org.apache.stormcrawler.filtering.regex;
import java.util.regex.Pattern;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/RegexURLFilterBase.java b/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLFilterBase.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/RegexURLFilterBase.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLFilterBase.java
index 384ed9e0..5ab83b97 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/RegexURLFilterBase.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLFilterBase.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.regex;
+package org.apache.stormcrawler.filtering.regex;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import java.io.BufferedReader;
@@ -28,6 +26,8 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilter;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/RegexURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLNormalizer.java
similarity index 98%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/RegexURLNormalizer.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLNormalizer.java
index 8c074999..0f3ea736 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/regex/RegexURLNormalizer.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLNormalizer.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.regex;
+package org.apache.stormcrawler.filtering.regex;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import java.io.FileNotFoundException;
@@ -35,6 +33,8 @@ import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import javax.xml.parsers.DocumentBuilderFactory;
import org.apache.commons.lang.StringUtils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilter;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/robots/RobotsFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/robots/RobotsFilter.java
similarity index 86%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/robots/RobotsFilter.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/robots/RobotsFilter.java
index e352eee7..28465b08 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/robots/RobotsFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/robots/RobotsFilter.java
@@ -12,18 +12,18 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.robots;
+package org.apache.stormcrawler.filtering.robots;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
-import com.digitalpebble.stormcrawler.protocol.HttpRobotRulesParser;
-import com.digitalpebble.stormcrawler.protocol.ProtocolFactory;
import com.fasterxml.jackson.databind.JsonNode;
import crawlercommons.robots.BaseRobotRules;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Map;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilter;
+import org.apache.stormcrawler.protocol.HttpRobotRulesParser;
+import org.apache.stormcrawler.protocol.ProtocolFactory;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -37,7 +37,7 @@ import org.jetbrains.annotations.Nullable;
*
* <pre>
* {
- * "class": "com.digitalpebble.stormcrawler.filtering.robots.RobotsFilter",
+ * "class": "org.apache.stormcrawler.filtering.robots.RobotsFilter",
* "name": "RobotsFilter",
* "params": {
* "fromCacheOnly": true
@@ -47,7 +47,7 @@ import org.jetbrains.annotations.Nullable;
*/
public class RobotsFilter extends URLFilter {
- private com.digitalpebble.stormcrawler.protocol.HttpRobotRulesParser robots;
+ private org.apache.stormcrawler.protocol.HttpRobotRulesParser robots;
private ProtocolFactory factory;
private boolean fromCacheOnly = true;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/sitemap/SitemapFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/sitemap/SitemapFilter.java
similarity index 86%
rename from core/src/main/java/com/digitalpebble/stormcrawler/filtering/sitemap/SitemapFilter.java
rename to core/src/main/java/org/apache/stormcrawler/filtering/sitemap/SitemapFilter.java
index 6aaca967..4c79d492 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/filtering/sitemap/SitemapFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/sitemap/SitemapFilter.java
@@ -12,12 +12,12 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering.sitemap;
+package org.apache.stormcrawler.filtering.sitemap;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.bolt.SiteMapParserBolt;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
import java.net.URL;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.bolt.SiteMapParserBolt;
+import org.apache.stormcrawler.filtering.URLFilter;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -28,7 +28,7 @@ import org.jetbrains.annotations.Nullable;
*
* <pre>
* {
- * "class": "com.digitalpebble.stormcrawler.filtering.sitemap.SitemapFilter",
+ * "class": "org.apache.stormcrawler.filtering.sitemap.SitemapFilter",
* "name": "SitemapFilter"
* }
* </pre>
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/indexing/AbstractIndexerBolt.java b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/indexing/AbstractIndexerBolt.java
rename to core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
index ea731c94..bbc8725c 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/indexing/AbstractIndexerBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.indexing;
+package org.apache.stormcrawler.indexing;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.RobotsTags;
-import com.digitalpebble.stormcrawler.util.URLUtil;
import crawlercommons.domains.PaidLevelDomain;
import java.net.MalformedURLException;
import java.net.URL;
@@ -37,6 +33,10 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.RobotsTags;
+import org.apache.stormcrawler.util.URLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -325,7 +325,7 @@ public abstract class AbstractIndexerBolt extends BaseRichBolt {
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declareStream(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
new Fields("url", "metadata", "status"));
}
}
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/indexing/DummyIndexer.java b/core/src/main/java/org/apache/stormcrawler/indexing/DummyIndexer.java
similarity index 89%
rename from core/src/main/java/com/digitalpebble/stormcrawler/indexing/DummyIndexer.java
rename to core/src/main/java/org/apache/stormcrawler/indexing/DummyIndexer.java
index 02781db1..32a6a29f 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/indexing/DummyIndexer.java
+++ b/core/src/main/java/org/apache/stormcrawler/indexing/DummyIndexer.java
@@ -12,15 +12,15 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.indexing;
+package org.apache.stormcrawler.indexing;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.Status;
import java.util.Map;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.Status;
/**
* * Any tuple that went through all the previous bolts is sent to the status stream with a Status
@@ -43,7 +43,7 @@ public class DummyIndexer extends AbstractIndexerBolt {
Metadata metadata = (Metadata) tuple.getValueByField("metadata");
_collector.emit(
- com.digitalpebble.stormcrawler.Constants.StatusStreamName,
+ org.apache.stormcrawler.Constants.StatusStreamName,
tuple,
new Values(url, metadata, Status.FETCHED));
_collector.ack(tuple);
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/indexing/StdOutIndexer.java b/core/src/main/java/org/apache/stormcrawler/indexing/StdOutIndexer.java
similarity index 93%
rename from core/src/main/java/com/digitalpebble/stormcrawler/indexing/StdOutIndexer.java
rename to core/src/main/java/org/apache/stormcrawler/indexing/StdOutIndexer.java
index 7ddf9f76..0c529dbf 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/indexing/StdOutIndexer.java
+++ b/core/src/main/java/org/apache/stormcrawler/indexing/StdOutIndexer.java
@@ -12,12 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.indexing;
+package org.apache.stormcrawler.indexing;
-import static com.digitalpebble.stormcrawler.Constants.StatusStreamName;
+import static org.apache.stormcrawler.Constants.StatusStreamName;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.Status;
import java.util.Iterator;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
@@ -25,6 +23,8 @@ import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.Status;
/**
* Indexer which generates fields for indexing and sends them to the standard output. Useful for
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/jsoup/LDJsonParseFilter.java b/core/src/main/java/org/apache/stormcrawler/jsoup/LDJsonParseFilter.java
similarity index 91%
rename from core/src/main/java/com/digitalpebble/stormcrawler/jsoup/LDJsonParseFilter.java
rename to core/src/main/java/org/apache/stormcrawler/jsoup/LDJsonParseFilter.java
index 7a362dfa..de12fd22 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/jsoup/LDJsonParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/jsoup/LDJsonParseFilter.java
@@ -12,13 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.jsoup;
+package org.apache.stormcrawler.jsoup;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.JSoupFilter;
-import com.digitalpebble.stormcrawler.parse.ParseData;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
-import com.digitalpebble.stormcrawler.util.AbstractConfigurable;
import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -27,6 +22,11 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.JSoupFilter;
+import org.apache.stormcrawler.parse.ParseData;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.util.AbstractConfigurable;
import org.jetbrains.annotations.NotNull;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/jsoup/LinkParseFilter.java b/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
similarity index 89%
rename from core/src/main/java/com/digitalpebble/stormcrawler/jsoup/LinkParseFilter.java
rename to core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
index 0232d53f..c2c8f499 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/jsoup/LinkParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
@@ -12,15 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.jsoup;
-
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilters;
-import com.digitalpebble.stormcrawler.parse.Outlink;
-import com.digitalpebble.stormcrawler.parse.ParseData;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
-import com.digitalpebble.stormcrawler.util.MetadataTransfer;
-import com.digitalpebble.stormcrawler.util.URLUtil;
+package org.apache.stormcrawler.jsoup;
+
import com.fasterxml.jackson.databind.JsonNode;
import java.net.MalformedURLException;
import java.net.URL;
@@ -28,6 +21,13 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilters;
+import org.apache.stormcrawler.parse.Outlink;
+import org.apache.stormcrawler.parse.ParseData;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.util.MetadataTransfer;
+import org.apache.stormcrawler.util.URLUtil;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory;
*
* <pre>{@code
* {
- * "class": "com.digitalpebble.stormcrawler.parse.filter.LinkParseFilter",
+ * "class": "org.apache.stormcrawler.parse.filter.LinkParseFilter",
* "name": "LinkParseFilter",
* "params": {
* "pattern": "//IMG/@src",
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/jsoup/XPathFilter.java b/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java
similarity index 92%
rename from core/src/main/java/com/digitalpebble/stormcrawler/jsoup/XPathFilter.java
rename to core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java
index ac31f127..80bf5717 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/jsoup/XPathFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java
@@ -12,13 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.jsoup;
+package org.apache.stormcrawler.jsoup;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.JSoupFilter;
-import com.digitalpebble.stormcrawler.parse.ParseData;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
-import com.digitalpebble.stormcrawler.util.AbstractConfigurable;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.IOException;
import java.util.ArrayList;
@@ -26,6 +21,11 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.JSoupFilter;
+import org.apache.stormcrawler.parse.ParseData;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.util.AbstractConfigurable;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/DocumentFragmentBuilder.java b/core/src/main/java/org/apache/stormcrawler/parse/DocumentFragmentBuilder.java
similarity index 98%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/DocumentFragmentBuilder.java
rename to core/src/main/java/org/apache/stormcrawler/parse/DocumentFragmentBuilder.java
index f20d7c51..985f2732 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/DocumentFragmentBuilder.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/DocumentFragmentBuilder.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
import org.apache.html.dom.HTMLDocumentImpl;
import org.jetbrains.annotations.NotNull;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/JSoupFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilter.java
similarity index 87%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/JSoupFilter.java
rename to core/src/main/java/org/apache/stormcrawler/parse/JSoupFilter.java
index 2ec376df..27863357 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/JSoupFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilter.java
@@ -12,15 +12,14 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
-import com.digitalpebble.stormcrawler.util.Configurable;
+import org.apache.stormcrawler.util.Configurable;
import org.jetbrains.annotations.NotNull;
/**
* Implementations of ParseFilter are responsible for extracting custom data from the crawled
- * content. They are used exclusively by {@link
- * com.digitalpebble.stormcrawler.bolt.JSoupParserBolt}.
+ * content. They are used exclusively by {@link org.apache.stormcrawler.bolt.JSoupParserBolt}.
*/
public interface JSoupFilter extends Configurable {
/**
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/JSoupFilters.java b/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java
similarity index 95%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/JSoupFilters.java
rename to core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java
index 70b145de..09c0e96f 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/JSoupFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
-import com.digitalpebble.stormcrawler.JSONResource;
-import com.digitalpebble.stormcrawler.util.AbstractConfigurable;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.Configurable;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
@@ -34,6 +30,10 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.Config;
import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.JSONResource;
+import org.apache.stormcrawler.util.AbstractConfigurable;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.Configurable;
import org.jetbrains.annotations.NotNull;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/Outlink.java b/core/src/main/java/org/apache/stormcrawler/parse/Outlink.java
similarity index 94%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/Outlink.java
rename to core/src/main/java/org/apache/stormcrawler/parse/Outlink.java
index dd5a0087..6b821072 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/Outlink.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/Outlink.java
@@ -12,9 +12,9 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
-import com.digitalpebble.stormcrawler.Metadata;
+import org.apache.stormcrawler.Metadata;
public class Outlink {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/ParseData.java b/core/src/main/java/org/apache/stormcrawler/parse/ParseData.java
similarity index 95%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/ParseData.java
rename to core/src/main/java/org/apache/stormcrawler/parse/ParseData.java
index b5a92d55..fdf6b8fc 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/ParseData.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/ParseData.java
@@ -12,9 +12,9 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
-import com.digitalpebble.stormcrawler.Metadata;
+import org.apache.stormcrawler.Metadata;
public class ParseData {
private byte[] content;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/ParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilter.java
similarity index 88%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/ParseFilter.java
rename to core/src/main/java/org/apache/stormcrawler/parse/ParseFilter.java
index 2a3a6390..d06c046d 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/ParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilter.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
-import com.digitalpebble.stormcrawler.util.AbstractConfigurable;
+import org.apache.stormcrawler.util.AbstractConfigurable;
import org.w3c.dom.DocumentFragment;
/**
* Implementations of ParseFilter are responsible for extracting custom data from the crawled
* content. They are used by parsing bolts such as {@link
- * com.digitalpebble.stormcrawler.bolt.JSoupParserBolt} or {@link
- * com.digitalpebble.stormcrawler.bolt.SiteMapParserBolt}.
+ * org.apache.stormcrawler.bolt.JSoupParserBolt} or {@link
+ * org.apache.stormcrawler.bolt.SiteMapParserBolt}.
*/
public abstract class ParseFilter extends AbstractConfigurable {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/ParseFilters.java b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java
similarity index 96%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/ParseFilters.java
rename to core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java
index 2e0fc095..d46db627 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/ParseFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java
@@ -12,11 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
-import com.digitalpebble.stormcrawler.JSONResource;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.Configurable;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
@@ -35,6 +32,9 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.Config;
import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.JSONResource;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.Configurable;
import org.jetbrains.annotations.NotNull;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/ParseResult.java b/core/src/main/java/org/apache/stormcrawler/parse/ParseResult.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/ParseResult.java
rename to core/src/main/java/org/apache/stormcrawler/parse/ParseResult.java
index e7ba91a1..4d09269f 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/ParseResult.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/ParseResult.java
@@ -12,14 +12,14 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
-import com.digitalpebble.stormcrawler.Metadata;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
public class ParseResult implements Iterable<Map.Entry<String, ParseData>> {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/TextExtractor.java b/core/src/main/java/org/apache/stormcrawler/parse/TextExtractor.java
similarity index 98%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/TextExtractor.java
rename to core/src/main/java/org/apache/stormcrawler/parse/TextExtractor.java
index f0b539ed..58ff4888 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/TextExtractor.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/TextExtractor.java
@@ -12,12 +12,12 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import org.apache.stormcrawler.util.ConfUtils;
import org.jetbrains.annotations.Contract;
import org.jsoup.helper.Validate;
import org.jsoup.internal.StringUtil;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/CollectionTagger.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/CollectionTagger.java
similarity index 95%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/CollectionTagger.java
rename to core/src/main/java/org/apache/stormcrawler/parse/filter/CollectionTagger.java
index 7e8a5b0f..15ba2439 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/CollectionTagger.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/CollectionTagger.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.JSONResource;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonMappingException;
@@ -30,6 +26,10 @@ import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.stormcrawler.JSONResource;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.util.ConfUtils;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java
similarity index 91%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java
rename to core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java
index ad2b59ab..c113f143 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
import com.fasterxml.jackson.databind.JsonNode;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
import org.jetbrains.annotations.NotNull;
import org.w3c.dom.DocumentFragment;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/DebugParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/DebugParseFilter.java
similarity index 92%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/DebugParseFilter.java
rename to core/src/main/java/org/apache/stormcrawler/parse/filter/DebugParseFilter.java
index cc01589e..9bed1dad 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/DebugParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/DebugParseFilter.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
import java.io.IOException;
@@ -23,6 +21,8 @@ import java.io.OutputStream;
import java.nio.file.Files;
import java.util.Map;
import org.apache.commons.io.FileUtils;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
import org.apache.xml.serialize.XMLSerializer;
import org.jetbrains.annotations.NotNull;
import org.w3c.dom.DocumentFragment;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/DomainParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java
similarity index 86%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/DomainParseFilter.java
rename to core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java
index f3b786d5..d98b0cc6 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/DomainParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
-import com.digitalpebble.stormcrawler.util.URLPartitioner;
import com.fasterxml.jackson.databind.JsonNode;
import java.util.HashMap;
import java.util.Map;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.util.URLPartitioner;
import org.jetbrains.annotations.NotNull;
import org.w3c.dom.DocumentFragment;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/LDJsonParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/LDJsonParseFilter.java
similarity index 93%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/LDJsonParseFilter.java
rename to core/src/main/java/org/apache/stormcrawler/parse/filter/LDJsonParseFilter.java
index 88342758..b700e57b 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/LDJsonParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/LDJsonParseFilter.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.ParseData;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -29,6 +25,10 @@ import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.ParseData;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/LinkParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
similarity index 89%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/LinkParseFilter.java
rename to core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
index 6e3d9e30..e818ac9e 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/LinkParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
@@ -12,15 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
-
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilters;
-import com.digitalpebble.stormcrawler.parse.Outlink;
-import com.digitalpebble.stormcrawler.parse.ParseData;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
-import com.digitalpebble.stormcrawler.util.MetadataTransfer;
-import com.digitalpebble.stormcrawler.util.URLUtil;
+package org.apache.stormcrawler.parse.filter;
+
import com.fasterxml.jackson.databind.JsonNode;
import java.net.MalformedURLException;
import java.net.URL;
@@ -28,6 +21,13 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilters;
+import org.apache.stormcrawler.parse.Outlink;
+import org.apache.stormcrawler.parse.ParseData;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.util.MetadataTransfer;
+import org.apache.stormcrawler.util.URLUtil;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,7 +38,7 @@ import org.w3c.dom.DocumentFragment;
*
* <pre>{@code
* {
- * "class": "com.digitalpebble.stormcrawler.parse.filter.LinkParseFilter",
+ * "class": "org.apache.stormcrawler.parse.filter.LinkParseFilter",
* "name": "LinkParseFilter",
* "params": {
* "pattern": "//IMG/@src",
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/MD5SignatureParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/MD5SignatureParseFilter.java
similarity index 92%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/MD5SignatureParseFilter.java
rename to core/src/main/java/org/apache/stormcrawler/parse/filter/MD5SignatureParseFilter.java
index 7733cb22..cb581492 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/MD5SignatureParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/MD5SignatureParseFilter.java
@@ -12,17 +12,17 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.ParseData;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
import com.fasterxml.jackson.databind.JsonNode;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.storm.shade.org.apache.commons.lang.StringUtils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.ParseData;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
import org.jetbrains.annotations.NotNull;
import org.w3c.dom.DocumentFragment;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/MimeTypeNormalization.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/MimeTypeNormalization.java
similarity index 91%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/MimeTypeNormalization.java
rename to core/src/main/java/org/apache/stormcrawler/parse/filter/MimeTypeNormalization.java
index 6704de21..1e3988f9 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/MimeTypeNormalization.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/MimeTypeNormalization.java
@@ -12,12 +12,12 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
import org.apache.storm.shade.org.apache.commons.lang.StringUtils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
import org.w3c.dom.DocumentFragment;
/**
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/XPathFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java
similarity index 96%
rename from core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/XPathFilter.java
rename to core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java
index 92aa675a..d03eeb84 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/parse/filter/XPathFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.ParseData;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.IOException;
import java.io.StringWriter;
@@ -34,6 +30,10 @@ import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.commons.lang.StringUtils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.ParseData;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
import org.apache.xml.serialize.Method;
import org.apache.xml.serialize.OutputFormat;
import org.apache.xml.serialize.XMLSerializer;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/AbstractQueryingSpout.java b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractQueryingSpout.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/AbstractQueryingSpout.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/AbstractQueryingSpout.java
index bb06a09a..1ef6a721 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/AbstractQueryingSpout.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractQueryingSpout.java
@@ -12,11 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence;
+package org.apache.stormcrawler.persistence;
-import com.digitalpebble.stormcrawler.persistence.urlbuffer.URLBuffer;
-import com.digitalpebble.stormcrawler.util.CollectionMetric;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import java.time.Instant;
@@ -33,6 +30,9 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.persistence.urlbuffer.URLBuffer;
+import org.apache.stormcrawler.util.CollectionMetric;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/AbstractStatusUpdaterBolt.java b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/AbstractStatusUpdaterBolt.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java
index 0cacff46..a490a181 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/AbstractStatusUpdaterBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence;
+package org.apache.stormcrawler.persistence;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.MetadataTransfer;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import java.time.Instant;
@@ -36,6 +32,10 @@ import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.MetadataTransfer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/AdaptiveScheduler.java b/core/src/main/java/org/apache/stormcrawler/persistence/AdaptiveScheduler.java
similarity index 94%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/AdaptiveScheduler.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/AdaptiveScheduler.java
index 7b4526ac..19684051 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/AdaptiveScheduler.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/AdaptiveScheduler.java
@@ -12,19 +12,19 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence;
+package org.apache.stormcrawler.persistence;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.filter.MD5SignatureParseFilter;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.time.Duration;
import java.util.Calendar;
import java.util.Date;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.filter.MD5SignatureParseFilter;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.LoggerFactory;
/**
@@ -72,7 +72,7 @@ import org.slf4j.LoggerFactory;
* (crawler-conf.yaml):
*
* <pre>
- * scheduler.class: "com.digitalpebble.stormcrawler.persistence.AdaptiveScheduler"
+ * scheduler.class: "org.apache.stormcrawler.persistence.AdaptiveScheduler"
* # set last-modified time ({@value HttpHeaders#LAST_MODIFIED}) used in HTTP If-Modified-Since request header field
* scheduler.adaptive.setLastModified: true
* # min. interval in minutes (default: 1h)
@@ -100,10 +100,10 @@ import org.slf4j.LoggerFactory;
* configured accordingly:
*
* <pre>
- * "com.digitalpebble.stormcrawler.parse.ParseFilters": [
+ * "org.apache.stormcrawler.parse.ParseFilters": [
* ...,
* {
- * "class": "com.digitalpebble.stormcrawler.parse.filter.MD5SignatureParseFilter",
+ * "class": "org.apache.stormcrawler.parse.filter.MD5SignatureParseFilter",
* "name": "MD5Digest",
* "params": {
* "useText": "false",
@@ -145,8 +145,8 @@ public class AdaptiveScheduler extends DefaultScheduler {
/**
* Name of the signature key in metadata, must be defined as "keyName" in the
- * configuration of {@link com.digitalpebble.stormcrawler.parse.filter.MD5SignatureParseFilter}
- * . This key must be listed in "metadata.persist".
+ * configuration of {@link org.apache.stormcrawler.parse.filter.MD5SignatureParseFilter} . This
+ * key must be listed in "metadata.persist".
*/
public static final String SIGNATURE_KEY = "signature";
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/DefaultScheduler.java b/core/src/main/java/org/apache/stormcrawler/persistence/DefaultScheduler.java
similarity index 94%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/DefaultScheduler.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/DefaultScheduler.java
index 28ca3c55..13b2bb90 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/DefaultScheduler.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/DefaultScheduler.java
@@ -12,14 +12,14 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence;
+package org.apache.stormcrawler.persistence;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.ConfUtils;
/** Schedules a nextFetchDate based on the configuration */
public class DefaultScheduler extends Scheduler {
@@ -41,7 +41,7 @@ public class DefaultScheduler extends Scheduler {
* (non-Javadoc)
*
* @see
- * com.digitalpebble.stormcrawler.persistence.Scheduler#init(java.util.Map)
+ * org.apache.stormcrawler.persistence.Scheduler#init(java.util.Map)
*/
@Override
public void init(Map<String, Object> stormConf) {
@@ -90,9 +90,9 @@ public class DefaultScheduler extends Scheduler {
/*
* (non-Javadoc)
*
- * @see com.digitalpebble.stormcrawler.persistence.Scheduler#schedule(com.
+ * @see org.apache.stormcrawler.persistence.Scheduler#schedule(com.
* digitalpebble. stormcrawler.persistence .Status,
- * com.digitalpebble.stormcrawler.Metadata)
+ * org.apache.stormcrawler.Metadata)
*/
@Override
public Optional<Date> schedule(Status status, Metadata metadata) {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/EmptyQueueListener.java b/core/src/main/java/org/apache/stormcrawler/persistence/EmptyQueueListener.java
similarity index 94%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/EmptyQueueListener.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/EmptyQueueListener.java
index 77d0bf5d..110ba2c1 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/EmptyQueueListener.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/EmptyQueueListener.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence;
+package org.apache.stormcrawler.persistence;
/** Used by URLBuffer to inform the spouts when a queue has no more URLs in it */
@FunctionalInterface
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/MemoryStatusUpdater.java b/core/src/main/java/org/apache/stormcrawler/persistence/MemoryStatusUpdater.java
similarity index 90%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/MemoryStatusUpdater.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/MemoryStatusUpdater.java
index 9b7de929..c133c9c5 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/MemoryStatusUpdater.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/MemoryStatusUpdater.java
@@ -12,13 +12,13 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence;
+package org.apache.stormcrawler.persistence;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.spout.MemorySpout;
import java.util.Date;
import java.util.Optional;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.spout.MemorySpout;
/**
* Use in combination with the MemorySpout for testing in local mode. There is no guarantee that
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/Scheduler.java b/core/src/main/java/org/apache/stormcrawler/persistence/Scheduler.java
similarity index 90%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/Scheduler.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/Scheduler.java
index c5d80466..e53153eb 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/Scheduler.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/Scheduler.java
@@ -12,14 +12,14 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence;
+package org.apache.stormcrawler.persistence;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.InitialisationUtil;
import java.util.Date;
import java.util.Map;
import java.util.Optional;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.InitialisationUtil;
public abstract class Scheduler {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/Status.java b/core/src/main/java/org/apache/stormcrawler/persistence/Status.java
similarity index 96%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/Status.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/Status.java
index 6c75fd89..abf8c977 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/Status.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/Status.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence;
+package org.apache.stormcrawler.persistence;
public enum Status {
DISCOVERED,
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/StdOutStatusUpdater.java b/core/src/main/java/org/apache/stormcrawler/persistence/StdOutStatusUpdater.java
similarity index 94%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/StdOutStatusUpdater.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/StdOutStatusUpdater.java
index bd0753a6..760bc8dd 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/StdOutStatusUpdater.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/StdOutStatusUpdater.java
@@ -12,12 +12,12 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence;
+package org.apache.stormcrawler.persistence;
-import com.digitalpebble.stormcrawler.Metadata;
import java.util.Date;
import java.util.Optional;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
/**
* Dummy status updater which dumps the content of the incoming tuples to the standard output.
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/AbstractURLBuffer.java b/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/AbstractURLBuffer.java
similarity index 93%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/AbstractURLBuffer.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/AbstractURLBuffer.java
index 4d8b8973..ae5acc14 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/AbstractURLBuffer.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/AbstractURLBuffer.java
@@ -12,11 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence.urlbuffer;
+package org.apache.stormcrawler.persistence.urlbuffer;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.EmptyQueueListener;
-import com.digitalpebble.stormcrawler.util.URLPartitioner;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -24,6 +21,9 @@ import java.util.LinkedList;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.EmptyQueueListener;
+import org.apache.stormcrawler.util.URLPartitioner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/PriorityURLBuffer.java b/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/PriorityURLBuffer.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/PriorityURLBuffer.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/PriorityURLBuffer.java
index dd97016e..eda922e9 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/PriorityURLBuffer.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/PriorityURLBuffer.java
@@ -12,9 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence.urlbuffer;
+package org.apache.stormcrawler.persistence.urlbuffer;
-import com.digitalpebble.stormcrawler.Metadata;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
@@ -26,6 +25,7 @@ import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Metadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/SchedulingURLBuffer.java b/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/SchedulingURLBuffer.java
similarity index 98%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/SchedulingURLBuffer.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/SchedulingURLBuffer.java
index 14908067..26bc4881 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/SchedulingURLBuffer.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/SchedulingURLBuffer.java
@@ -12,9 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence.urlbuffer;
+package org.apache.stormcrawler.persistence.urlbuffer;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
@@ -27,6 +26,7 @@ import java.util.Map.Entry;
import java.util.Queue;
import java.util.concurrent.TimeUnit;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.util.ConfUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/SimpleURLBuffer.java b/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/SimpleURLBuffer.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/SimpleURLBuffer.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/SimpleURLBuffer.java
index c575bf86..77ed8c6b 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/SimpleURLBuffer.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/SimpleURLBuffer.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence.urlbuffer;
+package org.apache.stormcrawler.persistence.urlbuffer;
import java.util.Iterator;
import java.util.Map;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/URLBuffer.java b/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/URLBuffer.java
similarity index 90%
rename from core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/URLBuffer.java
rename to core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/URLBuffer.java
index 39e4c97c..826e76f6 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/persistence/urlbuffer/URLBuffer.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/URLBuffer.java
@@ -12,15 +12,15 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence.urlbuffer;
+package org.apache.stormcrawler.persistence.urlbuffer;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.EmptyQueueListener;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.InitialisationUtil;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.EmptyQueueListener;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.InitialisationUtil;
import org.jetbrains.annotations.NotNull;
/**
@@ -29,7 +29,7 @@ import org.jetbrains.annotations.NotNull;
*
* <p>Configured by setting
*
- * <p>urlbuffer.class: "com.digitalpebble.stormcrawler.persistence.SimpleURLBuffer"
+ * <p>urlbuffer.class: "org.apache.stormcrawler.persistence.SimpleURLBuffer"
*
* <p>in the configuration
*
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/AbstractHttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/AbstractHttpProtocol.java
similarity index 94%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/AbstractHttpProtocol.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/AbstractHttpProtocol.java
index 0686698d..0405d494 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/AbstractHttpProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/AbstractHttpProtocol.java
@@ -12,23 +12,23 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
-import com.digitalpebble.stormcrawler.proxy.ProxyManager;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.InitialisationUtil;
import crawlercommons.robots.BaseRobotRules;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.Config;
+import org.apache.stormcrawler.proxy.ProxyManager;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.InitialisationUtil;
import org.slf4j.LoggerFactory;
public abstract class AbstractHttpProtocol implements Protocol {
private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(AbstractHttpProtocol.class);
- private com.digitalpebble.stormcrawler.protocol.HttpRobotRulesParser robots;
+ private org.apache.stormcrawler.protocol.HttpRobotRulesParser robots;
protected boolean skipRobots = false;
@@ -100,7 +100,7 @@ public abstract class AbstractHttpProtocol implements Protocol {
// SingleProxyManager by
// checking whether legacy proxy field is set
(ConfUtils.getString(conf, "http.proxy.host", null) != null)
- ? "com.digitalpebble.stormcrawler.proxy.SingleProxyManager"
+ ? "org.apache.stormcrawler.proxy.SingleProxyManager"
: null);
// conditionally load proxy manager
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/DelegatorProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java
similarity index 96%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/DelegatorProtocol.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java
index bc397523..cf398ff1 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/DelegatorProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.InitialisationUtil;
import crawlercommons.robots.BaseRobotRules;
import java.util.ArrayList;
import java.util.LinkedList;
@@ -23,6 +21,8 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.InitialisationUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.LoggerFactory;
@@ -35,19 +35,19 @@ import org.slf4j.LoggerFactory;
*
* <pre>
* protocol.delegator.config:
- * - className: "com.digitalpebble.stormcrawler.protocol.httpclient.HttpProtocol"
+ * - className: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
* filters:
* domain: "example.com"
* depth: "3"
* test
* operator: OR
- * - className: "com.digitalpebble.stormcrawler.protocol.okhttp.HttpProtocol"
+ * - className: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
* filters:
* robots.txt:
* regex:
* - \.pdf
* - \.doc
- * - className: "com.digitalpebble.stormcrawler.protocol.selenium.SeleniumProtocol"
+ * - className: "org.apache.stormcrawler.protocol.selenium.SeleniumProtocol"
* </pre>
*
* Typically, the last one in the list must not have filters as it is used as a default value. The
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/HttpHeaders.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpHeaders.java
similarity index 98%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/HttpHeaders.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/HttpHeaders.java
index 3d84db3d..7a016e33 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/HttpHeaders.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpHeaders.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
import java.time.ZoneId;
import java.time.ZoneOffset;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/HttpRobotRulesParser.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
index f68fbb77..125cee07 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/HttpRobotRulesParser.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.github.benmanes.caffeine.cache.Cache;
import com.google.common.primitives.Ints;
import crawlercommons.robots.BaseRobotRules;
@@ -27,6 +25,8 @@ import java.util.Locale;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.ConfUtils;
/**
* This class is used for parsing robots for urls belonging to HTTP protocol. It extends the generic
@@ -79,7 +79,7 @@ public class HttpRobotRulesParser extends RobotRulesParser {
/**
* Returns the robots rules from the cache or empty rules if not found
*
- * @see com.digitalpebble.stormcrawler.filtering.robots.RobotsFilter
+ * @see org.apache.stormcrawler.filtering.robots.RobotsFilter
*/
public BaseRobotRules getRobotRulesSetFromCache(URL url) {
String cacheKey = getCacheKey(url);
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/Protocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/Protocol.java
similarity index 96%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/Protocol.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/Protocol.java
index 3e63f3c2..99842f4d 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/Protocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/Protocol.java
@@ -12,11 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.StringTabScheme;
import crawlercommons.robots.BaseRobotRules;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@@ -33,6 +30,9 @@ import org.apache.commons.cli.Options;
import org.apache.commons.io.FileUtils;
import org.apache.storm.Config;
import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.StringTabScheme;
public interface Protocol {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/ProtocolFactory.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java
similarity index 92%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/ProtocolFactory.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java
index 90c94d48..cb3c154e 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/ProtocolFactory.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java
@@ -12,14 +12,14 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.InitialisationUtil;
import java.net.URL;
import java.util.HashMap;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.Config;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.InitialisationUtil;
import org.slf4j.LoggerFactory;
public class ProtocolFactory {
@@ -69,10 +69,10 @@ public class ProtocolFactory {
// set the default values
if (protocol.equalsIgnoreCase("http")) {
protocolimplementation =
- "com.digitalpebble.stormcrawler.protocol.httpclient.HttpProtocol";
+ "org.apache.stormcrawler.protocol.httpclient.HttpProtocol";
} else if (protocol.equalsIgnoreCase("https")) {
protocolimplementation =
- "com.digitalpebble.stormcrawler.protocol.httpclient.HttpProtocol";
+ "org.apache.stormcrawler.protocol.httpclient.HttpProtocol";
} else throw new RuntimeException(paramName + "should not have an empty value");
}
// we have a value -> is it correct?
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/ProtocolResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/ProtocolResponse.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java
index 48b0c0ef..6f230dcd 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/ProtocolResponse.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java
@@ -12,9 +12,9 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
-import com.digitalpebble.stormcrawler.Metadata;
+import org.apache.stormcrawler.Metadata;
public class ProtocolResponse {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/RobotRules.java b/core/src/main/java/org/apache/stormcrawler/protocol/RobotRules.java
similarity index 98%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/RobotRules.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/RobotRules.java
index ca305c91..1f412a6a 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/RobotRules.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/RobotRules.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
import crawlercommons.robots.BaseRobotRules;
import java.util.List;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/RobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
similarity index 98%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/RobotRulesParser.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
index 7164e0e5..2cdee19c 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/RobotRulesParser.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
@@ -12,9 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import crawlercommons.robots.BaseRobotRules;
@@ -30,6 +29,7 @@ import java.util.StringTokenizer;
import java.util.regex.Pattern;
import javax.security.auth.login.Configuration;
import org.apache.storm.Config;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/file/FileProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileProtocol.java
similarity index 82%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/file/FileProtocol.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/file/FileProtocol.java
index 85570675..ce21db5e 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/file/FileProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileProtocol.java
@@ -12,15 +12,15 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol.file;
+package org.apache.stormcrawler.protocol.file;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.Protocol;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
-import com.digitalpebble.stormcrawler.protocol.RobotRulesParser;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import crawlercommons.robots.BaseRobotRules;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.Protocol;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.protocol.RobotRulesParser;
+import org.apache.stormcrawler.util.ConfUtils;
public class FileProtocol implements Protocol {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/file/FileResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java
similarity index 95%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/file/FileResponse.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java
index 6f950acd..f245a20b 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/file/FileResponse.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java
@@ -12,11 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol.file;
+package org.apache.stormcrawler.protocol.file;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -28,6 +25,9 @@ import java.util.Date;
import java.util.Locale;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpStatus;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.slf4j.LoggerFactory;
public class FileResponse {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/httpclient/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java
similarity index 95%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/httpclient/HttpProtocol.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java
index 0f31f151..1661105f 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/httpclient/HttpProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java
@@ -12,18 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol.httpclient;
-
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.protocol.AbstractHttpProtocol;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.Protocol;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
-import com.digitalpebble.stormcrawler.proxy.*;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.CookieConverter;
+package org.apache.stormcrawler.protocol.httpclient;
+
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
@@ -61,6 +51,16 @@ import org.apache.http.message.BasicHeader;
import org.apache.http.util.Args;
import org.apache.http.util.ByteArrayBuffer;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.AbstractHttpProtocol;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.Protocol;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.proxy.*;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.CookieConverter;
import org.jetbrains.annotations.Nullable;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/okhttp/DNSResolutionListener.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/DNSResolutionListener.java
similarity index 96%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/okhttp/DNSResolutionListener.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/okhttp/DNSResolutionListener.java
index fef915ea..df315a50 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/okhttp/DNSResolutionListener.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/DNSResolutionListener.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol.okhttp;
+package org.apache.stormcrawler.protocol.okhttp;
import java.net.InetAddress;
import java.util.List;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/okhttp/HttpProtocol.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
index 3855624d..84d18638 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/okhttp/HttpProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
@@ -12,17 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol.okhttp;
-
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.AbstractHttpProtocol;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse.TrimmedContentReason;
-import com.digitalpebble.stormcrawler.proxy.SCProxy;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.CookieConverter;
+package org.apache.stormcrawler.protocol.okhttp;
+
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
@@ -62,6 +53,15 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.mutable.MutableObject;
import org.apache.http.cookie.Cookie;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.AbstractHttpProtocol;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.protocol.ProtocolResponse.TrimmedContentReason;
+import org.apache.stormcrawler.proxy.SCProxy;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.CookieConverter;
import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory;
@@ -591,6 +591,6 @@ public class HttpProtocol extends AbstractHttpProtocol {
}
public static void main(String args[]) throws Exception {
- com.digitalpebble.stormcrawler.protocol.Protocol.main(new HttpProtocol(), args);
+ org.apache.stormcrawler.protocol.Protocol.main(new HttpProtocol(), args);
}
}
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/NavigationFilter.java b/core/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilter.java
similarity index 83%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/NavigationFilter.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilter.java
index ac6fd01f..0601c2e7 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/NavigationFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilter.java
@@ -12,11 +12,11 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol.selenium;
+package org.apache.stormcrawler.protocol.selenium;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
-import com.digitalpebble.stormcrawler.util.AbstractConfigurable;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.util.AbstractConfigurable;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.openqa.selenium.remote.RemoteWebDriver;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/NavigationFilters.java b/core/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilters.java
similarity index 91%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/NavigationFilters.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilters.java
index 8e32371a..263e4b96 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/NavigationFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilters.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol.selenium;
+package org.apache.stormcrawler.protocol.selenium;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.Configurable;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
@@ -25,6 +21,10 @@ import java.io.InputStream;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.Configurable;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.openqa.selenium.remote.RemoteWebDriver;
@@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory;
/**
* Wrapper for the NavigationFilter defined in a JSON configuration
*
- * @see com.digitalpebble.stormcrawler.util.Configurable#createConfiguredInstance(Class, Class, Map,
+ * @see org.apache.stormcrawler.util.Configurable#createConfiguredInstance(Class, Class, Map,
* JsonNode) for more information.
*/
public class NavigationFilters extends NavigationFilter {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/RemoteDriverProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
index 4eab4d44..85dc7ae9 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol.selenium;
+package org.apache.stormcrawler.protocol.selenium;
-import com.digitalpebble.stormcrawler.protocol.Protocol;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
@@ -24,6 +22,8 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.storm.Config;
+import org.apache.stormcrawler.protocol.Protocol;
+import org.apache.stormcrawler.util.ConfUtils;
import org.openqa.selenium.WebDriver.Timeouts;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.openqa.selenium.remote.RemoteWebDriver;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/SeleniumProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java
similarity index 92%
rename from core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/SeleniumProtocol.java
rename to core/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java
index 0e2cff99..6b3cd85d 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/SeleniumProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java
@@ -12,15 +12,15 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol.selenium;
+package org.apache.stormcrawler.protocol.selenium;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.AbstractHttpProtocol;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
import java.time.Instant;
import java.util.concurrent.LinkedBlockingQueue;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.AbstractHttpProtocol;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.openqa.selenium.remote.RemoteWebDriver;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/proxy/MultiProxyManager.java b/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/proxy/MultiProxyManager.java
rename to core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
index 31e7ed39..43810c3f 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/proxy/MultiProxyManager.java
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
@@ -12,11 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.proxy;
+package org.apache.stormcrawler.proxy;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.httpclient.HttpProtocol;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -26,6 +23,9 @@ import java.util.Scanner;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang.ArrayUtils;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.httpclient.HttpProtocol;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.LoggerFactory;
/** MultiProxyManager is a ProxyManager implementation for a multiple proxy endpoints */
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/proxy/ProxyManager.java b/core/src/main/java/org/apache/stormcrawler/proxy/ProxyManager.java
similarity index 91%
rename from core/src/main/java/com/digitalpebble/stormcrawler/proxy/ProxyManager.java
rename to core/src/main/java/org/apache/stormcrawler/proxy/ProxyManager.java
index 102dea07..89fe66ab 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/proxy/ProxyManager.java
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/ProxyManager.java
@@ -12,10 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.proxy;
+package org.apache.stormcrawler.proxy;
-import com.digitalpebble.stormcrawler.Metadata;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
/**
* Proxy manager is an abstract class specification that details the required interface of a proxy
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/proxy/SCProxy.java b/core/src/main/java/org/apache/stormcrawler/proxy/SCProxy.java
similarity index 99%
rename from core/src/main/java/com/digitalpebble/stormcrawler/proxy/SCProxy.java
rename to core/src/main/java/org/apache/stormcrawler/proxy/SCProxy.java
index 88df1aae..b0dfc8f3 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/proxy/SCProxy.java
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/SCProxy.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.proxy;
+package org.apache.stormcrawler.proxy;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/proxy/SingleProxyManager.java b/core/src/main/java/org/apache/stormcrawler/proxy/SingleProxyManager.java
similarity index 94%
rename from core/src/main/java/com/digitalpebble/stormcrawler/proxy/SingleProxyManager.java
rename to core/src/main/java/org/apache/stormcrawler/proxy/SingleProxyManager.java
index a39e0811..80945318 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/proxy/SingleProxyManager.java
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/SingleProxyManager.java
@@ -12,11 +12,11 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.proxy;
+package org.apache.stormcrawler.proxy;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.ConfUtils;
/** SingleProxyManager is a ProxyManager implementation for a single proxy endpoint */
public class SingleProxyManager implements ProxyManager {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/spout/FileSpout.java b/core/src/main/java/org/apache/stormcrawler/spout/FileSpout.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/spout/FileSpout.java
rename to core/src/main/java/org/apache/stormcrawler/spout/FileSpout.java
index 3ae49c36..834f0008 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/spout/FileSpout.java
+++ b/core/src/main/java/org/apache/stormcrawler/spout/FileSpout.java
@@ -12,11 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.spout;
+package org.apache.stormcrawler.spout;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.StringTabScheme;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
@@ -35,6 +32,9 @@ import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.StringTabScheme;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/spout/MemorySpout.java b/core/src/main/java/org/apache/stormcrawler/spout/MemorySpout.java
similarity index 95%
rename from core/src/main/java/com/digitalpebble/stormcrawler/spout/MemorySpout.java
rename to core/src/main/java/org/apache/stormcrawler/spout/MemorySpout.java
index dcb2a3cc..0bc281bf 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/spout/MemorySpout.java
+++ b/core/src/main/java/org/apache/stormcrawler/spout/MemorySpout.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.spout;
+package org.apache.stormcrawler.spout;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.StringTabScheme;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Date;
@@ -30,6 +26,10 @@ import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.StringTabScheme;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/AbstractConfigurable.java b/core/src/main/java/org/apache/stormcrawler/util/AbstractConfigurable.java
similarity index 96%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/AbstractConfigurable.java
rename to core/src/main/java/org/apache/stormcrawler/util/AbstractConfigurable.java
index 391c991d..dcf1cbc4 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/AbstractConfigurable.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/AbstractConfigurable.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import com.fasterxml.jackson.databind.JsonNode;
import java.util.Map;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/CharsetIdentification.java b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java
similarity index 98%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/CharsetIdentification.java
rename to core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java
index 54348d44..14d59d52 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/CharsetIdentification.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java
@@ -1,7 +1,5 @@
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import java.io.ByteArrayInputStream;
@@ -15,6 +13,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.ByteOrderMark;
import org.apache.commons.io.input.BOMInputStream;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.HttpHeaders;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Parser;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/CollectionMetric.java b/core/src/main/java/org/apache/stormcrawler/util/CollectionMetric.java
similarity index 96%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/CollectionMetric.java
rename to core/src/main/java/org/apache/stormcrawler/util/CollectionMetric.java
index 6863b499..58c33272 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/CollectionMetric.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/CollectionMetric.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import java.util.LinkedList;
import java.util.List;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/ConfUtils.java b/core/src/main/java/org/apache/stormcrawler/util/ConfUtils.java
similarity index 99%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/ConfUtils.java
rename to core/src/main/java/org/apache/stormcrawler/util/ConfUtils.java
index ffbcd0ac..3fef86ce 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/ConfUtils.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/ConfUtils.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/Configurable.java b/core/src/main/java/org/apache/stormcrawler/util/Configurable.java
similarity index 99%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/Configurable.java
rename to core/src/main/java/org/apache/stormcrawler/util/Configurable.java
index b442505b..e8603629 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/Configurable.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/Configurable.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import com.fasterxml.jackson.databind.JsonNode;
import java.util.List;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/ConfigurableHelper.java b/core/src/main/java/org/apache/stormcrawler/util/ConfigurableHelper.java
similarity index 99%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/ConfigurableHelper.java
rename to core/src/main/java/org/apache/stormcrawler/util/ConfigurableHelper.java
index 7e97f6ae..3f6dcb50 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/ConfigurableHelper.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/ConfigurableHelper.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.NullNode;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/CookieConverter.java b/core/src/main/java/org/apache/stormcrawler/util/CookieConverter.java
similarity index 99%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/CookieConverter.java
rename to core/src/main/java/org/apache/stormcrawler/util/CookieConverter.java
index 971c2254..305d7c97 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/CookieConverter.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/CookieConverter.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import java.net.URL;
import java.text.ParseException;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/InitialisationUtil.java b/core/src/main/java/org/apache/stormcrawler/util/InitialisationUtil.java
similarity index 99%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/InitialisationUtil.java
rename to core/src/main/java/org/apache/stormcrawler/util/InitialisationUtil.java
index 8985b513..f2199247 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/InitialisationUtil.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/InitialisationUtil.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/MetadataTransfer.java b/core/src/main/java/org/apache/stormcrawler/util/MetadataTransfer.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/MetadataTransfer.java
rename to core/src/main/java/org/apache/stormcrawler/util/MetadataTransfer.java
index 84b40d7e..f6de0d2a 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/MetadataTransfer.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/MetadataTransfer.java
@@ -12,14 +12,14 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
/**
* Implements the logic of how the metadata should be passed to the outlinks, what should be stored
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/PerSecondReducer.java b/core/src/main/java/org/apache/stormcrawler/util/PerSecondReducer.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/PerSecondReducer.java
rename to core/src/main/java/org/apache/stormcrawler/util/PerSecondReducer.java
index 8d4bec7c..e83898a1 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/PerSecondReducer.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/PerSecondReducer.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import org.apache.storm.metric.api.IReducer;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/RefreshTag.java b/core/src/main/java/org/apache/stormcrawler/util/RefreshTag.java
similarity index 97%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/RefreshTag.java
rename to core/src/main/java/org/apache/stormcrawler/util/RefreshTag.java
index 5c42fe67..f9c007f4 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/RefreshTag.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/RefreshTag.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/RobotsTags.java b/core/src/main/java/org/apache/stormcrawler/util/RobotsTags.java
similarity index 98%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/RobotsTags.java
rename to core/src/main/java/org/apache/stormcrawler/util/RobotsTags.java
index 4d876bc2..be02254c 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/RobotsTags.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/RobotsTags.java
@@ -12,14 +12,14 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
-import com.digitalpebble.stormcrawler.Metadata;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
+import org.apache.stormcrawler.Metadata;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/StringTabScheme.java b/core/src/main/java/org/apache/stormcrawler/util/StringTabScheme.java
similarity index 95%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/StringTabScheme.java
rename to core/src/main/java/org/apache/stormcrawler/util/StringTabScheme.java
index 426f2ba8..1382480b 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/StringTabScheme.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/StringTabScheme.java
@@ -12,15 +12,15 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
-import com.digitalpebble.stormcrawler.Metadata;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.List;
import org.apache.storm.spout.Scheme;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Metadata;
/** Converts a byte array into URL + metadata */
public class StringTabScheme implements Scheme {
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/URLPartitioner.java b/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java
similarity index 96%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/URLPartitioner.java
rename to core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java
index 5945c82d..3c9ba5bd 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/URLPartitioner.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
import crawlercommons.domains.PaidLevelDomain;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/URLStreamGrouping.java b/core/src/main/java/org/apache/stormcrawler/util/URLStreamGrouping.java
similarity index 94%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/URLStreamGrouping.java
rename to core/src/main/java/org/apache/stormcrawler/util/URLStreamGrouping.java
index 4467333d..baeeace7 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/URLStreamGrouping.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/URLStreamGrouping.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
import com.google.common.collect.ImmutableList;
import java.io.Serializable;
import java.util.HashMap;
@@ -25,6 +23,8 @@ import org.apache.storm.generated.GlobalStreamId;
import org.apache.storm.grouping.CustomStreamGrouping;
import org.apache.storm.shade.org.apache.commons.lang.StringUtils;
import org.apache.storm.task.WorkerTopologyContext;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,7 +41,7 @@ import org.slf4j.LoggerFactory;
* grouping:
* type: CUSTOM
* customClass:
- * className: "com.digitalpebble.stormcrawler.util.URLStreamGrouping"
+ * className: "org.apache.stormcrawler.util.URLStreamGrouping"
* constructorArgs:
* - "byDomain"
* }</pre>
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/util/URLUtil.java b/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java
similarity index 99%
rename from core/src/main/java/com/digitalpebble/stormcrawler/util/URLUtil.java
rename to core/src/main/java/org/apache/stormcrawler/util/URLUtil.java
index 2555e127..d59c957e 100644
--- a/core/src/main/java/com/digitalpebble/stormcrawler/util/URLUtil.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import java.net.IDN;
import java.net.MalformedURLException;
diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml
index f1bea6e0..a8725338 100644
--- a/core/src/main/resources/crawler-default.yaml
+++ b/core/src/main/resources/crawler-default.yaml
@@ -45,7 +45,7 @@ config:
# alternative values are "byIP" and "byDomain"
partition.url.mode: "byHost"
- urlbuffer.class: "com.digitalpebble.stormcrawler.persistence.urlbuffer.SimpleURLBuffer"
+ urlbuffer.class: "org.apache.stormcrawler.persistence.urlbuffer.SimpleURLBuffer"
# Lists the metadata to transfer to outlinks
# Used by Fetcher and SiteMapParser for redirections,
@@ -152,9 +152,9 @@ config:
robots.error.cache.spec: "maximumSize=10000,expireAfterWrite=1h"
protocols: "http,https,file"
- http.protocol.implementation: "com.digitalpebble.stormcrawler.protocol.httpclient.HttpProtocol"
- https.protocol.implementation: "com.digitalpebble.stormcrawler.protocol.httpclient.HttpProtocol"
- file.protocol.implementation: "com.digitalpebble.stormcrawler.protocol.file.FileProtocol"
+ http.protocol.implementation: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
+ https.protocol.implementation: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
+ file.protocol.implementation: "org.apache.stormcrawler.protocol.file.FileProtocol"
# number of instances for each protocol implementation
protocol.instances.num: 1
@@ -254,7 +254,7 @@ config:
# - VIDEO
# Default implementation of Scheduler
- scheduler.class: "com.digitalpebble.stormcrawler.persistence.DefaultScheduler"
+ scheduler.class: "org.apache.stormcrawler.persistence.DefaultScheduler"
# revisit a page daily (value in minutes)
# set it to -1 to never refetch a page
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/MetadataTest.java b/core/src/test/java/org/apache/stormcrawler/MetadataTest.java
similarity index 97%
rename from core/src/test/java/com/digitalpebble/stormcrawler/MetadataTest.java
rename to core/src/test/java/org/apache/stormcrawler/MetadataTest.java
index d714f978..6af18c77 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/MetadataTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/MetadataTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler;
+package org.apache.stormcrawler;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/TestMetadataSerialization.java b/core/src/test/java/org/apache/stormcrawler/TestMetadataSerialization.java
similarity index 98%
rename from core/src/test/java/com/digitalpebble/stormcrawler/TestMetadataSerialization.java
rename to core/src/test/java/org/apache/stormcrawler/TestMetadataSerialization.java
index bcf711aa..a30fa85e 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/TestMetadataSerialization.java
+++ b/core/src/test/java/org/apache/stormcrawler/TestMetadataSerialization.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler;
+package org.apache.stormcrawler;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/TestOutputCollector.java b/core/src/test/java/org/apache/stormcrawler/TestOutputCollector.java
similarity index 98%
rename from core/src/test/java/com/digitalpebble/stormcrawler/TestOutputCollector.java
rename to core/src/test/java/org/apache/stormcrawler/TestOutputCollector.java
index 9303e800..a3063609 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/TestOutputCollector.java
+++ b/core/src/test/java/org/apache/stormcrawler/TestOutputCollector.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler;
+package org.apache.stormcrawler;
import java.util.ArrayList;
import java.util.Collection;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/TestUtil.java b/core/src/test/java/org/apache/stormcrawler/TestUtil.java
similarity index 99%
rename from core/src/test/java/com/digitalpebble/stormcrawler/TestUtil.java
rename to core/src/test/java/org/apache/stormcrawler/TestUtil.java
index 1eb0aeb9..55203c80 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/TestUtil.java
+++ b/core/src/test/java/org/apache/stormcrawler/TestUtil.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler;
+package org.apache.stormcrawler;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/bolt/AbstractFetcherBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
similarity index 95%
rename from core/src/test/java/com/digitalpebble/stormcrawler/bolt/AbstractFetcherBoltTest.java
rename to core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
index b712c42e..a251159d 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/bolt/AbstractFetcherBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
+package org.apache.stormcrawler.bolt;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
import static com.github.tomakehurst.wiremock.client.WireMock.get;
@@ -21,9 +21,6 @@ import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.TestOutputCollector;
-import com.digitalpebble.stormcrawler.TestUtil;
import com.github.tomakehurst.wiremock.junit.WireMockRule;
import java.io.IOException;
import java.util.HashMap;
@@ -33,6 +30,9 @@ import org.apache.storm.task.OutputCollector;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.TestOutputCollector;
+import org.apache.stormcrawler.TestUtil;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/bolt/FeedParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java
similarity index 90%
rename from core/src/test/java/com/digitalpebble/stormcrawler/bolt/FeedParserBoltTest.java
rename to core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java
index 01424bc3..d55b7b72 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/bolt/FeedParserBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java
@@ -12,19 +12,19 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
-
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.parse.ParsingTester;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
+package org.apache.stormcrawler.bolt;
+
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.storm.task.OutputCollector;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.parse.ParsingTester;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/bolt/FetcherBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/FetcherBoltTest.java
similarity index 95%
rename from core/src/test/java/com/digitalpebble/stormcrawler/bolt/FetcherBoltTest.java
rename to core/src/test/java/org/apache/stormcrawler/bolt/FetcherBoltTest.java
index 5b0bcecd..7f204870 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/bolt/FetcherBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/FetcherBoltTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
+package org.apache.stormcrawler.bolt;
import org.junit.Before;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/bolt/JSoupParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java
similarity index 96%
rename from core/src/test/java/com/digitalpebble/stormcrawler/bolt/JSoupParserBoltTest.java
rename to core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java
index 1605de48..6adfaf30 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/bolt/JSoupParserBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java
@@ -12,19 +12,19 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
-
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.parse.ParsingTester;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.RobotsTags;
+package org.apache.stormcrawler.bolt;
+
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.storm.task.OutputCollector;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.parse.ParsingTester;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.RobotsTags;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/bolt/SimpleFetcherBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/SimpleFetcherBoltTest.java
similarity index 95%
rename from core/src/test/java/com/digitalpebble/stormcrawler/bolt/SimpleFetcherBoltTest.java
rename to core/src/test/java/org/apache/stormcrawler/bolt/SimpleFetcherBoltTest.java
index 6fc56dd2..ec6e5941 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/bolt/SimpleFetcherBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/SimpleFetcherBoltTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
+package org.apache.stormcrawler.bolt;
import org.junit.Before;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/bolt/SiteMapParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java
similarity index 98%
rename from core/src/test/java/com/digitalpebble/stormcrawler/bolt/SiteMapParserBoltTest.java
rename to core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java
index 34eb2031..004801c8 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/bolt/SiteMapParserBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java
@@ -12,13 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.bolt;
+package org.apache.stormcrawler.bolt;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.parse.ParsingTester;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
import crawlercommons.sitemaps.extension.Extension;
import java.io.IOException;
import java.util.Arrays;
@@ -28,6 +23,11 @@ import java.util.List;
import java.util.Map;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.parse.ParsingTester;
+import org.apache.stormcrawler.protocol.HttpHeaders;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/BasicURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java
similarity index 94%
rename from core/src/test/java/com/digitalpebble/stormcrawler/filtering/BasicURLFilterTest.java
rename to core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java
index 2ef78bb7..24d48471 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/BasicURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering;
+package org.apache.stormcrawler.filtering;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.basic.BasicURLFilter;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.basic.BasicURLFilter;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
similarity index 98%
rename from core/src/test/java/com/digitalpebble/stormcrawler/filtering/BasicURLNormalizerTest.java
rename to core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
index 953b9924..654d4438 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/BasicURLNormalizerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
@@ -12,12 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering;
+package org.apache.stormcrawler.filtering;
import static org.junit.Assert.assertEquals;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.basic.BasicURLNormalizer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
@@ -28,6 +26,8 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.basic.BasicURLNormalizer;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/FastURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java
similarity index 94%
rename from core/src/test/java/com/digitalpebble/stormcrawler/filtering/FastURLFilterTest.java
rename to core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java
index 480a4fd6..142bb34a 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/FastURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering;
+package org.apache.stormcrawler.filtering;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.regex.FastURLFilter;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.regex.FastURLFilter;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/HostURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
similarity index 96%
rename from core/src/test/java/com/digitalpebble/stormcrawler/filtering/HostURLFilterTest.java
rename to core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
index 16cf6e82..c540a5ba 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/HostURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering;
+package org.apache.stormcrawler.filtering;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.host.HostURLFilter;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.host.HostURLFilter;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/MaxDepthFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java
similarity index 93%
rename from core/src/test/java/com/digitalpebble/stormcrawler/filtering/MaxDepthFilterTest.java
rename to core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java
index 83433a0f..24c2ffbc 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/MaxDepthFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java
@@ -12,17 +12,17 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering;
+package org.apache.stormcrawler.filtering;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.depth.MaxDepthFilter;
-import com.digitalpebble.stormcrawler.util.MetadataTransfer;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.depth.MaxDepthFilter;
+import org.apache.stormcrawler.util.MetadataTransfer;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/MetadataFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java
similarity index 94%
rename from core/src/test/java/com/digitalpebble/stormcrawler/filtering/MetadataFilterTest.java
rename to core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java
index 9d335224..fdc0a680 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/MetadataFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering;
+package org.apache.stormcrawler.filtering;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.metadata.MetadataFilter;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.metadata.MetadataFilter;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/RegexFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java
similarity index 95%
rename from core/src/test/java/com/digitalpebble/stormcrawler/filtering/RegexFilterTest.java
rename to core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java
index 637b706a..ebf8973b 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/filtering/RegexFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.filtering;
+package org.apache.stormcrawler.filtering;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.regex.RegexURLFilter;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.regex.RegexURLFilter;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/ClassInheritingFomAbstractAndInterface.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFomAbstractAndInterface.java
similarity index 80%
rename from core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/ClassInheritingFomAbstractAndInterface.java
rename to core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFomAbstractAndInterface.java
index 7b5f7a7a..77e159b0 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/ClassInheritingFomAbstractAndInterface.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFomAbstractAndInterface.java
@@ -12,10 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.helper.initialisation;
+package org.apache.stormcrawler.helper.initialisation;
-import com.digitalpebble.stormcrawler.helper.initialisation.base.AbstractClass;
-import com.digitalpebble.stormcrawler.helper.initialisation.base.ITestInterface;
+import org.apache.stormcrawler.helper.initialisation.base.AbstractClass;
+import org.apache.stormcrawler.helper.initialisation.base.ITestInterface;
public class ClassInheritingFomAbstractAndInterface extends AbstractClass
implements ITestInterface {}
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/ClassInheritingFromAbstractClassOnly.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromAbstractClassOnly.java
similarity index 85%
rename from core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/ClassInheritingFromAbstractClassOnly.java
rename to core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromAbstractClassOnly.java
index ebbee97a..38134886 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/ClassInheritingFromAbstractClassOnly.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromAbstractClassOnly.java
@@ -12,8 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.helper.initialisation;
+package org.apache.stormcrawler.helper.initialisation;
-import com.digitalpebble.stormcrawler.helper.initialisation.base.AbstractClass;
+import org.apache.stormcrawler.helper.initialisation.base.AbstractClass;
public class ClassInheritingFromAbstractClassOnly extends AbstractClass {}
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/ClassInheritingFromOpenClass.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromOpenClass.java
similarity index 84%
rename from core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/ClassInheritingFromOpenClass.java
rename to core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromOpenClass.java
index 67f8fc8c..4af19847 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/ClassInheritingFromOpenClass.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromOpenClass.java
@@ -12,8 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.helper.initialisation;
+package org.apache.stormcrawler.helper.initialisation;
-import com.digitalpebble.stormcrawler.helper.initialisation.base.OpenClassWithAbstractClassAndInterface;
+import org.apache.stormcrawler.helper.initialisation.base.OpenClassWithAbstractClassAndInterface;
public class ClassInheritingFromOpenClass extends OpenClassWithAbstractClassAndInterface {}
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/ClassWithoutValidConstructor.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassWithoutValidConstructor.java
similarity index 86%
rename from core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/ClassWithoutValidConstructor.java
rename to core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassWithoutValidConstructor.java
index a501a65e..f93db3f0 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/ClassWithoutValidConstructor.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassWithoutValidConstructor.java
@@ -12,9 +12,9 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.helper.initialisation;
+package org.apache.stormcrawler.helper.initialisation;
-import com.digitalpebble.stormcrawler.helper.initialisation.base.AbstractClass;
+import org.apache.stormcrawler.helper.initialisation.base.AbstractClass;
public class ClassWithoutValidConstructor extends AbstractClass {
public ClassWithoutValidConstructor(int i) {
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/FinalClassToInitialize.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/FinalClassToInitialize.java
similarity index 93%
rename from core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/FinalClassToInitialize.java
rename to core/src/test/java/org/apache/stormcrawler/helper/initialisation/FinalClassToInitialize.java
index b05b79f8..ff997d4f 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/FinalClassToInitialize.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/FinalClassToInitialize.java
@@ -12,6 +12,6 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.helper.initialisation;
+package org.apache.stormcrawler.helper.initialisation;
public final class FinalClassToInitialize {}
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/SimpleOpenClass.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/SimpleOpenClass.java
similarity index 92%
rename from core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/SimpleOpenClass.java
rename to core/src/test/java/org/apache/stormcrawler/helper/initialisation/SimpleOpenClass.java
index a6815e59..4a24a06b 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/SimpleOpenClass.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/SimpleOpenClass.java
@@ -12,6 +12,6 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.helper.initialisation;
+package org.apache.stormcrawler.helper.initialisation;
public class SimpleOpenClass {}
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/base/AbstractClass.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/AbstractClass.java
similarity index 94%
rename from core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/base/AbstractClass.java
rename to core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/AbstractClass.java
index b6fd332a..e9f5893c 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/base/AbstractClass.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/AbstractClass.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.helper.initialisation.base;
+package org.apache.stormcrawler.helper.initialisation.base;
public abstract class AbstractClass {
private int variable;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/base/ITestInterface.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/ITestInterface.java
similarity index 92%
rename from core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/base/ITestInterface.java
rename to core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/ITestInterface.java
index ad6f2ac9..742c8f94 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/base/ITestInterface.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/ITestInterface.java
@@ -12,6 +12,6 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.helper.initialisation.base;
+package org.apache.stormcrawler.helper.initialisation.base;
public interface ITestInterface {}
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/base/OpenClassWithAbstractClassAndInterface.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/OpenClassWithAbstractClassAndInterface.java
similarity index 93%
rename from core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/base/OpenClassWithAbstractClassAndInterface.java
rename to core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/OpenClassWithAbstractClassAndInterface.java
index 55ad926f..34c10ce9 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/helper/initialisation/base/OpenClassWithAbstractClassAndInterface.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/OpenClassWithAbstractClassAndInterface.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.helper.initialisation.base;
+package org.apache.stormcrawler.helper.initialisation.base;
public class OpenClassWithAbstractClassAndInterface extends AbstractClass
implements ITestInterface {}
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/indexer/BasicIndexingTest.java b/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java
similarity index 98%
rename from core/src/test/java/com/digitalpebble/stormcrawler/indexer/BasicIndexingTest.java
rename to core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java
index 809f274d..4ebb3e75 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/indexer/BasicIndexingTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java
@@ -15,15 +15,15 @@
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.indexer;
+package org.apache.stormcrawler.indexer;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.indexing.AbstractIndexerBolt;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/indexer/DummyIndexer.java b/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java
similarity index 94%
rename from core/src/test/java/com/digitalpebble/stormcrawler/indexer/DummyIndexer.java
rename to core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java
index d9409110..42112ac7 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/indexer/DummyIndexer.java
+++ b/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.indexer;
+package org.apache.stormcrawler.indexer;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.indexing.AbstractIndexerBolt;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -23,6 +21,8 @@ import org.apache.commons.lang.StringUtils;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
public class DummyIndexer extends AbstractIndexerBolt {
OutputCollector _collector;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/indexer/IndexerTester.java b/core/src/test/java/org/apache/stormcrawler/indexer/IndexerTester.java
similarity index 89%
rename from core/src/test/java/com/digitalpebble/stormcrawler/indexer/IndexerTester.java
rename to core/src/test/java/org/apache/stormcrawler/indexer/IndexerTester.java
index 7515cf43..ea0c0f18 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/indexer/IndexerTester.java
+++ b/core/src/test/java/org/apache/stormcrawler/indexer/IndexerTester.java
@@ -15,19 +15,19 @@
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.indexer;
+package org.apache.stormcrawler.indexer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestOutputCollector;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.indexing.AbstractIndexerBolt;
import java.io.IOException;
import java.util.Map;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestOutputCollector;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
import org.junit.After;
public class IndexerTester {
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/json/JsoupFilterTest.java b/core/src/test/java/org/apache/stormcrawler/json/JsoupFilterTest.java
similarity index 90%
rename from core/src/test/java/com/digitalpebble/stormcrawler/json/JsoupFilterTest.java
rename to core/src/test/java/org/apache/stormcrawler/json/JsoupFilterTest.java
index 1c112cdd..b6ae472b 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/json/JsoupFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/json/JsoupFilterTest.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.json;
+package org.apache.stormcrawler.json;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.bolt.JSoupParserBolt;
-import com.digitalpebble.stormcrawler.parse.ParsingTester;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.apache.storm.task.OutputCollector;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.bolt.JSoupParserBolt;
+import org.apache.stormcrawler.parse.ParsingTester;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/jsoup/JSoupFiltersTest.java b/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java
similarity index 93%
rename from core/src/test/java/com/digitalpebble/stormcrawler/jsoup/JSoupFiltersTest.java
rename to core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java
index 5b079ca8..84e9a015 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/jsoup/JSoupFiltersTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.jsoup;
+package org.apache.stormcrawler.jsoup;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.bolt.JSoupParserBolt;
-import com.digitalpebble.stormcrawler.parse.ParsingTester;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.apache.storm.task.OutputCollector;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.bolt.JSoupParserBolt;
+import org.apache.stormcrawler.parse.ParsingTester;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/parse/DuplicateLinksTest.java b/core/src/test/java/org/apache/stormcrawler/parse/DuplicateLinksTest.java
similarity index 87%
rename from core/src/test/java/com/digitalpebble/stormcrawler/parse/DuplicateLinksTest.java
rename to core/src/test/java/org/apache/stormcrawler/parse/DuplicateLinksTest.java
index dd99bba9..21c49e37 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/parse/DuplicateLinksTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/DuplicateLinksTest.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.bolt.JSoupParserBolt;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.storm.task.OutputCollector;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.bolt.JSoupParserBolt;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/parse/ParsingTester.java b/core/src/test/java/org/apache/stormcrawler/parse/ParsingTester.java
similarity index 93%
rename from core/src/test/java/com/digitalpebble/stormcrawler/parse/ParsingTester.java
rename to core/src/test/java/org/apache/stormcrawler/parse/ParsingTester.java
index fcab86ca..37f1f42b 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/parse/ParsingTester.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/ParsingTester.java
@@ -12,14 +12,11 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestOutputCollector;
-import com.digitalpebble.stormcrawler.TestUtil;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.HashMap;
@@ -28,6 +25,9 @@ import org.apache.commons.io.IOUtils;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestOutputCollector;
+import org.apache.stormcrawler.TestUtil;
import org.junit.After;
public class ParsingTester {
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/parse/StackOverflowTest.java b/core/src/test/java/org/apache/stormcrawler/parse/StackOverflowTest.java
similarity index 91%
rename from core/src/test/java/com/digitalpebble/stormcrawler/parse/StackOverflowTest.java
rename to core/src/test/java/org/apache/stormcrawler/parse/StackOverflowTest.java
index f6b875d2..e047d6d2 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/parse/StackOverflowTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/StackOverflowTest.java
@@ -12,15 +12,15 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
import static org.junit.Assert.assertNotNull;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.bolt.JSoupParserBolt;
import java.io.IOException;
import java.util.List;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.bolt.JSoupParserBolt;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/parse/TextExtractorTest.java b/core/src/test/java/org/apache/stormcrawler/parse/TextExtractorTest.java
similarity index 98%
rename from core/src/test/java/com/digitalpebble/stormcrawler/parse/TextExtractorTest.java
rename to core/src/test/java/org/apache/stormcrawler/parse/TextExtractorTest.java
index 79e92d31..6dfbb205 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/parse/TextExtractorTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/TextExtractorTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse;
+package org.apache.stormcrawler.parse;
import static org.junit.Assert.assertEquals;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/CSVMetadataFilterTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/CSVMetadataFilterTest.java
similarity index 88%
rename from core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/CSVMetadataFilterTest.java
rename to core/src/test/java/org/apache/stormcrawler/parse/filter/CSVMetadataFilterTest.java
index bac90b81..d4a17242 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/CSVMetadataFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/CSVMetadataFilterTest.java
@@ -12,13 +12,13 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.bolt.JSoupParserBolt;
-import com.digitalpebble.stormcrawler.parse.ParsingTester;
import java.io.IOException;
import java.util.List;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.bolt.JSoupParserBolt;
+import org.apache.stormcrawler.parse.ParsingTester;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/CollectionTaggerTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/CollectionTaggerTest.java
similarity index 91%
rename from core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/CollectionTaggerTest.java
rename to core/src/test/java/org/apache/stormcrawler/parse/filter/CollectionTaggerTest.java
index 5707d370..fcb12ae5 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/CollectionTaggerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/CollectionTaggerTest.java
@@ -1,9 +1,9 @@
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
import com.fasterxml.jackson.databind.node.NullNode;
import java.io.IOException;
import java.util.HashMap;
+import org.apache.stormcrawler.parse.ParseResult;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/SubDocumentsFilterTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsFilterTest.java
similarity index 87%
rename from core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/SubDocumentsFilterTest.java
rename to core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsFilterTest.java
index 0b884d72..434b768e 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/SubDocumentsFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsFilterTest.java
@@ -12,14 +12,14 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.bolt.JSoupParserBolt;
-import com.digitalpebble.stormcrawler.parse.ParsingTester;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.bolt.JSoupParserBolt;
+import org.apache.stormcrawler.parse.ParsingTester;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/SubDocumentsParseFilter.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsParseFilter.java
similarity index 92%
rename from core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/SubDocumentsParseFilter.java
rename to core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsParseFilter.java
index 250791f8..a7dbdcd6 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/SubDocumentsParseFilter.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsParseFilter.java
@@ -12,11 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.parse.ParseData;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -24,6 +21,9 @@ import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
+import org.apache.stormcrawler.parse.ParseData;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.DocumentFragment;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/XPathFilterTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java
similarity index 93%
rename from core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/XPathFilterTest.java
rename to core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java
index 236d6e3a..dc1cdfde 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/parse/filter/XPathFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java
@@ -12,13 +12,13 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.bolt.JSoupParserBolt;
-import com.digitalpebble.stormcrawler.parse.ParsingTester;
import java.io.IOException;
import java.util.List;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.bolt.JSoupParserBolt;
+import org.apache.stormcrawler.parse.ParsingTester;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/persistence/AdaptiveSchedulerTest.java b/core/src/test/java/org/apache/stormcrawler/persistence/AdaptiveSchedulerTest.java
similarity index 97%
rename from core/src/test/java/com/digitalpebble/stormcrawler/persistence/AdaptiveSchedulerTest.java
rename to core/src/test/java/org/apache/stormcrawler/persistence/AdaptiveSchedulerTest.java
index b66af26b..78f3b24b 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/persistence/AdaptiveSchedulerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/persistence/AdaptiveSchedulerTest.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence;
+package org.apache.stormcrawler.persistence;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
import java.net.MalformedURLException;
import java.time.Instant;
import java.time.ZonedDateTime;
@@ -28,6 +26,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.lang.time.DateUtils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.HttpHeaders;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/persistence/DefaultSchedulerTest.java b/core/src/test/java/org/apache/stormcrawler/persistence/DefaultSchedulerTest.java
similarity index 97%
rename from core/src/test/java/com/digitalpebble/stormcrawler/persistence/DefaultSchedulerTest.java
rename to core/src/test/java/org/apache/stormcrawler/persistence/DefaultSchedulerTest.java
index 632d19cd..5e9f3390 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/persistence/DefaultSchedulerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/persistence/DefaultSchedulerTest.java
@@ -12,9 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence;
+package org.apache.stormcrawler.persistence;
-import com.digitalpebble.stormcrawler.Metadata;
import java.net.MalformedURLException;
import java.util.Calendar;
import java.util.Date;
@@ -22,6 +21,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.lang.time.DateUtils;
+import org.apache.stormcrawler.Metadata;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/persistence/URLBufferTest.java b/core/src/test/java/org/apache/stormcrawler/persistence/URLBufferTest.java
similarity index 90%
rename from core/src/test/java/com/digitalpebble/stormcrawler/persistence/URLBufferTest.java
rename to core/src/test/java/org/apache/stormcrawler/persistence/URLBufferTest.java
index bb134989..7bce5e88 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/persistence/URLBufferTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/persistence/URLBufferTest.java
@@ -12,13 +12,13 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.persistence;
+package org.apache.stormcrawler.persistence;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.urlbuffer.PriorityURLBuffer;
-import com.digitalpebble.stormcrawler.persistence.urlbuffer.SimpleURLBuffer;
-import com.digitalpebble.stormcrawler.persistence.urlbuffer.URLBuffer;
import java.net.MalformedURLException;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.urlbuffer.PriorityURLBuffer;
+import org.apache.stormcrawler.persistence.urlbuffer.SimpleURLBuffer;
+import org.apache.stormcrawler.persistence.urlbuffer.URLBuffer;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/protocol/AbstractProtocolTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/AbstractProtocolTest.java
similarity index 98%
rename from core/src/test/java/com/digitalpebble/stormcrawler/protocol/AbstractProtocolTest.java
rename to core/src/test/java/org/apache/stormcrawler/protocol/AbstractProtocolTest.java
index 781807ad..0beb67c6 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/protocol/AbstractProtocolTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/AbstractProtocolTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
import static org.junit.Assert.assertNotEquals;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/protocol/DelegationProtocolTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/DelegationProtocolTest.java
similarity index 86%
rename from core/src/test/java/com/digitalpebble/stormcrawler/protocol/DelegationProtocolTest.java
rename to core/src/test/java/org/apache/stormcrawler/protocol/DelegationProtocolTest.java
index ef10e7b0..30ae94f1 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/protocol/DelegationProtocolTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/DelegationProtocolTest.java
@@ -12,22 +12,20 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.DelegatorProtocol.FilteredProtocol;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.io.FileNotFoundException;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.DelegatorProtocol.FilteredProtocol;
+import org.apache.stormcrawler.util.ConfUtils;
import org.junit.Assert;
import org.junit.Test;
public class DelegationProtocolTest {
- private static final String OKHTTP =
- "com.digitalpebble.stormcrawler.protocol.okhttp.HttpProtocol";
- private static final String APACHE =
- "com.digitalpebble.stormcrawler.protocol.httpclient.HttpProtocol";
+ private static final String OKHTTP = "org.apache.stormcrawler.protocol.okhttp.HttpProtocol";
+ private static final String APACHE = "org.apache.stormcrawler.protocol.httpclient.HttpProtocol";
@Test
public void getProtocolTest() throws FileNotFoundException {
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/protocol/DummyProtocol.java b/core/src/test/java/org/apache/stormcrawler/protocol/DummyProtocol.java
similarity index 92%
rename from core/src/test/java/com/digitalpebble/stormcrawler/protocol/DummyProtocol.java
rename to core/src/test/java/org/apache/stormcrawler/protocol/DummyProtocol.java
index 27ebdcb0..2417a966 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/protocol/DummyProtocol.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/DummyProtocol.java
@@ -12,11 +12,11 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
-import com.digitalpebble.stormcrawler.Metadata;
import crawlercommons.robots.BaseRobotRules;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
public class DummyProtocol implements Protocol {
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/protocol/HttpHeadersTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HttpHeadersTest.java
similarity index 96%
rename from core/src/test/java/com/digitalpebble/stormcrawler/protocol/HttpHeadersTest.java
rename to core/src/test/java/org/apache/stormcrawler/protocol/HttpHeadersTest.java
index 4cae727d..385a22b2 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/protocol/HttpHeadersTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/HttpHeadersTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/protocol/HttpRobotRulesParserTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserTest.java
similarity index 99%
rename from core/src/test/java/com/digitalpebble/stormcrawler/protocol/HttpRobotRulesParserTest.java
rename to core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserTest.java
index ec837e7a..3681aa3b 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/protocol/HttpRobotRulesParserTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol;
+package org.apache.stormcrawler.protocol;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
import static com.github.tomakehurst.wiremock.client.WireMock.configureFor;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/protocol/selenium/ProtocolTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/selenium/ProtocolTest.java
similarity index 96%
rename from core/src/test/java/com/digitalpebble/stormcrawler/protocol/selenium/ProtocolTest.java
rename to core/src/test/java/org/apache/stormcrawler/protocol/selenium/ProtocolTest.java
index 28f36a4d..253c9f79 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/protocol/selenium/ProtocolTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/selenium/ProtocolTest.java
@@ -12,13 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.protocol.selenium;
+package org.apache.stormcrawler.protocol.selenium;
import static org.awaitility.Awaitility.await;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.AbstractProtocolTest;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
@@ -27,6 +24,9 @@ import java.util.Map;
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.storm.Config;
import org.apache.storm.utils.MutableObject;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.AbstractProtocolTest;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/proxy/MultiProxyManagerTest.java b/core/src/test/java/org/apache/stormcrawler/proxy/MultiProxyManagerTest.java
similarity index 99%
rename from core/src/test/java/com/digitalpebble/stormcrawler/proxy/MultiProxyManagerTest.java
rename to core/src/test/java/org/apache/stormcrawler/proxy/MultiProxyManagerTest.java
index 9d42645f..3d33a14e 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/proxy/MultiProxyManagerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/proxy/MultiProxyManagerTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.proxy;
+package org.apache.stormcrawler.proxy;
import java.io.FileWriter;
import java.io.IOException;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/proxy/SCProxyTest.java b/core/src/test/java/org/apache/stormcrawler/proxy/SCProxyTest.java
similarity index 98%
rename from core/src/test/java/com/digitalpebble/stormcrawler/proxy/SCProxyTest.java
rename to core/src/test/java/org/apache/stormcrawler/proxy/SCProxyTest.java
index e05a355b..4d500416 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/proxy/SCProxyTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/proxy/SCProxyTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.proxy;
+package org.apache.stormcrawler.proxy;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/proxy/SingleProxyManagerTest.java b/core/src/test/java/org/apache/stormcrawler/proxy/SingleProxyManagerTest.java
similarity index 97%
rename from core/src/test/java/com/digitalpebble/stormcrawler/proxy/SingleProxyManagerTest.java
rename to core/src/test/java/org/apache/stormcrawler/proxy/SingleProxyManagerTest.java
index fa6465b9..7f440e15 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/proxy/SingleProxyManagerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/proxy/SingleProxyManagerTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.proxy;
+package org.apache.stormcrawler.proxy;
import org.apache.storm.Config;
import org.junit.Assert;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/util/ConfUtilsTest.java b/core/src/test/java/org/apache/stormcrawler/util/ConfUtilsTest.java
similarity index 98%
rename from core/src/test/java/com/digitalpebble/stormcrawler/util/ConfUtilsTest.java
rename to core/src/test/java/org/apache/stormcrawler/util/ConfUtilsTest.java
index 088c1cfe..238a9b30 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/util/ConfUtilsTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/ConfUtilsTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import java.io.IOException;
import java.net.MalformedURLException;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/util/CookieConverterTest.java b/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java
similarity index 99%
rename from core/src/test/java/com/digitalpebble/stormcrawler/util/CookieConverterTest.java
rename to core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java
index 76693239..abf73e59 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/util/CookieConverterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import java.net.MalformedURLException;
import java.net.URL;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/util/InitialisationUtilTest.java b/core/src/test/java/org/apache/stormcrawler/util/InitialisationUtilTest.java
similarity index 97%
rename from core/src/test/java/com/digitalpebble/stormcrawler/util/InitialisationUtilTest.java
rename to core/src/test/java/org/apache/stormcrawler/util/InitialisationUtilTest.java
index be0a3e4b..4aee5f8b 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/util/InitialisationUtilTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/InitialisationUtilTest.java
@@ -12,10 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
-import com.digitalpebble.stormcrawler.helper.initialisation.*;
-import com.digitalpebble.stormcrawler.helper.initialisation.base.*;
+import org.apache.stormcrawler.helper.initialisation.*;
+import org.apache.stormcrawler.helper.initialisation.base.*;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/util/MetadataTransferTest.java b/core/src/test/java/org/apache/stormcrawler/util/MetadataTransferTest.java
similarity index 98%
rename from core/src/test/java/com/digitalpebble/stormcrawler/util/MetadataTransferTest.java
rename to core/src/test/java/org/apache/stormcrawler/util/MetadataTransferTest.java
index d6292a85..3b07f568 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/util/MetadataTransferTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/MetadataTransferTest.java
@@ -12,14 +12,14 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
-import com.digitalpebble.stormcrawler.Metadata;
import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.stormcrawler.Metadata;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/util/RefreshTagTest.java b/core/src/test/java/org/apache/stormcrawler/util/RefreshTagTest.java
similarity index 97%
rename from core/src/test/java/com/digitalpebble/stormcrawler/util/RefreshTagTest.java
rename to core/src/test/java/org/apache/stormcrawler/util/RefreshTagTest.java
index 009321d5..d824785a 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/util/RefreshTagTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/RefreshTagTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
import java.io.IOException;
import java.net.MalformedURLException;
diff --git a/core/src/test/java/com/digitalpebble/stormcrawler/util/RobotsTagsTest.java b/core/src/test/java/org/apache/stormcrawler/util/RobotsTagsTest.java
similarity index 95%
rename from core/src/test/java/com/digitalpebble/stormcrawler/util/RobotsTagsTest.java
rename to core/src/test/java/org/apache/stormcrawler/util/RobotsTagsTest.java
index 2cfddf9d..2c1f25c7 100644
--- a/core/src/test/java/com/digitalpebble/stormcrawler/util/RobotsTagsTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/RobotsTagsTest.java
@@ -12,10 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.util;
+package org.apache.stormcrawler.util;
-import com.digitalpebble.stormcrawler.Metadata;
import java.net.MalformedURLException;
+import org.apache.stormcrawler.Metadata;
import org.junit.Assert;
import org.junit.Test;
diff --git a/core/src/test/resources/basicurlnormalizer.json b/core/src/test/resources/basicurlnormalizer.json
index dd906a98..5f2d0f1c 100644
--- a/core/src/test/resources/basicurlnormalizer.json
+++ b/core/src/test/resources/basicurlnormalizer.json
@@ -1,6 +1,6 @@
{
- "com.digitalpebble.stormcrawler.filtering.URLFilters": [{
- "class": "com.digitalpebble.stormcrawler.filtering.basic.BasicURLNormalizer",
+ "org.apache.stormcrawler.filtering.URLFilters": [{
+ "class": "org.apache.stormcrawler.filtering.basic.BasicURLNormalizer",
"name": "BasicURLNormalizer",
"params": {
"removeAnchorPart": true,
diff --git a/core/src/test/resources/delegator-conf.yaml b/core/src/test/resources/delegator-conf.yaml
index 4422f3c0..4788b78b 100644
--- a/core/src/test/resources/delegator-conf.yaml
+++ b/core/src/test/resources/delegator-conf.yaml
@@ -1,28 +1,28 @@
config:
protocol.delegator.config:
- - className: "com.digitalpebble.stormcrawler.protocol.DummyProtocol"
+ - className: "org.apache.stormcrawler.protocol.DummyProtocol"
id: "first"
filters:
domain: "example.com"
depth: "3"
test: "true"
- - className: "com.digitalpebble.stormcrawler.protocol.DummyProtocol"
+ - className: "org.apache.stormcrawler.protocol.DummyProtocol"
id: "second"
filters:
js: "true"
- - className: "com.digitalpebble.stormcrawler.protocol.DummyProtocol"
+ - className: "org.apache.stormcrawler.protocol.DummyProtocol"
id: "third"
operator: OR
filters:
ping:
pong:
- - className: "com.digitalpebble.stormcrawler.protocol.DummyProtocol"
+ - className: "org.apache.stormcrawler.protocol.DummyProtocol"
id: "fourth"
operator: OR
regex:
- \.pdf
- \.doc
- - className: "com.digitalpebble.stormcrawler.protocol.DummyProtocol"
+ - className: "org.apache.stormcrawler.protocol.DummyProtocol"
id: "default"
diff --git a/core/src/test/resources/test.jsoupfilters.json b/core/src/test/resources/test.jsoupfilters.json
index e3316b66..8eb1da8e 100644
--- a/core/src/test/resources/test.jsoupfilters.json
+++ b/core/src/test/resources/test.jsoupfilters.json
@@ -1,7 +1,7 @@
{
- "com.digitalpebble.stormcrawler.parse.JSoupFilters" : [
+ "org.apache.stormcrawler.parse.JSoupFilters" : [
{
- "class": "com.digitalpebble.stormcrawler.jsoup.XPathFilter",
+ "class": "org.apache.stormcrawler.jsoup.XPathFilter",
"name": "XPathFilter",
"params": {
"concept": "//SPAN[@class=\"concept\"]/tidyText()",
@@ -12,14 +12,14 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.jsoup.LinkParseFilter",
+ "class": "org.apache.stormcrawler.jsoup.LinkParseFilter",
"name": "LinkParseFilter",
"params": {
"pattern": "//IMG/@src"
}
},
{
- "class" : "com.digitalpebble.stormcrawler.jsoup.LDJsonParseFilter",
+ "class" : "org.apache.stormcrawler.jsoup.LDJsonParseFilter",
"name" : "LDJsonParseFilter",
"params" : {
"streetAddress" : "/location/streetAddress"
diff --git a/core/src/test/resources/test.parsefilters.json b/core/src/test/resources/test.parsefilters.json
index 3d300622..d32c5b52 100644
--- a/core/src/test/resources/test.parsefilters.json
+++ b/core/src/test/resources/test.parsefilters.json
@@ -1,7 +1,7 @@
{
- "com.digitalpebble.stormcrawler.parse.ParseFilters": [
+ "org.apache.stormcrawler.parse.ParseFilters": [
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.XPathFilter",
+ "class": "org.apache.stormcrawler.parse.filter.XPathFilter",
"name": "XPathFilter",
"params": {
"concept": "//SPAN[@class=\"concept\"]",
@@ -12,14 +12,14 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.LDJsonParseFilter",
+ "class": "org.apache.stormcrawler.parse.filter.LDJsonParseFilter",
"name": "LDJsonParseFilter",
"params": {
"streetAddress": "/location/streetAddress"
}
},
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.CommaSeparatedToMultivaluedMetadata",
+ "class": "org.apache.stormcrawler.parse.filter.CommaSeparatedToMultivaluedMetadata",
"name": "CommaSeparatedToMultivaluedMetadata",
"params": {
"keys": ["keywords", "someValue"]
diff --git a/core/src/test/resources/test.subdocfilter.json b/core/src/test/resources/test.subdocfilter.json
index 814d8a85..7d0e84fc 100644
--- a/core/src/test/resources/test.subdocfilter.json
+++ b/core/src/test/resources/test.subdocfilter.json
@@ -1,8 +1,8 @@
{
- "com.digitalpebble.stormcrawler.parse.ParseFilters": [
+ "org.apache.stormcrawler.parse.ParseFilters": [
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.SubDocumentsParseFilter",
+ "class": "org.apache.stormcrawler.parse.filter.SubDocumentsParseFilter",
"name": "Subdocs"
}
]
-}
\ No newline at end of file
+}
diff --git a/external/aws/README.md b/external/aws/README.md
index d2353605..6edd4885 100644
--- a/external/aws/README.md
+++ b/external/aws/README.md
@@ -9,7 +9,7 @@ Add storm-crawler-aws to the dependencies of your project\:
```xml
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-aws</artifactId>
<version>XXXX</version>
</dependency>
diff --git a/external/aws/pom.xml b/external/aws/pom.xml
index d44cd2c6..7f6c5606 100644
--- a/external/aws/pom.xml
+++ b/external/aws/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-external</artifactId>
<version>2.12-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
@@ -13,7 +13,7 @@
<packaging>jar</packaging>
<name>storm-crawler-aws</name>
- <url>https://github.com/DigitalPebble/storm-crawler/tree/master/external/aws</url>
+ <url>https://github.com/apache/incubator-stormcrawler/tree/master/external/aws</url>
<description>AWS resources for StormCrawler</description>
<properties>
diff --git a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchConstants.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchConstants.java
similarity index 96%
rename from external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchConstants.java
rename to external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchConstants.java
index df70c347..86bb74e7 100644
--- a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchConstants.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchConstants.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.aws.bolt;
+package org.apache.stormcrawler.aws.bolt;
public interface CloudSearchConstants {
public static final String CLOUDSEARCH_PREFIX = "cloudsearch.";
diff --git a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
similarity index 97%
rename from external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
rename to external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
index b545d048..e3ae8b4c 100644
--- a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
@@ -12,9 +12,9 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.aws.bolt;
+package org.apache.stormcrawler.aws.bolt;
-import static com.digitalpebble.stormcrawler.Constants.StatusStreamName;
+import static org.apache.stormcrawler.Constants.StatusStreamName;
import com.amazonaws.regions.RegionUtils;
import com.amazonaws.services.cloudsearchdomain.AmazonCloudSearchDomainClient;
@@ -29,10 +29,6 @@ import com.amazonaws.services.cloudsearchv2.model.DescribeIndexFieldsRequest;
import com.amazonaws.services.cloudsearchv2.model.DescribeIndexFieldsResult;
import com.amazonaws.services.cloudsearchv2.model.DomainStatus;
import com.amazonaws.services.cloudsearchv2.model.IndexFieldStatus;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.indexing.AbstractIndexerBolt;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -59,6 +55,10 @@ import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchUtils.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchUtils.java
similarity index 98%
rename from external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchUtils.java
rename to external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchUtils.java
index 58256245..3e4852d8 100644
--- a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/bolt/CloudSearchUtils.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchUtils.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.aws.bolt;
+package org.apache.stormcrawler.aws.bolt;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
diff --git a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/s3/AbstractS3CacheBolt.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/AbstractS3CacheBolt.java
similarity index 96%
rename from external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/s3/AbstractS3CacheBolt.java
rename to external/aws/src/main/java/org/apache/stormcrawler/aws/s3/AbstractS3CacheBolt.java
index 19d65f8e..f4ec5eb1 100644
--- a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/s3/AbstractS3CacheBolt.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/AbstractS3CacheBolt.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.aws.s3;
+package org.apache.stormcrawler.aws.s3;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
@@ -20,7 +20,6 @@ import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.regions.RegionUtils;
import com.amazonaws.services.s3.AmazonS3Client;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.metric.api.MultiCountMetric;
@@ -29,6 +28,7 @@ import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
+import org.apache.stormcrawler.util.ConfUtils;
public abstract class AbstractS3CacheBolt extends BaseRichBolt {
diff --git a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/s3/S3CacheChecker.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3CacheChecker.java
similarity index 96%
rename from external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/s3/S3CacheChecker.java
rename to external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3CacheChecker.java
index 6077291d..e7145d8c 100644
--- a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/s3/S3CacheChecker.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3CacheChecker.java
@@ -12,12 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.aws.s3;
+package org.apache.stormcrawler.aws.s3;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.S3Object;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Map;
@@ -29,6 +27,8 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/s3/S3Cacher.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3Cacher.java
similarity index 97%
rename from external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/s3/S3Cacher.java
rename to external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3Cacher.java
index b7a6f3a2..4aadb86c 100644
--- a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/s3/S3Cacher.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3Cacher.java
@@ -12,13 +12,11 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.aws.s3;
+package org.apache.stormcrawler.aws.s3;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectResult;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
@@ -29,6 +27,8 @@ import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/s3/S3ContentCacher.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3ContentCacher.java
similarity index 94%
rename from external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/s3/S3ContentCacher.java
rename to external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3ContentCacher.java
index 45f3ee5a..bb282c07 100644
--- a/external/aws/src/main/java/com/digitalpebble/stormcrawler/aws/s3/S3ContentCacher.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3ContentCacher.java
@@ -12,9 +12,9 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.aws.s3;
+package org.apache.stormcrawler.aws.s3;
-import com.digitalpebble.stormcrawler.Metadata;
+import org.apache.stormcrawler.Metadata;
/** Caches byte[] content into S3 */
public class S3ContentCacher extends S3Cacher {
diff --git a/external/elasticsearch/README.md b/external/elasticsearch/README.md
index d6a7f7fc..a8cfe1a9 100644
--- a/external/elasticsearch/README.md
+++ b/external/elasticsearch/README.md
@@ -2,10 +2,10 @@ storm-crawler-elasticsearch
===========================
A collection of resources for [Elasticsearch](https://www.elastic.co/products/elasticsearch):
-* [IndexerBolt](https://github.com/DigitalPebble/storm-crawler/blob/master/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/IndexerBolt.java) for indexing documents crawled with StormCrawler
-* [Spouts](https://github.com/DigitalPebble/storm-crawler/blob/master/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/AggregationSpout.java) and [StatusUpdaterBolt](https://github.com/DigitalPebble/storm-crawler/blob/master/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/StatusUpdaterBolt.java) for persisting URL information in recursive crawls
-* [MetricsConsumer](https://github.com/DigitalPebble/storm-crawler/blob/master/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/metrics/MetricsConsumer.java)
-* [StatusMetricsBolt](https://github.com/DigitalPebble/storm-crawler/blob/master/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/metrics/StatusMetricsBolt.java) for sending the breakdown of URLs per status as metrics and display its evolution over time.
+* [IndexerBolt](https://github.org/apache/incubator-stormcrawler/blob/master/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/bolt/IndexerBolt.java) for indexing documents crawled with StormCrawler
+* [Spouts](https://github.org/apache/incubator-stormcrawler/blob/master/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/AggregationSpout.java) and [StatusUpdaterBolt](https://github.org/apache/incubator-stormcrawler/blob/master/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/StatusUpdaterBolt.java) for persisting URL information in recursive crawls
+* [MetricsConsumer](https://github.org/apache/incubator-stormcrawler/blob/master/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/metrics/MetricsConsumer.java)
+* [StatusMetricsBolt](https://github.org/apache/incubator-stormcrawler/blob/master/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/metrics/StatusMetricsBolt.java) for sending the breakdown of URLs per status as metrics and display its evolution over time.
as well as an archetype containing a basic crawl topology and its configuration.
@@ -16,7 +16,7 @@ Getting started
Use the archetype for Elasticsearch with:
-`mvn archetype:generate -DarchetypeGroupId=com.digitalpebble.stormcrawler -DarchetypeArtifactId=storm-crawler-elasticsearch-archetype -DarchetypeVersion=2.11`
+`mvn archetype:generate -DarchetypeGroupId=org.apache.stormcrawler -DarchetypeArtifactId=storm-crawler-elasticsearch-archetype -DarchetypeVersion=2.11`
You'll be asked to enter a groupId (e.g. com.mycompany.crawler), an artefactId (e.g. stormcrawler), a version, a package name and details about the user agent to use.
@@ -56,7 +56,7 @@ The crawler config YAML must be updated to use an optional argument as shown bel
```
#Metrics consumers:
topology.metrics.consumer.register:
- - class: "com.digitalpebble.stormcrawler.elasticsearch.metrics.MetricsConsumer"
+ - class: "org.apache.stormcrawler.elasticsearch.metrics.MetricsConsumer"
parallelism.hint: 1
argument: "yyyy-MM-dd"
```
diff --git a/external/elasticsearch/archetype/pom.xml b/external/elasticsearch/archetype/pom.xml
index 13d58a54..1468b4b5 100644
--- a/external/elasticsearch/archetype/pom.xml
+++ b/external/elasticsearch/archetype/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler</artifactId>
<version>2.12-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
diff --git a/external/elasticsearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/elasticsearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml
index 45645409..d21929e7 100644
--- a/external/elasticsearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml
+++ b/external/elasticsearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml
@@ -17,8 +17,8 @@ config:
# mandatory when using Flux
topology.kryo.register:
- - com.digitalpebble.stormcrawler.Metadata
- - com.digitalpebble.stormcrawler.persistence.Status
+ - org.apache.stormcrawler.Metadata
+ - org.apache.stormcrawler.persistence.Status
# Lists the metadata to transfer to outlinks
# Used by Fetcher and SiteMapParser for redirections,
@@ -54,8 +54,8 @@ config:
# Finally, an email so that they can get in touch with you
http.agent.email: "${http-agent-email}"
- http.protocol.implementation: "com.digitalpebble.stormcrawler.protocol.okhttp.HttpProtocol"
- https.protocol.implementation: "com.digitalpebble.stormcrawler.protocol.okhttp.HttpProtocol"
+ http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
+ https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
# The maximum number of bytes for returned HTTP response bodies.
# The fetched page will be trimmed to 65KB in this case
diff --git a/external/elasticsearch/archetype/src/main/resources/archetype-resources/es-conf.yaml b/external/elasticsearch/archetype/src/main/resources/archetype-resources/es-conf.yaml
index 0aaabe44..848173f5 100644
--- a/external/elasticsearch/archetype/src/main/resources/archetype-resources/es-conf.yaml
+++ b/external/elasticsearch/archetype/src/main/resources/archetype-resources/es-conf.yaml
@@ -89,7 +89,7 @@ config:
es.status.recentDate.min.gap: -1
topology.metrics.consumer.register:
- - class: "com.digitalpebble.stormcrawler.elasticsearch.metrics.MetricsConsumer"
+ - class: "org.apache.stormcrawler.elasticsearch.metrics.MetricsConsumer"
parallelism.hint: 1
#whitelist:
# - "fetcher_counter"
diff --git a/external/elasticsearch/archetype/src/main/resources/archetype-resources/es-crawler.flux b/external/elasticsearch/archetype/src/main/resources/archetype-resources/es-crawler.flux
index 54b3ed72..92d59672 100644
--- a/external/elasticsearch/archetype/src/main/resources/archetype-resources/es-crawler.flux
+++ b/external/elasticsearch/archetype/src/main/resources/archetype-resources/es-crawler.flux
@@ -15,39 +15,39 @@ includes:
spouts:
- id: "spout"
- className: "com.digitalpebble.stormcrawler.elasticsearch.persistence.AggregationSpout"
+ className: "org.apache.stormcrawler.elasticsearch.persistence.AggregationSpout"
parallelism: 10
bolts:
- id: "partitioner"
- className: "com.digitalpebble.stormcrawler.bolt.URLPartitionerBolt"
+ className: "org.apache.stormcrawler.bolt.URLPartitionerBolt"
parallelism: 1
- id: "fetcher"
- className: "com.digitalpebble.stormcrawler.bolt.FetcherBolt"
+ className: "org.apache.stormcrawler.bolt.FetcherBolt"
parallelism: 1
- id: "sitemap"
- className: "com.digitalpebble.stormcrawler.bolt.SiteMapParserBolt"
+ className: "org.apache.stormcrawler.bolt.SiteMapParserBolt"
parallelism: 1
- id: "parse"
- className: "com.digitalpebble.stormcrawler.bolt.JSoupParserBolt"
+ className: "org.apache.stormcrawler.bolt.JSoupParserBolt"
parallelism: 1
- id: "shunt"
- className: "com.digitalpebble.stormcrawler.tika.RedirectionBolt"
+ className: "org.apache.stormcrawler.tika.RedirectionBolt"
parallelism: 1
- id: "tika"
- className: "com.digitalpebble.stormcrawler.tika.ParserBolt"
+ className: "org.apache.stormcrawler.tika.ParserBolt"
parallelism: 1
- id: "index"
- className: "com.digitalpebble.stormcrawler.elasticsearch.bolt.IndexerBolt"
+ className: "org.apache.stormcrawler.elasticsearch.bolt.IndexerBolt"
parallelism: 1
- id: "status"
- className: "com.digitalpebble.stormcrawler.elasticsearch.persistence.StatusUpdaterBolt"
+ className: "org.apache.stormcrawler.elasticsearch.persistence.StatusUpdaterBolt"
parallelism: 1
- id: "deleter"
- className: "com.digitalpebble.stormcrawler.elasticsearch.bolt.DeletionBolt"
+ className: "org.apache.stormcrawler.elasticsearch.bolt.DeletionBolt"
parallelism: 1
- id: "status_metrics"
- className: "com.digitalpebble.stormcrawler.elasticsearch.metrics.StatusMetricsBolt"
+ className: "org.apache.stormcrawler.elasticsearch.metrics.StatusMetricsBolt"
parallelism: 1
streams:
diff --git a/external/elasticsearch/archetype/src/main/resources/archetype-resources/es-injection.flux b/external/elasticsearch/archetype/src/main/resources/archetype-resources/es-injection.flux
index 1a7ec5ec..ad0e772b 100644
--- a/external/elasticsearch/archetype/src/main/resources/archetype-resources/es-injection.flux
+++ b/external/elasticsearch/archetype/src/main/resources/archetype-resources/es-injection.flux
@@ -15,7 +15,7 @@ includes:
spouts:
- id: "filespout"
- className: "com.digitalpebble.stormcrawler.spout.FileSpout"
+ className: "org.apache.stormcrawler.spout.FileSpout"
parallelism: 1
constructorArgs:
- "."
@@ -24,11 +24,11 @@ spouts:
bolts:
- id: "filter"
- className: "com.digitalpebble.stormcrawler.bolt.URLFilterBolt"
+ className: "org.apache.stormcrawler.bolt.URLFilterBolt"
parallelism: 1
- id: "status"
- className: "com.digitalpebble.stormcrawler.elasticsearch.persistence.StatusUpdaterBolt"
+ className: "org.apache.stormcrawler.elasticsearch.persistence.StatusUpdaterBolt"
parallelism: 1
streams:
@@ -45,6 +45,6 @@ streams:
streamId: "status"
type: CUSTOM
customClass:
- className: "com.digitalpebble.stormcrawler.util.URLStreamGrouping"
+ className: "org.apache.stormcrawler.util.URLStreamGrouping"
constructorArgs:
- "byDomain"
diff --git a/external/elasticsearch/archetype/src/main/resources/archetype-resources/pom.xml b/external/elasticsearch/archetype/src/main/resources/archetype-resources/pom.xml
index 2e4aefb0..ffa65609 100644
--- a/external/elasticsearch/archetype/src/main/resources/archetype-resources/pom.xml
+++ b/external/elasticsearch/archetype/src/main/resources/archetype-resources/pom.xml
@@ -100,12 +100,12 @@
<dependencies>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-core</artifactId>
<version>${stormcrawler.version}</version>
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-elasticsearch</artifactId>
<version>${stormcrawler.version}</version>
</dependency>
@@ -121,7 +121,7 @@
<version>${storm.version}</version>
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-tika</artifactId>
<version>${stormcrawler.version}</version>
</dependency>
diff --git a/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/java/ESCrawlTopology.java b/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/java/ESCrawlTopology.java
index 21d38c68..c9eeaf71 100644
--- a/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/java/ESCrawlTopology.java
+++ b/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/java/ESCrawlTopology.java
@@ -23,24 +23,24 @@ import org.apache.storm.metric.LoggingMetricsConsumer;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
-import com.digitalpebble.stormcrawler.ConfigurableTopology;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.bolt.FetcherBolt;
-import com.digitalpebble.stormcrawler.bolt.JSoupParserBolt;
-import com.digitalpebble.stormcrawler.bolt.SiteMapParserBolt;
-import com.digitalpebble.stormcrawler.bolt.URLFilterBolt;
-import com.digitalpebble.stormcrawler.bolt.URLPartitionerBolt;
-import com.digitalpebble.stormcrawler.elasticsearch.bolt.DeletionBolt;
-import com.digitalpebble.stormcrawler.elasticsearch.bolt.IndexerBolt;
-import com.digitalpebble.stormcrawler.elasticsearch.metrics.MetricsConsumer;
-import com.digitalpebble.stormcrawler.elasticsearch.metrics.StatusMetricsBolt;
-import com.digitalpebble.stormcrawler.elasticsearch.persistence.AggregationSpout;
-import com.digitalpebble.stormcrawler.elasticsearch.persistence.StatusUpdaterBolt;
-import com.digitalpebble.stormcrawler.spout.FileSpout;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.URLStreamGrouping;
-import com.digitalpebble.stormcrawler.tika.ParserBolt;
-import com.digitalpebble.stormcrawler.tika.RedirectionBolt;
+import org.apache.stormcrawler.ConfigurableTopology;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.bolt.FetcherBolt;
+import org.apache.stormcrawler.bolt.JSoupParserBolt;
+import org.apache.stormcrawler.bolt.SiteMapParserBolt;
+import org.apache.stormcrawler.bolt.URLFilterBolt;
+import org.apache.stormcrawler.bolt.URLPartitionerBolt;
+import org.apache.stormcrawler.elasticsearch.bolt.DeletionBolt;
+import org.apache.stormcrawler.elasticsearch.bolt.IndexerBolt;
+import org.apache.stormcrawler.elasticsearch.metrics.MetricsConsumer;
+import org.apache.stormcrawler.elasticsearch.metrics.StatusMetricsBolt;
+import org.apache.stormcrawler.elasticsearch.persistence.AggregationSpout;
+import org.apache.stormcrawler.elasticsearch.persistence.StatusUpdaterBolt;
+import org.apache.stormcrawler.spout.FileSpout;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLStreamGrouping;
+import org.apache.stormcrawler.tika.ParserBolt;
+import org.apache.stormcrawler.tika.RedirectionBolt;
/**
* Dummy topology to play with the spouts and bolts on ElasticSearch
diff --git a/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json b/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json
index c48b4134..4d87d8d5 100644
--- a/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json
+++ b/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json
@@ -1,7 +1,7 @@
{
- "com.digitalpebble.stormcrawler.parse.JSoupFilters": [
+ "org.apache.stormcrawler.parse.JSoupFilters": [
{
- "class": "com.digitalpebble.stormcrawler.jsoup.XPathFilter",
+ "class": "org.apache.stormcrawler.jsoup.XPathFilter",
"name": "XPathFilter",
"params": {
"canonical": "//*[@rel=\"canonical\"]/@href",
@@ -17,7 +17,7 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.jsoup.LinkParseFilter",
+ "class": "org.apache.stormcrawler.jsoup.LinkParseFilter",
"name": "LinkParseFilter",
"params": {
"pattern": "//FRAME/@src"
diff --git a/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json b/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json
index 04eaa895..5d525830 100644
--- a/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json
+++ b/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json
@@ -1,7 +1,7 @@
{
- "com.digitalpebble.stormcrawler.parse.ParseFilters": [
+ "org.apache.stormcrawler.parse.ParseFilters": [
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.DomainParseFilter",
+ "class": "org.apache.stormcrawler.parse.filter.DomainParseFilter",
"name": "DomainParseFilter",
"params": {
"key": "domain",
@@ -9,11 +9,11 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.MimeTypeNormalization",
+ "class": "org.apache.stormcrawler.parse.filter.MimeTypeNormalization",
"name": "MimeTypeNormalization"
},
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.CommaSeparatedToMultivaluedMetadata",
+ "class": "org.apache.stormcrawler.parse.filter.CommaSeparatedToMultivaluedMetadata",
"name": "CommaSeparatedToMultivaluedMetadata",
"params": {
"keys": ["parse.keywords"]
diff --git a/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json b/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json
index ec6740ef..6098631b 100644
--- a/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json
+++ b/external/elasticsearch/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json
@@ -1,7 +1,7 @@
{
- "com.digitalpebble.stormcrawler.filtering.URLFilters": [
+ "org.apache.stormcrawler.filtering.URLFilters": [
{
- "class": "com.digitalpebble.stormcrawler.filtering.basic.BasicURLFilter",
+ "class": "org.apache.stormcrawler.filtering.basic.BasicURLFilter",
"name": "BasicURLFilter",
"params": {
"maxPathRepetition": 3,
@@ -9,14 +9,14 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.depth.MaxDepthFilter",
+ "class": "org.apache.stormcrawler.filtering.depth.MaxDepthFilter",
"name": "MaxDepthFilter",
"params": {
"maxDepth": -1
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.basic.BasicURLNormalizer",
+ "class": "org.apache.stormcrawler.filtering.basic.BasicURLNormalizer",
"name": "BasicURLNormalizer",
"params": {
"removeAnchorPart": true,
@@ -27,7 +27,7 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.host.HostURLFilter",
+ "class": "org.apache.stormcrawler.filtering.host.HostURLFilter",
"name": "HostURLFilter",
"params": {
"ignoreOutsideHost": false,
@@ -35,25 +35,25 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.regex.RegexURLNormalizer",
+ "class": "org.apache.stormcrawler.filtering.regex.RegexURLNormalizer",
"name": "RegexURLNormalizer",
"params": {
"regexNormalizerFile": "default-regex-normalizers.xml"
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.regex.RegexURLFilter",
+ "class": "org.apache.stormcrawler.filtering.regex.RegexURLFilter",
"name": "RegexURLFilter",
"params": {
"regexFilterFile": "default-regex-filters.txt"
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.basic.SelfURLFilter",
+ "class": "org.apache.stormcrawler.filtering.basic.SelfURLFilter",
"name": "SelfURLFilter"
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.sitemap.SitemapFilter",
+ "class": "org.apache.stormcrawler.filtering.sitemap.SitemapFilter",
"name": "SitemapFilter"
}
]
diff --git a/external/elasticsearch/pom.xml b/external/elasticsearch/pom.xml
index 349096f6..0b0b8232 100644
--- a/external/elasticsearch/pom.xml
+++ b/external/elasticsearch/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-external</artifactId>
<version>2.12-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
@@ -17,7 +17,7 @@
<packaging>jar</packaging>
<name>storm-crawler-elasticsearch</name>
- <url>https://github.com/DigitalPebble/storm-crawler/tree/master/external/elasticsearch</url>
+ <url>https://github.com/apache/incubator-stormcrawler/tree/master/external/elasticsearch</url>
<description>Elasticsearch resources for StormCrawler</description>
<build>
@@ -62,7 +62,7 @@
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/BulkItemResponseToFailedFlag.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/BulkItemResponseToFailedFlag.java
similarity index 98%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/BulkItemResponseToFailedFlag.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/BulkItemResponseToFailedFlag.java
index 2c80b236..d9492215 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/BulkItemResponseToFailedFlag.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/BulkItemResponseToFailedFlag.java
@@ -1,4 +1,4 @@
-package com.digitalpebble.stormcrawler.elasticsearch;
+package org.apache.stormcrawler.elasticsearch;
import java.io.IOException;
import org.elasticsearch.action.DocWriteRequest;
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/ElasticSearchConnection.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/ElasticSearchConnection.java
similarity index 99%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/ElasticSearchConnection.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/ElasticSearchConnection.java
index b102fff1..b79cc501 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/ElasticSearchConnection.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/ElasticSearchConnection.java
@@ -12,12 +12,11 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch;
+package org.apache.stormcrawler.elasticsearch;
import static org.elasticsearch.client.RestClientBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS;
import static org.elasticsearch.client.RestClientBuilder.DEFAULT_SOCKET_TIMEOUT_MILLIS;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@@ -30,6 +29,7 @@ import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.storm.shade.org.apache.commons.lang.StringUtils;
+import org.apache.stormcrawler.util.ConfUtils;
import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/DeletionBolt.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/bolt/DeletionBolt.java
similarity index 93%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/DeletionBolt.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/bolt/DeletionBolt.java
index 78aca720..cbdfa9f9 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/DeletionBolt.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/bolt/DeletionBolt.java
@@ -1,8 +1,5 @@
-package com.digitalpebble.stormcrawler.elasticsearch.bolt;
+package org.apache.stormcrawler.elasticsearch.bolt;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.elasticsearch.ElasticSearchConnection;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Map;
@@ -11,6 +8,9 @@ import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.elasticsearch.ElasticSearchConnection;
+import org.apache.stormcrawler.util.ConfUtils;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/IndexerBolt.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/bolt/IndexerBolt.java
similarity index 96%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/IndexerBolt.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/bolt/IndexerBolt.java
index 5728be09..4509282b 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/IndexerBolt.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/bolt/IndexerBolt.java
@@ -12,19 +12,11 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.bolt;
+package org.apache.stormcrawler.elasticsearch.bolt;
-import static com.digitalpebble.stormcrawler.Constants.StatusStreamName;
+import static org.apache.stormcrawler.Constants.StatusStreamName;
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.elasticsearch.BulkItemResponseToFailedFlag;
-import com.digitalpebble.stormcrawler.elasticsearch.ElasticSearchConnection;
-import com.digitalpebble.stormcrawler.indexing.AbstractIndexerBolt;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.PerSecondReducer;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
@@ -41,6 +33,14 @@ import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.elasticsearch.BulkItemResponseToFailedFlag;
+import org.apache.stormcrawler.elasticsearch.ElasticSearchConnection;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.PerSecondReducer;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkProcessor;
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/filtering/JSONURLFilterWrapper.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/filtering/JSONURLFilterWrapper.java
similarity index 93%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/filtering/JSONURLFilterWrapper.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/filtering/JSONURLFilterWrapper.java
index 566bcba2..f07c67f9 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/filtering/JSONURLFilterWrapper.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/filtering/JSONURLFilterWrapper.java
@@ -12,18 +12,18 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.filtering;
+package org.apache.stormcrawler.elasticsearch.filtering;
-import com.digitalpebble.stormcrawler.JSONResource;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.elasticsearch.ElasticSearchConnection;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.ByteArrayInputStream;
import java.net.URL;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
+import org.apache.stormcrawler.JSONResource;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.elasticsearch.ElasticSearchConnection;
+import org.apache.stormcrawler.filtering.URLFilter;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.RequestOptions;
@@ -43,12 +43,12 @@ import org.slf4j.LoggerFactory;
*
* <pre>
* {
- * "class": "com.digitalpebble.stormcrawler.elasticsearch.filtering.JSONURLFilterWrapper",
+ * "class": "org.apache.stormcrawler.elasticsearch.filtering.JSONURLFilterWrapper",
* "name": "ESFastURLFilter",
* "params": {
* "refresh": "60",
* "delegate": {
- * "class": "com.digitalpebble.stormcrawler.filtering.regex.FastURLFilter",
+ * "class": "org.apache.stormcrawler.filtering.regex.FastURLFilter",
* "params": {
* "file": "fast.urlfilter.json"
* }
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/metrics/MetricsConsumer.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/metrics/MetricsConsumer.java
similarity index 95%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/metrics/MetricsConsumer.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/metrics/MetricsConsumer.java
index ef7c9342..148e72d7 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/metrics/MetricsConsumer.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/metrics/MetricsConsumer.java
@@ -12,12 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.metrics;
+package org.apache.stormcrawler.elasticsearch.metrics;
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
-import com.digitalpebble.stormcrawler.elasticsearch.ElasticSearchConnection;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
@@ -27,6 +25,8 @@ import java.util.Map.Entry;
import org.apache.storm.metric.api.IMetricsConsumer;
import org.apache.storm.task.IErrorReporter;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.elasticsearch.ElasticSearchConnection;
+import org.apache.stormcrawler.util.ConfUtils;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.xcontent.XContentBuilder;
import org.slf4j.Logger;
@@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory;
*
* <pre>
* topology.metrics.consumer.register:
- * - class: "com.digitalpebble.stormcrawler.elasticsearch.metrics.MetricsConsumer"
+ * - class: "org.apache.stormcrawler.elasticsearch.metrics.MetricsConsumer"
* parallelism.hint: 1
* argument: "yyyy-MM-dd"
* </pre>
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/metrics/StatusMetricsBolt.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/metrics/StatusMetricsBolt.java
similarity index 96%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/metrics/StatusMetricsBolt.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/metrics/StatusMetricsBolt.java
index 7212cff9..eece2e10 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/metrics/StatusMetricsBolt.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/metrics/StatusMetricsBolt.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.metrics;
+package org.apache.stormcrawler.elasticsearch.metrics;
-import com.digitalpebble.stormcrawler.elasticsearch.ElasticSearchConnection;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.util.HashMap;
import java.util.Map;
import org.apache.storm.Config;
@@ -25,6 +23,8 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.utils.TupleUtils;
+import org.apache.stormcrawler.elasticsearch.ElasticSearchConnection;
+import org.apache.stormcrawler.util.ConfUtils;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.core.CountRequest;
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/parse/filter/JSONResourceWrapper.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/parse/filter/JSONResourceWrapper.java
similarity index 92%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/parse/filter/JSONResourceWrapper.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/parse/filter/JSONResourceWrapper.java
index 0cfb980d..fc8b6005 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/parse/filter/JSONResourceWrapper.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/parse/filter/JSONResourceWrapper.java
@@ -12,17 +12,17 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.parse.filter;
+package org.apache.stormcrawler.elasticsearch.parse.filter;
-import com.digitalpebble.stormcrawler.JSONResource;
-import com.digitalpebble.stormcrawler.elasticsearch.ElasticSearchConnection;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.ByteArrayInputStream;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
+import org.apache.stormcrawler.JSONResource;
+import org.apache.stormcrawler.elasticsearch.ElasticSearchConnection;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.RequestOptions;
@@ -42,12 +42,12 @@ import org.w3c.dom.DocumentFragment;
*
* <pre>
* {
- * "class": "com.digitalpebble.stormcrawler.elasticsearch.parse.filter.JSONResourceWrapper",
+ * "class": "org.apache.stormcrawler.elasticsearch.parse.filter.JSONResourceWrapper",
* "name": "ESCollectionTagger",
* "params": {
* "refresh": "60",
* "delegate": {
- * "class": "com.digitalpebble.stormcrawler.parse.filter.CollectionTagger",
+ * "class": "org.apache.stormcrawler.parse.filter.CollectionTagger",
* "params": {
* "file": "collections.json"
* }
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/AbstractSpout.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/AbstractSpout.java
similarity index 96%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/AbstractSpout.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/AbstractSpout.java
index cdaf4a86..7969a293 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/AbstractSpout.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/AbstractSpout.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.persistence;
+package org.apache.stormcrawler.elasticsearch.persistence;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.elasticsearch.ElasticSearchConnection;
-import com.digitalpebble.stormcrawler.persistence.AbstractQueryingSpout;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.io.IOException;
import java.util.Date;
import java.util.Iterator;
@@ -27,6 +23,10 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.elasticsearch.ElasticSearchConnection;
+import org.apache.stormcrawler.persistence.AbstractQueryingSpout;
+import org.apache.stormcrawler.util.ConfUtils;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.search.SearchHit;
import org.slf4j.Logger;
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/AggregationSpout.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/AggregationSpout.java
similarity index 98%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/AggregationSpout.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/AggregationSpout.java
index 82fa5893..bafa8e90 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/AggregationSpout.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/AggregationSpout.java
@@ -12,12 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.persistence;
+package org.apache.stormcrawler.elasticsearch.persistence;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.time.Instant;
import java.util.Calendar;
import java.util.Date;
@@ -29,6 +27,8 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.ConfUtils;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/CollapsingSpout.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/CollapsingSpout.java
similarity index 98%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/CollapsingSpout.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/CollapsingSpout.java
index 4b615bd0..f8dd9079 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/CollapsingSpout.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/CollapsingSpout.java
@@ -12,11 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.persistence;
+package org.apache.stormcrawler.elasticsearch.persistence;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.time.Instant;
import java.util.Date;
import java.util.LinkedList;
@@ -26,6 +25,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.util.ConfUtils;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/HybridSpout.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/HybridSpout.java
similarity index 97%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/HybridSpout.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/HybridSpout.java
index b9b96f91..06a2a06d 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/HybridSpout.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/HybridSpout.java
@@ -12,12 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.persistence;
+package org.apache.stormcrawler.elasticsearch.persistence;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import com.digitalpebble.stormcrawler.persistence.EmptyQueueListener;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import java.time.Instant;
@@ -26,6 +24,8 @@ import java.util.List;
import java.util.Map;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.persistence.EmptyQueueListener;
+import org.apache.stormcrawler.util.ConfUtils;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/ScrollSpout.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/ScrollSpout.java
similarity index 95%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/ScrollSpout.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/ScrollSpout.java
index 60aac54f..8a0cfe8b 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/ScrollSpout.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/ScrollSpout.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.persistence;
+package org.apache.stormcrawler.elasticsearch.persistence;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.AbstractStatusUpdaterBolt;
-import com.digitalpebble.stormcrawler.persistence.Status;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -26,6 +22,10 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt;
+import org.apache.stormcrawler.persistence.Status;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
diff --git a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/StatusUpdaterBolt.java b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/StatusUpdaterBolt.java
similarity index 96%
rename from external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/StatusUpdaterBolt.java
rename to external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/StatusUpdaterBolt.java
index a036ef3f..8a9b3ef0 100644
--- a/external/elasticsearch/src/main/java/com/digitalpebble/stormcrawler/elasticsearch/persistence/StatusUpdaterBolt.java
+++ b/external/elasticsearch/src/main/java/org/apache/stormcrawler/elasticsearch/persistence/StatusUpdaterBolt.java
@@ -12,16 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.persistence;
-
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.elasticsearch.BulkItemResponseToFailedFlag;
-import com.digitalpebble.stormcrawler.elasticsearch.ElasticSearchConnection;
-import com.digitalpebble.stormcrawler.persistence.AbstractStatusUpdaterBolt;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.PerSecondReducer;
-import com.digitalpebble.stormcrawler.util.URLPartitioner;
+package org.apache.stormcrawler.elasticsearch.persistence;
+
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
@@ -36,6 +28,14 @@ import org.apache.storm.metric.api.MultiReducedMetric;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.elasticsearch.BulkItemResponseToFailedFlag;
+import org.apache.stormcrawler.elasticsearch.ElasticSearchConnection;
+import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.PerSecondReducer;
+import org.apache.stormcrawler.util.URLPartitioner;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkProcessor;
diff --git a/external/elasticsearch/src/test/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/IndexerBoltTest.java b/external/elasticsearch/src/test/java/org/apache/stormcrawler/elasticsearch/bolt/IndexerBoltTest.java
similarity index 94%
rename from external/elasticsearch/src/test/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/IndexerBoltTest.java
rename to external/elasticsearch/src/test/java/org/apache/stormcrawler/elasticsearch/bolt/IndexerBoltTest.java
index 107eea89..a808e2cc 100644
--- a/external/elasticsearch/src/test/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/IndexerBoltTest.java
+++ b/external/elasticsearch/src/test/java/org/apache/stormcrawler/elasticsearch/bolt/IndexerBoltTest.java
@@ -12,22 +12,22 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.bolt;
+package org.apache.stormcrawler.elasticsearch.bolt;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestOutputCollector;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.indexing.AbstractIndexerBolt;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.*;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestOutputCollector;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
import org.junit.*;
import org.junit.rules.Timeout;
import org.slf4j.Logger;
diff --git a/external/elasticsearch/src/test/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/StatusBoltTest.java b/external/elasticsearch/src/test/java/org/apache/stormcrawler/elasticsearch/bolt/StatusBoltTest.java
similarity index 94%
rename from external/elasticsearch/src/test/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/StatusBoltTest.java
rename to external/elasticsearch/src/test/java/org/apache/stormcrawler/elasticsearch/bolt/StatusBoltTest.java
index 3c65aecf..425fbd7d 100644
--- a/external/elasticsearch/src/test/java/com/digitalpebble/stormcrawler/elasticsearch/bolt/StatusBoltTest.java
+++ b/external/elasticsearch/src/test/java/org/apache/stormcrawler/elasticsearch/bolt/StatusBoltTest.java
@@ -12,18 +12,13 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.elasticsearch.bolt;
+package org.apache.stormcrawler.elasticsearch.bolt;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestOutputCollector;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.elasticsearch.persistence.StatusUpdaterBolt;
-import com.digitalpebble.stormcrawler.persistence.Status;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -41,6 +36,11 @@ import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestOutputCollector;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.elasticsearch.persistence.StatusUpdaterBolt;
+import org.apache.stormcrawler.persistence.Status;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.RequestOptions;
@@ -141,7 +141,7 @@ public class StatusBoltTest {
conf.put("es.status.addresses", container.getHttpHostAddress());
- conf.put("scheduler.class", "com.digitalpebble.stormcrawler.persistence.DefaultScheduler");
+ conf.put("scheduler.class", "org.apache.stormcrawler.persistence.DefaultScheduler");
conf.put("status.updater.cache.spec", "maximumSize=10000,expireAfterAccess=1h");
diff --git a/external/langid/pom.xml b/external/langid/pom.xml
index 6522963c..8eb0b136 100644
--- a/external/langid/pom.xml
+++ b/external/langid/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-external</artifactId>
<version>2.12-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
@@ -13,7 +13,7 @@
<packaging>jar</packaging>
<name>storm-crawler-langid</name>
- <url>https://github.com/DigitalPebble/storm-crawler/tree/master/external/langid</url>
+ <url>https://github.com/apache/incubator-stormcrawler/tree/master/external/langid</url>
<description>Language Identification for StormCrawler</description>
<dependencies>
diff --git a/external/langid/src/main/java/com/digitalpebble/stormcrawler/parse/filter/LanguageID.java b/external/langid/src/main/java/org/apache/stormcrawler/parse/filter/LanguageID.java
similarity index 93%
rename from external/langid/src/main/java/com/digitalpebble/stormcrawler/parse/filter/LanguageID.java
rename to external/langid/src/main/java/org/apache/stormcrawler/parse/filter/LanguageID.java
index d2585238..0b7dd9dc 100644
--- a/external/langid/src/main/java/com/digitalpebble/stormcrawler/parse/filter/LanguageID.java
+++ b/external/langid/src/main/java/org/apache/stormcrawler/parse/filter/LanguageID.java
@@ -12,11 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.parse.filter;
+package org.apache.stormcrawler.parse.filter;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
import com.fasterxml.jackson.databind.JsonNode;
import com.optimaize.langdetect.DetectedLanguage;
import com.optimaize.langdetect.LanguageDetector;
@@ -34,6 +31,9 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory;
import org.w3c.dom.DocumentFragment;
@@ -42,8 +42,8 @@ import org.w3c.dom.DocumentFragment;
* Language identification; the language codes gets stored in the metadata. <br>
* To use it, just add the module as a dependency in your pom and include
*
- * <p>```json { "class": "com.digitalpebble.stormcrawler.parse.filter.LanguageID", "name":
- * "LanguageID", "params": { "key": "lang" , "minProb": 0.99 , "extracted": "parse.lang"} } ```
+ * <p>```json { "class": "org.apache.stormcrawler.parse.filter.LanguageID", "name": "LanguageID",
+ * "params": { "key": "lang" , "minProb": 0.99 , "extracted": "parse.lang"} } ```
*
* <p>in the parse filter config. Any value found in the metadata under the key specified by
* _extracted_ will be normalised and stored in the metadata, otherwise the languages above the
diff --git a/external/opensearch/README.md b/external/opensearch/README.md
index a1916119..6b10ce78 100644
--- a/external/opensearch/README.md
+++ b/external/opensearch/README.md
@@ -2,10 +2,10 @@ storm-crawler-opensearch
===========================
A collection of resources for [OpenSearch](https://opensearch.org/):
-* [IndexerBolt](https://github.com/DigitalPebble/storm-crawler/blob/master/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/bolt/IndexerBolt.java) for indexing documents crawled with StormCrawler
-* [Spouts](https://github.com/DigitalPebble/storm-crawler/blob/master/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/AggregationSpout.java) and [StatusUpdaterBolt](https://github.com/DigitalPebble/storm-crawler/blob/master/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java) for persisting URL information in recursive crawls
-* [MetricsConsumer](https://github.com/DigitalPebble/storm-crawler/blob/master/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/metrics/MetricsConsumer.java)
-* [StatusMetricsBolt](https://github.com/DigitalPebble/storm-crawler/blob/master/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/metrics/StatusMetricsBolt.java) for sending the breakdown of URLs per status as metrics and display its evolution over time.
+* [IndexerBolt](https://github.com/apache/incubator-stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java) for indexing documents crawled with StormCrawler
+* [Spouts](https://github.com/apache/incubator-stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java) and [StatusUpdaterBolt](https://github.com/apache/incubator-stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java) for persisting URL information in recursive crawls
+* [MetricsConsumer](https://github.com/apache/incubator-stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java)
+* [StatusMetricsBolt](https://github.com/apache/incubator-stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java) for sending the breakdown of URLs per status as metrics and display its evolution over time.
as well as resources for building basic real-time monitoring dashboards for the crawls, see below.
@@ -16,7 +16,7 @@ Getting started
The easiest way is currently to use the archetype for OpenSearch with:
-`mvn archetype:generate -DarchetypeGroupId=com.digitalpebble.stormcrawler -DarchetypeArtifactId=storm-crawler-opensearch-archetype -DarchetypeVersion=2.11`
+`mvn archetype:generate -DarchetypeGroupId=org.apache.stormcrawler -DarchetypeArtifactId=storm-crawler-opensearch-archetype -DarchetypeVersion=2.11`
You'll be asked to enter a groupId (e.g. com.mycompany.crawler), an artefactId (e.g. stormcrawler), a version, a package name and details about the user agent to use.
@@ -56,7 +56,7 @@ The crawler config YAML must be updated to use an optional argument as shown bel
```
#Metrics consumers:
topology.metrics.consumer.register:
- - class: "com.digitalpebble.stormcrawler.opensearch.metrics.MetricsConsumer"
+ - class: "org.apache.stormcrawler.opensearch.metrics.MetricsConsumer"
parallelism.hint: 1
argument: "yyyy-MM-dd"
```
diff --git a/external/opensearch/archetype/pom.xml b/external/opensearch/archetype/pom.xml
index e24e2954..8573a894 100644
--- a/external/opensearch/archetype/pom.xml
+++ b/external/opensearch/archetype/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler</artifactId>
<version>2.12-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml
index 45645409..d21929e7 100644
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml
@@ -17,8 +17,8 @@ config:
# mandatory when using Flux
topology.kryo.register:
- - com.digitalpebble.stormcrawler.Metadata
- - com.digitalpebble.stormcrawler.persistence.Status
+ - org.apache.stormcrawler.Metadata
+ - org.apache.stormcrawler.persistence.Status
# Lists the metadata to transfer to outlinks
# Used by Fetcher and SiteMapParser for redirections,
@@ -54,8 +54,8 @@ config:
# Finally, an email so that they can get in touch with you
http.agent.email: "${http-agent-email}"
- http.protocol.implementation: "com.digitalpebble.stormcrawler.protocol.okhttp.HttpProtocol"
- https.protocol.implementation: "com.digitalpebble.stormcrawler.protocol.okhttp.HttpProtocol"
+ http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
+ https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
# The maximum number of bytes for returned HTTP response bodies.
# The fetched page will be trimmed to 65KB in this case
diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler.flux b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler.flux
index 6df56df1..42121431 100644
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler.flux
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler.flux
@@ -15,39 +15,39 @@ includes:
spouts:
- id: "spout"
- className: "com.digitalpebble.stormcrawler.opensearch.persistence.AggregationSpout"
+ className: "org.apache.stormcrawler.opensearch.persistence.AggregationSpout"
parallelism: 10
bolts:
- id: "partitioner"
- className: "com.digitalpebble.stormcrawler.bolt.URLPartitionerBolt"
+ className: "org.apache.stormcrawler.bolt.URLPartitionerBolt"
parallelism: 1
- id: "fetcher"
- className: "com.digitalpebble.stormcrawler.bolt.FetcherBolt"
+ className: "org.apache.stormcrawler.bolt.FetcherBolt"
parallelism: 1
- id: "sitemap"
- className: "com.digitalpebble.stormcrawler.bolt.SiteMapParserBolt"
+ className: "org.apache.stormcrawler.bolt.SiteMapParserBolt"
parallelism: 1
- id: "parse"
- className: "com.digitalpebble.stormcrawler.bolt.JSoupParserBolt"
+ className: "org.apache.stormcrawler.bolt.JSoupParserBolt"
parallelism: 1
- id: "shunt"
- className: "com.digitalpebble.stormcrawler.tika.RedirectionBolt"
+ className: "org.apache.stormcrawler.tika.RedirectionBolt"
parallelism: 1
- id: "tika"
- className: "com.digitalpebble.stormcrawler.tika.ParserBolt"
+ className: "org.apache.stormcrawler.tika.ParserBolt"
parallelism: 1
- id: "index"
- className: "com.digitalpebble.stormcrawler.opensearch.bolt.IndexerBolt"
+ className: "org.apache.stormcrawler.opensearch.bolt.IndexerBolt"
parallelism: 1
- id: "status"
- className: "com.digitalpebble.stormcrawler.opensearch.persistence.StatusUpdaterBolt"
+ className: "org.apache.stormcrawler.opensearch.persistence.StatusUpdaterBolt"
parallelism: 1
- id: "deleter"
- className: "com.digitalpebble.stormcrawler.opensearch.bolt.DeletionBolt"
+ className: "org.apache.stormcrawler.opensearch.bolt.DeletionBolt"
parallelism: 1
- id: "status_metrics"
- className: "com.digitalpebble.stormcrawler.opensearch.metrics.StatusMetricsBolt"
+ className: "org.apache.stormcrawler.opensearch.metrics.StatusMetricsBolt"
parallelism: 1
streams:
diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/injection.flux b/external/opensearch/archetype/src/main/resources/archetype-resources/injection.flux
index ab3091be..060c1052 100644
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/injection.flux
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/injection.flux
@@ -15,7 +15,7 @@ includes:
spouts:
- id: "filespout"
- className: "com.digitalpebble.stormcrawler.spout.FileSpout"
+ className: "org.apache.stormcrawler.spout.FileSpout"
parallelism: 1
constructorArgs:
- "."
@@ -24,11 +24,11 @@ spouts:
bolts:
- id: "filter"
- className: "com.digitalpebble.stormcrawler.bolt.URLFilterBolt"
+ className: "org.apache.stormcrawler.bolt.URLFilterBolt"
parallelism: 1
- id: "status"
- className: "com.digitalpebble.stormcrawler.opensearch.persistence.StatusUpdaterBolt"
+ className: "org.apache.stormcrawler.opensearch.persistence.StatusUpdaterBolt"
parallelism: 1
streams:
@@ -45,6 +45,6 @@ streams:
streamId: "status"
type: CUSTOM
customClass:
- className: "com.digitalpebble.stormcrawler.util.URLStreamGrouping"
+ className: "org.apache.stormcrawler.util.URLStreamGrouping"
constructorArgs:
- "byDomain"
diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml b/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml
index 99a706ea..f37b8d5a 100644
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml
@@ -83,7 +83,7 @@ config:
opensearch.status.recentDate.min.gap: -1
topology.metrics.consumer.register:
- - class: "com.digitalpebble.stormcrawler.opensearch.metrics.MetricsConsumer"
+ - class: "org.apache.stormcrawler.opensearch.metrics.MetricsConsumer"
parallelism.hint: 1
#whitelist:
# - "fetcher_counter"
diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/pom.xml b/external/opensearch/archetype/src/main/resources/archetype-resources/pom.xml
index c4573e66..5403052a 100644
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/pom.xml
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/pom.xml
@@ -100,12 +100,12 @@
<dependencies>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-core</artifactId>
<version>${stormcrawler.version}</version>
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-opensearch</artifactId>
<version>${stormcrawler.version}</version>
</dependency>
@@ -121,7 +121,7 @@
<version>${storm.version}</version>
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-tika</artifactId>
<version>${stormcrawler.version}</version>
</dependency>
diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json b/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json
index c48b4134..4d87d8d5 100644
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json
@@ -1,7 +1,7 @@
{
- "com.digitalpebble.stormcrawler.parse.JSoupFilters": [
+ "org.apache.stormcrawler.parse.JSoupFilters": [
{
- "class": "com.digitalpebble.stormcrawler.jsoup.XPathFilter",
+ "class": "org.apache.stormcrawler.jsoup.XPathFilter",
"name": "XPathFilter",
"params": {
"canonical": "//*[@rel=\"canonical\"]/@href",
@@ -17,7 +17,7 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.jsoup.LinkParseFilter",
+ "class": "org.apache.stormcrawler.jsoup.LinkParseFilter",
"name": "LinkParseFilter",
"params": {
"pattern": "//FRAME/@src"
diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json b/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json
index 04eaa895..5d525830 100644
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json
@@ -1,7 +1,7 @@
{
- "com.digitalpebble.stormcrawler.parse.ParseFilters": [
+ "org.apache.stormcrawler.parse.ParseFilters": [
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.DomainParseFilter",
+ "class": "org.apache.stormcrawler.parse.filter.DomainParseFilter",
"name": "DomainParseFilter",
"params": {
"key": "domain",
@@ -9,11 +9,11 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.MimeTypeNormalization",
+ "class": "org.apache.stormcrawler.parse.filter.MimeTypeNormalization",
"name": "MimeTypeNormalization"
},
{
- "class": "com.digitalpebble.stormcrawler.parse.filter.CommaSeparatedToMultivaluedMetadata",
+ "class": "org.apache.stormcrawler.parse.filter.CommaSeparatedToMultivaluedMetadata",
"name": "CommaSeparatedToMultivaluedMetadata",
"params": {
"keys": ["parse.keywords"]
diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json b/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json
index ec6740ef..6098631b 100644
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/src/main/resources/urlfilters.json
@@ -1,7 +1,7 @@
{
- "com.digitalpebble.stormcrawler.filtering.URLFilters": [
+ "org.apache.stormcrawler.filtering.URLFilters": [
{
- "class": "com.digitalpebble.stormcrawler.filtering.basic.BasicURLFilter",
+ "class": "org.apache.stormcrawler.filtering.basic.BasicURLFilter",
"name": "BasicURLFilter",
"params": {
"maxPathRepetition": 3,
@@ -9,14 +9,14 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.depth.MaxDepthFilter",
+ "class": "org.apache.stormcrawler.filtering.depth.MaxDepthFilter",
"name": "MaxDepthFilter",
"params": {
"maxDepth": -1
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.basic.BasicURLNormalizer",
+ "class": "org.apache.stormcrawler.filtering.basic.BasicURLNormalizer",
"name": "BasicURLNormalizer",
"params": {
"removeAnchorPart": true,
@@ -27,7 +27,7 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.host.HostURLFilter",
+ "class": "org.apache.stormcrawler.filtering.host.HostURLFilter",
"name": "HostURLFilter",
"params": {
"ignoreOutsideHost": false,
@@ -35,25 +35,25 @@
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.regex.RegexURLNormalizer",
+ "class": "org.apache.stormcrawler.filtering.regex.RegexURLNormalizer",
"name": "RegexURLNormalizer",
"params": {
"regexNormalizerFile": "default-regex-normalizers.xml"
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.regex.RegexURLFilter",
+ "class": "org.apache.stormcrawler.filtering.regex.RegexURLFilter",
"name": "RegexURLFilter",
"params": {
"regexFilterFile": "default-regex-filters.txt"
}
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.basic.SelfURLFilter",
+ "class": "org.apache.stormcrawler.filtering.basic.SelfURLFilter",
"name": "SelfURLFilter"
},
{
- "class": "com.digitalpebble.stormcrawler.filtering.sitemap.SitemapFilter",
+ "class": "org.apache.stormcrawler.filtering.sitemap.SitemapFilter",
"name": "SitemapFilter"
}
]
diff --git a/external/opensearch/opensearch-conf.yaml b/external/opensearch/opensearch-conf.yaml
index 99a706ea..f37b8d5a 100644
--- a/external/opensearch/opensearch-conf.yaml
+++ b/external/opensearch/opensearch-conf.yaml
@@ -83,7 +83,7 @@ config:
opensearch.status.recentDate.min.gap: -1
topology.metrics.consumer.register:
- - class: "com.digitalpebble.stormcrawler.opensearch.metrics.MetricsConsumer"
+ - class: "org.apache.stormcrawler.opensearch.metrics.MetricsConsumer"
parallelism.hint: 1
#whitelist:
# - "fetcher_counter"
diff --git a/external/opensearch/pom.xml b/external/opensearch/pom.xml
index e8e38436..6c0d4d1e 100644
--- a/external/opensearch/pom.xml
+++ b/external/opensearch/pom.xml
@@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-external</artifactId>
<version>2.12-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
@@ -20,7 +20,7 @@
<name>storm-crawler-opensearch</name>
<url>
- https://github.com/DigitalPebble/storm-crawler/tree/master/external/opensearch</url>
+ https://github.com/apache/incubator-stormcrawler/tree/master/external/opensearch</url>
<description>Opensearch resources for StormCrawler</description>
<build>
@@ -61,7 +61,7 @@
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
similarity index 98%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
index f00cdbec..a978ffe9 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
@@ -1,4 +1,4 @@
-package com.digitalpebble.stormcrawler.opensearch;
+package org.apache.stormcrawler.opensearch;
import java.io.IOException;
import org.jetbrains.annotations.NotNull;
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/Constants.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/Constants.java
similarity index 94%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/Constants.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/Constants.java
index 998deb28..19e504ca 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/Constants.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/Constants.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch;
+package org.apache.stormcrawler.opensearch;
public interface Constants {
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/IndexCreation.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
similarity index 98%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/IndexCreation.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
index 55358d74..1cbc5b59 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/IndexCreation.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch;
+package org.apache.stormcrawler.opensearch;
import com.google.common.base.Charsets;
import com.google.common.io.Resources;
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
similarity index 99%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/OpenSearchConnection.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
index ffff955e..26cbc180 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/OpenSearchConnection.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
@@ -12,12 +12,11 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch;
+package org.apache.stormcrawler.opensearch;
import static org.opensearch.client.RestClientBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS;
import static org.opensearch.client.RestClientBuilder.DEFAULT_SOCKET_TIMEOUT_MILLIS;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@@ -30,6 +29,7 @@ import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.storm.shade.org.apache.commons.lang.StringUtils;
+import org.apache.stormcrawler.util.ConfUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.opensearch.action.DocWriteRequest;
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
similarity index 97%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/bolt/DeletionBolt.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
index 39a3ab7e..1dc64df0 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/bolt/DeletionBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.bolt;
+package org.apache.stormcrawler.opensearch.bolt;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.opensearch.BulkItemResponseToFailedFlag;
-import com.digitalpebble.stormcrawler.opensearch.OpenSearchConnection;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
@@ -36,6 +32,10 @@ import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.opensearch.BulkItemResponseToFailedFlag;
+import org.apache.stormcrawler.opensearch.OpenSearchConnection;
+import org.apache.stormcrawler.util.ConfUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.opensearch.action.DocWriteRequest;
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/bolt/IndexerBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
similarity index 94%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/bolt/IndexerBolt.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
index ed813308..f7099646 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/bolt/IndexerBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
@@ -12,20 +12,11 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.bolt;
+package org.apache.stormcrawler.opensearch.bolt;
-import static com.digitalpebble.stormcrawler.Constants.StatusStreamName;
+import static org.apache.stormcrawler.Constants.StatusStreamName;
import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.indexing.AbstractIndexerBolt;
-import com.digitalpebble.stormcrawler.opensearch.BulkItemResponseToFailedFlag;
-import com.digitalpebble.stormcrawler.opensearch.IndexCreation;
-import com.digitalpebble.stormcrawler.opensearch.OpenSearchConnection;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.PerSecondReducer;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
@@ -49,6 +40,15 @@ import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
+import org.apache.stormcrawler.opensearch.BulkItemResponseToFailedFlag;
+import org.apache.stormcrawler.opensearch.IndexCreation;
+import org.apache.stormcrawler.opensearch.OpenSearchConnection;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.PerSecondReducer;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.opensearch.action.DocWriteRequest;
@@ -74,17 +74,11 @@ public class IndexerBolt extends AbstractIndexerBolt
private static final String OSBoltType = "indexer";
static final String OSIndexNameParamName =
- com.digitalpebble.stormcrawler.opensearch.Constants.PARAMPREFIX
- + OSBoltType
- + ".index.name";
+ org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".index.name";
private static final String OSCreateParamName =
- com.digitalpebble.stormcrawler.opensearch.Constants.PARAMPREFIX
- + OSBoltType
- + ".create";
+ org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".create";
private static final String OSIndexPipelineParamName =
- com.digitalpebble.stormcrawler.opensearch.Constants.PARAMPREFIX
- + OSBoltType
- + ".pipeline";
+ org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".pipeline";
private OutputCollector _collector;
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
similarity index 93%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
index 7e749967..a802cc40 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
@@ -12,18 +12,18 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.filtering;
+package org.apache.stormcrawler.opensearch.filtering;
-import com.digitalpebble.stormcrawler.JSONResource;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilter;
-import com.digitalpebble.stormcrawler.opensearch.OpenSearchConnection;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.ByteArrayInputStream;
import java.net.URL;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
+import org.apache.stormcrawler.JSONResource;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilter;
+import org.apache.stormcrawler.opensearch.OpenSearchConnection;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.opensearch.action.get.GetRequest;
@@ -43,12 +43,12 @@ import org.slf4j.LoggerFactory;
*
* <pre>
* {
- * "class": "com.digitalpebble.stormcrawler.elasticsearch.filtering.JSONURLFilterWrapper",
+ * "class": "org.apache.stormcrawler.elasticsearch.filtering.JSONURLFilterWrapper",
* "name": "ESFastURLFilter",
* "params": {
* "refresh": "60",
* "delegate": {
- * "class": "com.digitalpebble.stormcrawler.filtering.regex.FastURLFilter",
+ * "class": "org.apache.stormcrawler.filtering.regex.FastURLFilter",
* "params": {
* "file": "fast.urlfilter.json"
* }
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/metrics/MetricsConsumer.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
similarity index 94%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/metrics/MetricsConsumer.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
index 21d3bcf4..ec710419 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/metrics/MetricsConsumer.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
@@ -12,13 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.metrics;
+package org.apache.stormcrawler.opensearch.metrics;
import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
-import com.digitalpebble.stormcrawler.opensearch.IndexCreation;
-import com.digitalpebble.stormcrawler.opensearch.OpenSearchConnection;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Collection;
@@ -29,6 +26,9 @@ import java.util.Map.Entry;
import org.apache.storm.metric.api.IMetricsConsumer;
import org.apache.storm.task.IErrorReporter;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.opensearch.IndexCreation;
+import org.apache.stormcrawler.opensearch.OpenSearchConnection;
+import org.apache.stormcrawler.util.ConfUtils;
import org.opensearch.action.index.IndexRequest;
import org.opensearch.core.xcontent.XContentBuilder;
import org.slf4j.Logger;
@@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory;
*
* <pre>
* topology.metrics.consumer.register:
- * - class: "com.digitalpebble.stormcrawler.elasticsearch.metrics.MetricsConsumer"
+ * - class: "org.apache.stormcrawler.elasticsearch.metrics.MetricsConsumer"
* parallelism.hint: 1
* argument: "yyyy-MM-dd"
* </pre>
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/metrics/StatusMetricsBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
similarity index 95%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
index a0e82f6a..9ba92292 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
@@ -12,11 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.metrics;
+package org.apache.stormcrawler.opensearch.metrics;
-import com.digitalpebble.stormcrawler.opensearch.Constants;
-import com.digitalpebble.stormcrawler.opensearch.OpenSearchConnection;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.util.HashMap;
import java.util.Map;
import org.apache.storm.Config;
@@ -26,6 +23,9 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.utils.TupleUtils;
+import org.apache.stormcrawler.opensearch.Constants;
+import org.apache.stormcrawler.opensearch.OpenSearchConnection;
+import org.apache.stormcrawler.util.ConfUtils;
import org.opensearch.client.RequestOptions;
import org.opensearch.client.core.CountRequest;
import org.opensearch.client.core.CountResponse;
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
similarity index 93%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
index 226fe76c..f5718711 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
@@ -12,17 +12,17 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.parse.filter;
+package org.apache.stormcrawler.opensearch.parse.filter;
-import com.digitalpebble.stormcrawler.JSONResource;
-import com.digitalpebble.stormcrawler.opensearch.OpenSearchConnection;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.ByteArrayInputStream;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
+import org.apache.stormcrawler.JSONResource;
+import org.apache.stormcrawler.opensearch.OpenSearchConnection;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
import org.jetbrains.annotations.NotNull;
import org.opensearch.action.get.GetRequest;
import org.opensearch.action.get.GetResponse;
@@ -42,12 +42,12 @@ import org.w3c.dom.DocumentFragment;
*
* <pre>
* {
- * "class": "com.digitalpebble.stormcrawler.elasticsearch.parse.filter.JSONResourceWrapper",
+ * "class": "org.apache.stormcrawler.elasticsearch.parse.filter.JSONResourceWrapper",
* "name": "OpenSearchCollectionTagger",
* "params": {
* "refresh": "60",
* "delegate": {
- * "class": "com.digitalpebble.stormcrawler.parse.filter.CollectionTagger",
+ * "class": "org.apache.stormcrawler.parse.filter.CollectionTagger",
* "params": {
* "file": "collections.json"
* }
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/AbstractSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
similarity index 95%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/AbstractSpout.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
index 82c8dc45..f6e60e3d 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/AbstractSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
@@ -12,14 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.persistence;
-
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.opensearch.Constants;
-import com.digitalpebble.stormcrawler.opensearch.IndexCreation;
-import com.digitalpebble.stormcrawler.opensearch.OpenSearchConnection;
-import com.digitalpebble.stormcrawler.persistence.AbstractQueryingSpout;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
+package org.apache.stormcrawler.opensearch.persistence;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
@@ -29,6 +23,12 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.opensearch.Constants;
+import org.apache.stormcrawler.opensearch.IndexCreation;
+import org.apache.stormcrawler.opensearch.OpenSearchConnection;
+import org.apache.stormcrawler.persistence.AbstractQueryingSpout;
+import org.apache.stormcrawler.util.ConfUtils;
import org.opensearch.client.RestHighLevelClient;
import org.opensearch.search.SearchHit;
import org.slf4j.Logger;
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/AggregationSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
similarity index 98%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/AggregationSpout.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
index c223bc18..bdc81e5a 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/AggregationSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
@@ -12,13 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.persistence;
+package org.apache.stormcrawler.opensearch.persistence;
import static org.opensearch.index.query.QueryBuilders.boolQuery;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.opensearch.Constants;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.time.Instant;
import java.util.Calendar;
import java.util.Date;
@@ -30,6 +27,9 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.opensearch.Constants;
+import org.apache.stormcrawler.util.ConfUtils;
import org.joda.time.format.ISODateTimeFormat;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/HybridSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
similarity index 97%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/HybridSpout.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
index cb2703a8..6709ed00 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/HybridSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
@@ -12,13 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.persistence;
+package org.apache.stormcrawler.opensearch.persistence;
import static org.opensearch.index.query.QueryBuilders.boolQuery;
-import com.digitalpebble.stormcrawler.opensearch.Constants;
-import com.digitalpebble.stormcrawler.persistence.EmptyQueueListener;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import java.time.Instant;
@@ -27,6 +24,9 @@ import java.util.List;
import java.util.Map;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.opensearch.Constants;
+import org.apache.stormcrawler.persistence.EmptyQueueListener;
+import org.apache.stormcrawler.util.ConfUtils;
import org.joda.time.format.ISODateTimeFormat;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
diff --git a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
similarity index 96%
rename from external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
index bbb0b6ac..abdf5574 100644
--- a/external/opensearch/src/main/java/com/digitalpebble/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
@@ -12,18 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.persistence;
-
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.opensearch.BulkItemResponseToFailedFlag;
-import com.digitalpebble.stormcrawler.opensearch.Constants;
-import com.digitalpebble.stormcrawler.opensearch.IndexCreation;
-import com.digitalpebble.stormcrawler.opensearch.OpenSearchConnection;
-import com.digitalpebble.stormcrawler.persistence.AbstractStatusUpdaterBolt;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.PerSecondReducer;
-import com.digitalpebble.stormcrawler.util.URLPartitioner;
+package org.apache.stormcrawler.opensearch.persistence;
+
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
@@ -47,6 +37,16 @@ import org.apache.storm.metric.api.MultiReducedMetric;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.opensearch.BulkItemResponseToFailedFlag;
+import org.apache.stormcrawler.opensearch.Constants;
+import org.apache.stormcrawler.opensearch.IndexCreation;
+import org.apache.stormcrawler.opensearch.OpenSearchConnection;
+import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.PerSecondReducer;
+import org.apache.stormcrawler.util.URLPartitioner;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.opensearch.action.DocWriteRequest;
diff --git a/external/opensearch/src/test/java/com/digitalpebble/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
similarity index 97%
rename from external/opensearch/src/test/java/com/digitalpebble/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
index 404e3210..8be0f702 100644
--- a/external/opensearch/src/test/java/com/digitalpebble/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.bolt;
+package org.apache.stormcrawler.opensearch.bolt;
import org.junit.After;
import org.junit.Rule;
diff --git a/external/opensearch/src/test/java/com/digitalpebble/stormcrawler/opensearch/bolt/IndexerBoltTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java
similarity index 93%
rename from external/opensearch/src/test/java/com/digitalpebble/stormcrawler/opensearch/bolt/IndexerBoltTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java
index d359e3f3..ef1c98cd 100644
--- a/external/opensearch/src/test/java/com/digitalpebble/stormcrawler/opensearch/bolt/IndexerBoltTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java
@@ -12,17 +12,12 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.bolt;
+package org.apache.stormcrawler.opensearch.bolt;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestOutputCollector;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.indexing.AbstractIndexerBolt;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
@@ -32,6 +27,11 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestOutputCollector;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
diff --git a/external/opensearch/src/test/java/com/digitalpebble/stormcrawler/opensearch/bolt/StatusBoltTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
similarity index 92%
rename from external/opensearch/src/test/java/com/digitalpebble/stormcrawler/opensearch/bolt/StatusBoltTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
index 42417444..d35ae9df 100644
--- a/external/opensearch/src/test/java/com/digitalpebble/stormcrawler/opensearch/bolt/StatusBoltTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
@@ -12,18 +12,13 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.opensearch.bolt;
+package org.apache.stormcrawler.opensearch.bolt;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestOutputCollector;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.opensearch.persistence.StatusUpdaterBolt;
-import com.digitalpebble.stormcrawler.persistence.Status;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -36,6 +31,11 @@ import java.util.concurrent.TimeoutException;
import org.apache.http.HttpHost;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestOutputCollector;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.opensearch.persistence.StatusUpdaterBolt;
+import org.apache.stormcrawler.persistence.Status;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -93,7 +93,7 @@ public class StatusBoltTest extends AbstractOpenSearchTest {
"opensearch.status.addresses",
opensearchContainer.getHost() + ":" + opensearchContainer.getFirstMappedPort());
- conf.put("scheduler.class", "com.digitalpebble.stormcrawler.persistence.DefaultScheduler");
+ conf.put("scheduler.class", "org.apache.stormcrawler.persistence.DefaultScheduler");
conf.put("status.updater.cache.spec", "maximumSize=10000,expireAfterAccess=1h");
diff --git a/external/pom.xml b/external/pom.xml
index 1ee2ab0e..608aaa15 100644
--- a/external/pom.xml
+++ b/external/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler</artifactId>
<version>2.12-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
@@ -71,7 +71,7 @@
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-core</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/external/solr/README.md b/external/solr/README.md
index a6aa261a..b23d5fa8 100644
--- a/external/solr/README.md
+++ b/external/solr/README.md
@@ -9,7 +9,7 @@ In your project you can use this by adding the following dependency:
```xml
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-solr</artifactId>
<version>${stormcrawler.version}</version>
</dependency>
diff --git a/external/solr/pom.xml b/external/solr/pom.xml
index 38d6f492..a3ee90a1 100644
--- a/external/solr/pom.xml
+++ b/external/solr/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-external</artifactId>
<version>2.12-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
@@ -14,7 +14,7 @@
<name>storm-crawler-solr</name>
<url>
- https://github.com/DigitalPebble/storm-crawler/tree/master/external/solr</url>
+ https://github.com/apache/incubator-stormcrawler/tree/master/external/solr</url>
<description>Solr resources for StormCrawler</description>
<properties>
@@ -35,7 +35,7 @@
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
diff --git a/external/solr/solr-conf.yaml b/external/solr/solr-conf.yaml
index 802c488c..2e29cda1 100644
--- a/external/solr/solr-conf.yaml
+++ b/external/solr/solr-conf.yaml
@@ -34,5 +34,5 @@ config:
# the same applies for the spout/persistence bolt and the metricsconsumer
topology.metrics.consumer.register:
- - class: "com.digitalpebble.stormcrawler.solr.metrics.MetricsConsumer"
+ - class: "org.apache.stormcrawler.solr.metrics.MetricsConsumer"
parallelism.hint: 1
diff --git a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/SeedInjector.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/SeedInjector.java
similarity index 86%
rename from external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/SeedInjector.java
rename to external/solr/src/main/java/org/apache/stormcrawler/solr/SeedInjector.java
index 2d1d7459..a6c62520 100644
--- a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/SeedInjector.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/SeedInjector.java
@@ -12,14 +12,14 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.solr;
+package org.apache.stormcrawler.solr;
-import com.digitalpebble.stormcrawler.ConfigurableTopology;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.solr.persistence.StatusUpdaterBolt;
-import com.digitalpebble.stormcrawler.spout.FileSpout;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
+import org.apache.stormcrawler.ConfigurableTopology;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.solr.persistence.StatusUpdaterBolt;
+import org.apache.stormcrawler.spout.FileSpout;
/**
* Topology which reads from a file containing seeds and distributes to SQS queues based on the IP /
diff --git a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/SolrConnection.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/SolrConnection.java
similarity index 97%
rename from external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/SolrConnection.java
rename to external/solr/src/main/java/org/apache/stormcrawler/solr/SolrConnection.java
index e28ac2b5..6c80f172 100644
--- a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/SolrConnection.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/SolrConnection.java
@@ -12,9 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.solr;
+package org.apache.stormcrawler.solr;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
@@ -25,6 +24,7 @@ import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.storm.shade.org.apache.commons.lang.StringUtils;
+import org.apache.stormcrawler.util.ConfUtils;
public class SolrConnection {
diff --git a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/SolrCrawlTopology.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/SolrCrawlTopology.java
similarity index 76%
rename from external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/SolrCrawlTopology.java
rename to external/solr/src/main/java/org/apache/stormcrawler/solr/SolrCrawlTopology.java
index bb9ac596..2453b075 100644
--- a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/SolrCrawlTopology.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/SolrCrawlTopology.java
@@ -12,21 +12,21 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.solr;
+package org.apache.stormcrawler.solr;
-import com.digitalpebble.stormcrawler.ConfigurableTopology;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.bolt.FetcherBolt;
-import com.digitalpebble.stormcrawler.bolt.JSoupParserBolt;
-import com.digitalpebble.stormcrawler.bolt.SiteMapParserBolt;
-import com.digitalpebble.stormcrawler.bolt.URLPartitionerBolt;
-import com.digitalpebble.stormcrawler.solr.bolt.DeletionBolt;
-import com.digitalpebble.stormcrawler.solr.bolt.IndexerBolt;
-import com.digitalpebble.stormcrawler.solr.metrics.MetricsConsumer;
-import com.digitalpebble.stormcrawler.solr.persistence.SolrSpout;
-import com.digitalpebble.stormcrawler.solr.persistence.StatusUpdaterBolt;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
+import org.apache.stormcrawler.ConfigurableTopology;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.bolt.FetcherBolt;
+import org.apache.stormcrawler.bolt.JSoupParserBolt;
+import org.apache.stormcrawler.bolt.SiteMapParserBolt;
+import org.apache.stormcrawler.bolt.URLPartitionerBolt;
+import org.apache.stormcrawler.solr.bolt.DeletionBolt;
+import org.apache.stormcrawler.solr.bolt.IndexerBolt;
+import org.apache.stormcrawler.solr.metrics.MetricsConsumer;
+import org.apache.stormcrawler.solr.persistence.SolrSpout;
+import org.apache.stormcrawler.solr.persistence.StatusUpdaterBolt;
/** Dummy topology to play with the spouts and bolts on Solr */
public class SolrCrawlTopology extends ConfigurableTopology {
diff --git a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/bolt/DeletionBolt.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/DeletionBolt.java
similarity index 96%
rename from external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/bolt/DeletionBolt.java
rename to external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/DeletionBolt.java
index 7b888c21..2f1ee290 100644
--- a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/bolt/DeletionBolt.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/DeletionBolt.java
@@ -12,9 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.solr.bolt;
+package org.apache.stormcrawler.solr.bolt;
-import com.digitalpebble.stormcrawler.solr.SolrConnection;
import java.io.IOException;
import java.util.Map;
import org.apache.solr.client.solrj.SolrServerException;
@@ -23,6 +22,7 @@ import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.solr.SolrConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/bolt/IndexerBolt.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/IndexerBolt.java
similarity index 92%
rename from external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/bolt/IndexerBolt.java
rename to external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/IndexerBolt.java
index d0b947f3..c4e584a7 100644
--- a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/bolt/IndexerBolt.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/IndexerBolt.java
@@ -12,14 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.solr.bolt;
+package org.apache.stormcrawler.solr.bolt;
-import static com.digitalpebble.stormcrawler.Constants.StatusStreamName;
+import static org.apache.stormcrawler.Constants.StatusStreamName;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.indexing.AbstractIndexerBolt;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.solr.SolrConnection;
import java.util.Iterator;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
@@ -29,6 +25,10 @@ import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.solr.SolrConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/metrics/MetricsConsumer.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/metrics/MetricsConsumer.java
similarity index 96%
rename from external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/metrics/MetricsConsumer.java
rename to external/solr/src/main/java/org/apache/stormcrawler/solr/metrics/MetricsConsumer.java
index f219178b..15e98dc0 100644
--- a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/metrics/MetricsConsumer.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/metrics/MetricsConsumer.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.solr.metrics;
+package org.apache.stormcrawler.solr.metrics;
-import com.digitalpebble.stormcrawler.solr.SolrConnection;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Collection;
@@ -27,6 +25,8 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.storm.metric.api.IMetricsConsumer;
import org.apache.storm.task.IErrorReporter;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.solr.SolrConnection;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/persistence/SolrSpout.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/SolrSpout.java
similarity index 96%
rename from external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/persistence/SolrSpout.java
rename to external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/SolrSpout.java
index 04a0bd23..45c664eb 100644
--- a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/persistence/SolrSpout.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/SolrSpout.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.solr.persistence;
+package org.apache.stormcrawler.solr.persistence;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.AbstractQueryingSpout;
-import com.digitalpebble.stormcrawler.solr.SolrConnection;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.time.Instant;
import java.util.Collection;
import java.util.Iterator;
@@ -32,6 +28,10 @@ import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.AbstractQueryingSpout;
+import org.apache.stormcrawler.solr.SolrConnection;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/persistence/StatusUpdaterBolt.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/StatusUpdaterBolt.java
similarity index 88%
rename from external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/persistence/StatusUpdaterBolt.java
rename to external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/StatusUpdaterBolt.java
index 79604272..c05d09da 100644
--- a/external/solr/src/main/java/com/digitalpebble/stormcrawler/solr/persistence/StatusUpdaterBolt.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/StatusUpdaterBolt.java
@@ -12,14 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.solr.persistence;
-
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.AbstractStatusUpdaterBolt;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.solr.SolrConnection;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.URLUtil;
+package org.apache.stormcrawler.solr.persistence;
+
import java.util.Date;
import java.util.Iterator;
import java.util.Map;
@@ -29,6 +23,12 @@ import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.solr.SolrConnection;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/sql/pom.xml b/external/sql/pom.xml
index 889bb76f..b0630b07 100644
--- a/external/sql/pom.xml
+++ b/external/sql/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-external</artifactId>
<version>2.12-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
@@ -13,7 +13,7 @@
<packaging>jar</packaging>
<name>storm-crawler-sql</name>
- <url>https://github.com/DigitalPebble/storm-crawler/tree/master/external/sql</url>
+ <url>https://github.com/apache/incubator-stormcrawler/tree/master/external/sql</url>
<description>SQL-based resources for StormCrawler</description>
<dependencies>
diff --git a/external/sql/sql-conf.yaml b/external/sql/sql-conf.yaml
index a8ea8e7f..7e30d851 100644
--- a/external/sql/sql-conf.yaml
+++ b/external/sql/sql-conf.yaml
@@ -32,6 +32,6 @@ config:
# Metrics consumers:
topology.metrics.consumer.register:
- - class: "com.digitalpebble.stormcrawler.sql.metrics.MetricsConsumer"
+ - class: "org.apache.stormcrawler.sql.metrics.MetricsConsumer"
parallelism.hint: 1
diff --git a/external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/Constants.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/Constants.java
similarity index 96%
rename from external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/Constants.java
rename to external/sql/src/main/java/org/apache/stormcrawler/sql/Constants.java
index fd11751c..e5359f65 100644
--- a/external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/Constants.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/Constants.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.sql;
+package org.apache.stormcrawler.sql;
public class Constants {
diff --git a/external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/IndexerBolt.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/IndexerBolt.java
similarity index 94%
rename from external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/IndexerBolt.java
rename to external/sql/src/main/java/org/apache/stormcrawler/sql/IndexerBolt.java
index ff1e59df..ab713d6d 100644
--- a/external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/IndexerBolt.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/IndexerBolt.java
@@ -12,14 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.sql;
+package org.apache.stormcrawler.sql;
-import static com.digitalpebble.stormcrawler.Constants.StatusStreamName;
+import static org.apache.stormcrawler.Constants.StatusStreamName;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.indexing.AbstractIndexerBolt;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@@ -30,6 +26,10 @@ import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/SQLSpout.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLSpout.java
similarity index 96%
rename from external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/SQLSpout.java
rename to external/sql/src/main/java/org/apache/stormcrawler/sql/SQLSpout.java
index 1a03247e..bdd336f0 100644
--- a/external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/SQLSpout.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLSpout.java
@@ -12,12 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.sql;
+package org.apache.stormcrawler.sql;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.AbstractQueryingSpout;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.StringTabScheme;
import java.nio.ByteBuffer;
import java.sql.Connection;
import java.sql.ResultSet;
@@ -31,6 +27,10 @@ import org.apache.storm.spout.Scheme;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.AbstractQueryingSpout;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.StringTabScheme;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/SQLUtil.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLUtil.java
similarity index 97%
rename from external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/SQLUtil.java
rename to external/sql/src/main/java/org/apache/stormcrawler/sql/SQLUtil.java
index b6c270a0..636ec2af 100644
--- a/external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/SQLUtil.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLUtil.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.sql;
+package org.apache.stormcrawler.sql;
import java.sql.Connection;
import java.sql.DriverManager;
diff --git a/external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/StatusUpdaterBolt.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/StatusUpdaterBolt.java
similarity index 96%
rename from external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/StatusUpdaterBolt.java
rename to external/sql/src/main/java/org/apache/stormcrawler/sql/StatusUpdaterBolt.java
index d55d4df4..597bbfdd 100644
--- a/external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/StatusUpdaterBolt.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/StatusUpdaterBolt.java
@@ -12,13 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.sql;
+package org.apache.stormcrawler.sql;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.AbstractStatusUpdaterBolt;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.URLPartitioner;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@@ -35,6 +30,11 @@ import org.apache.storm.metric.api.MultiCountMetric;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLPartitioner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/metrics/MetricsConsumer.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/metrics/MetricsConsumer.java
similarity index 95%
rename from external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/metrics/MetricsConsumer.java
rename to external/sql/src/main/java/org/apache/stormcrawler/sql/metrics/MetricsConsumer.java
index 4a0f9936..69d25186 100644
--- a/external/sql/src/main/java/com/digitalpebble/stormcrawler/sql/metrics/MetricsConsumer.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/metrics/MetricsConsumer.java
@@ -12,11 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.sql.metrics;
+package org.apache.stormcrawler.sql.metrics;
-import com.digitalpebble.stormcrawler.sql.Constants;
-import com.digitalpebble.stormcrawler.sql.SQLUtil;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@@ -29,6 +26,9 @@ import java.util.Map.Entry;
import org.apache.storm.metric.api.IMetricsConsumer;
import org.apache.storm.task.IErrorReporter;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.sql.Constants;
+import org.apache.stormcrawler.sql.SQLUtil;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/tika/README.md b/external/tika/README.md
index 70f3904f..b3703722 100644
--- a/external/tika/README.md
+++ b/external/tika/README.md
@@ -4,7 +4,7 @@ Contains a bolt implementation which uses [Apache Tika](http://tika.apache.org/)
To use it alongside the JSoup parser i.e. let JSoup handle HTML content and Tika do everything else, you need to configure the JSoupParser with `jsoup.treat.non.html.as.error: false` so that documents that are not HTML don't get failed but passed on.
-The next step is to use a [RedirectionBolt](https://github.com/DigitalPebble/storm-crawler/blob/master/external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/RedirectionBolt.java) to send documents which have not been parsed with Jsoup to Tika on a bespoke stream called `tika`, finally the IndexingBolt needs to be connected to the outputs of both `shunt` and `tika` on the default stream. `tika` must also be connected to the StatusUpdaterBolt on the _status_ stream.
+The next step is to use a [RedirectionBolt](https://github.com/apache/incubator-stormcrawler/blob/master/external/tika/src/main/java/org/apache/stormcrawler/tika/RedirectionBolt.java) to send documents which have not been parsed with Jsoup to Tika on a bespoke stream called `tika`, finally the IndexingBolt needs to be connected to the outputs of both `shunt` and `tika` on the default stream. `tika` must also be connected to the StatusUpdaterBolt on the _status_ stream.
```
builder.setBolt("jsoup", new JSoupParserBolt()).localOrShuffleGrouping(
@@ -31,4 +31,4 @@ The next step is to use a [RedirectionBolt](https://github.com/DigitalPebble/sto
## Configure Tika
-The Tika parser bolt loads a Tika configuration file from the Java classpath. The default file name (path) is `tika-config.xml` and can be changed by the configuration `parser.tika.config.file`. See [configuring Tika](https://tika.apache.org/2.1.0/configuring.html) and the default configuration file [tika-config.xml](./src/main/resources/tika-config.xml).
\ No newline at end of file
+The Tika parser bolt loads a Tika configuration file from the Java classpath. The default file name (path) is `tika-config.xml` and can be changed by the configuration `parser.tika.config.file`. See [configuring Tika](https://tika.apache.org/2.1.0/configuring.html) and the default configuration file [tika-config.xml](./src/main/resources/tika-config.xml).
diff --git a/external/tika/pom.xml b/external/tika/pom.xml
index ca6ca14d..31c409db 100644
--- a/external/tika/pom.xml
+++ b/external/tika/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-external</artifactId>
<version>2.12-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
@@ -13,7 +13,7 @@
<packaging>jar</packaging>
<name>storm-crawler-tika</name>
- <url>https://github.com/DigitalPebble/storm-crawler/tree/master/external/tika</url>
+ <url>https://github.com/apache/incubator-stormcrawler/tree/master/external/tika</url>
<description>Tika-based parser bolt for StormCrawler</description>
<dependencies>
@@ -34,7 +34,7 @@
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
diff --git a/external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/DOMBuilder.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/DOMBuilder.java
similarity index 99%
rename from external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/DOMBuilder.java
rename to external/tika/src/main/java/org/apache/stormcrawler/tika/DOMBuilder.java
index 0a89c8c3..83bf2618 100644
--- a/external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/DOMBuilder.java
+++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/DOMBuilder.java
@@ -20,7 +20,7 @@
* avoid dependency on Xalan.
*/
-package com.digitalpebble.stormcrawler.tika;
+package org.apache.stormcrawler.tika;
import java.util.Locale;
import java.util.Stack;
diff --git a/external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
similarity index 93%
rename from external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/ParserBolt.java
rename to external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
index 1664efbd..571c6f72 100644
--- a/external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/ParserBolt.java
+++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
@@ -12,25 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.tika;
-
-import static com.digitalpebble.stormcrawler.Constants.StatusStreamName;
-
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.filtering.URLFilters;
-import com.digitalpebble.stormcrawler.parse.Outlink;
-import com.digitalpebble.stormcrawler.parse.ParseData;
-import com.digitalpebble.stormcrawler.parse.ParseFilter;
-import com.digitalpebble.stormcrawler.parse.ParseFilters;
-import com.digitalpebble.stormcrawler.parse.ParseResult;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.InitialisationUtil;
-import com.digitalpebble.stormcrawler.util.MetadataTransfer;
-import com.digitalpebble.stormcrawler.util.URLUtil;
+package org.apache.stormcrawler.tika;
+
+import static org.apache.stormcrawler.Constants.StatusStreamName;
+
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
@@ -46,6 +31,21 @@ import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilters;
+import org.apache.stormcrawler.parse.Outlink;
+import org.apache.stormcrawler.parse.ParseData;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseFilters;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.InitialisationUtil;
+import org.apache.stormcrawler.util.MetadataTransfer;
+import org.apache.stormcrawler.util.URLUtil;
import org.apache.tika.Tika;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.TikaCoreProperties;
diff --git a/external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/RedirectionBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/RedirectionBolt.java
similarity index 97%
rename from external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/RedirectionBolt.java
rename to external/tika/src/main/java/org/apache/stormcrawler/tika/RedirectionBolt.java
index abbe3a0c..2553178a 100644
--- a/external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/RedirectionBolt.java
+++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/RedirectionBolt.java
@@ -12,9 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.tika;
+package org.apache.stormcrawler.tika;
-import com.digitalpebble.stormcrawler.Metadata;
import java.util.Map;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -23,6 +22,7 @@ import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Metadata;
/**
* Uses Tika only if a document has not been parsed with anything else. Emits the tuples to be
diff --git a/external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/XMLCharacterRecognizer.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/XMLCharacterRecognizer.java
similarity index 98%
rename from external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/XMLCharacterRecognizer.java
rename to external/tika/src/main/java/org/apache/stormcrawler/tika/XMLCharacterRecognizer.java
index 620b318f..47a6ebca 100644
--- a/external/tika/src/main/java/com/digitalpebble/stormcrawler/tika/XMLCharacterRecognizer.java
+++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/XMLCharacterRecognizer.java
@@ -20,7 +20,7 @@
* avoid dependency on Xalan.
*/
-package com.digitalpebble.stormcrawler.tika;
+package org.apache.stormcrawler.tika;
/**
* Class used to verify whether the specified <var>ch</var> conforms to the XML 1.0 definition of
diff --git a/external/tika/src/test/java/com/digitalpebble/stormcrawler/tika/ParserBoltTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java
similarity index 90%
rename from external/tika/src/test/java/com/digitalpebble/stormcrawler/tika/ParserBoltTest.java
rename to external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java
index 1ccf263e..8cfdb321 100644
--- a/external/tika/src/test/java/com/digitalpebble/stormcrawler/tika/ParserBoltTest.java
+++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java
@@ -12,24 +12,24 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.tika;
+package org.apache.stormcrawler.tika;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.parse.ParsingTester;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.parse.ParsingTester;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
diff --git a/external/urlfrontier/README.md b/external/urlfrontier/README.md
index 8d44d997..ac9b5746 100644
--- a/external/urlfrontier/README.md
+++ b/external/urlfrontier/README.md
@@ -28,7 +28,7 @@ Your StormCrawler topology requires the following dependency in its pom.xml (jus
```
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-urlfrontier</artifactId>
<version>${stormcrawler.version}</version>
</dependency>
diff --git a/external/urlfrontier/pom.xml b/external/urlfrontier/pom.xml
index 67a4d41d..947564b1 100644
--- a/external/urlfrontier/pom.xml
+++ b/external/urlfrontier/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-external</artifactId>
<version>2.12-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
@@ -13,7 +13,7 @@
<packaging>jar</packaging>
<name>storm-crawler-urlfrontier</name>
- <url>https://github.com/DigitalPebble/storm-crawler/tree/master/external/urlfrontier</url>
+ <url>https://github.com/apache/incubator-stormcrawler/tree/master/external/urlfrontier</url>
<description>URL Frontier resources for StormCrawler</description>
<properties>
@@ -50,7 +50,7 @@
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
diff --git a/external/urlfrontier/src/main/java/com/digitalpebble/stormcrawler/urlfrontier/Constants.java b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Constants.java
similarity index 97%
rename from external/urlfrontier/src/main/java/com/digitalpebble/stormcrawler/urlfrontier/Constants.java
rename to external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Constants.java
index b890d616..3e3e7001 100644
--- a/external/urlfrontier/src/main/java/com/digitalpebble/stormcrawler/urlfrontier/Constants.java
+++ b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Constants.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.urlfrontier;
+package org.apache.stormcrawler.urlfrontier;
public final class Constants {
private Constants() {}
diff --git a/external/urlfrontier/src/main/java/com/digitalpebble/stormcrawler/urlfrontier/ManagedChannelUtil.java b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/ManagedChannelUtil.java
similarity index 93%
rename from external/urlfrontier/src/main/java/com/digitalpebble/stormcrawler/urlfrontier/ManagedChannelUtil.java
rename to external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/ManagedChannelUtil.java
index 6377bd59..7084ee13 100644
--- a/external/urlfrontier/src/main/java/com/digitalpebble/stormcrawler/urlfrontier/ManagedChannelUtil.java
+++ b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/ManagedChannelUtil.java
@@ -12,9 +12,9 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.urlfrontier;
+package org.apache.stormcrawler.urlfrontier;
-import static com.digitalpebble.stormcrawler.urlfrontier.Constants.URLFRONTIER_DEFAULT_PORT;
+import static org.apache.stormcrawler.urlfrontier.Constants.URLFRONTIER_DEFAULT_PORT;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
diff --git a/external/urlfrontier/src/main/java/com/digitalpebble/stormcrawler/urlfrontier/Spout.java b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Spout.java
similarity index 96%
rename from external/urlfrontier/src/main/java/com/digitalpebble/stormcrawler/urlfrontier/Spout.java
rename to external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Spout.java
index 36ff2e7d..a68f5d2f 100644
--- a/external/urlfrontier/src/main/java/com/digitalpebble/stormcrawler/urlfrontier/Spout.java
+++ b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Spout.java
@@ -12,13 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.urlfrontier;
+package org.apache.stormcrawler.urlfrontier;
-import static com.digitalpebble.stormcrawler.urlfrontier.Constants.*;
+import static org.apache.stormcrawler.urlfrontier.Constants.*;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.AbstractQueryingSpout;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import crawlercommons.urlfrontier.URLFrontierGrpc;
import crawlercommons.urlfrontier.URLFrontierGrpc.URLFrontierStub;
import crawlercommons.urlfrontier.Urlfrontier.GetParams;
@@ -32,6 +29,9 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.storm.Config;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.AbstractQueryingSpout;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/urlfrontier/src/main/java/com/digitalpebble/stormcrawler/urlfrontier/StatusUpdaterBolt.java b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBolt.java
similarity index 97%
rename from external/urlfrontier/src/main/java/com/digitalpebble/stormcrawler/urlfrontier/StatusUpdaterBolt.java
rename to external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBolt.java
index e718169a..4407764f 100644
--- a/external/urlfrontier/src/main/java/com/digitalpebble/stormcrawler/urlfrontier/StatusUpdaterBolt.java
+++ b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBolt.java
@@ -12,15 +12,10 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.urlfrontier;
+package org.apache.stormcrawler.urlfrontier;
-import static com.digitalpebble.stormcrawler.urlfrontier.Constants.*;
+import static org.apache.stormcrawler.urlfrontier.Constants.*;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.AbstractStatusUpdaterBolt;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
-import com.digitalpebble.stormcrawler.util.URLPartitioner;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
@@ -53,6 +48,11 @@ import org.apache.storm.metric.api.MultiCountMetric;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLPartitioner;
import org.apache.tika.utils.StringUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
diff --git a/external/urlfrontier/src/test/java/com/digitalpebble/stormcrawler/urlfrontier/StatusUpdaterBoltTest.java b/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBoltTest.java
similarity index 92%
rename from external/urlfrontier/src/test/java/com/digitalpebble/stormcrawler/urlfrontier/StatusUpdaterBoltTest.java
rename to external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBoltTest.java
index a1edea16..c783e9dc 100644
--- a/external/urlfrontier/src/test/java/com/digitalpebble/stormcrawler/urlfrontier/StatusUpdaterBoltTest.java
+++ b/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBoltTest.java
@@ -12,15 +12,11 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.urlfrontier;
+package org.apache.stormcrawler.urlfrontier;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestOutputCollector;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.persistence.Status;
import java.util.HashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -30,6 +26,10 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestOutputCollector;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.persistence.Status;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -79,12 +79,10 @@ public class StatusUpdaterBoltTest {
final var config = new HashMap<String, Object>();
config.put(
- "urlbuffer.class",
- "com.digitalpebble.stormcrawler.persistence.urlbuffer.SimpleURLBuffer");
+ "urlbuffer.class", "org.apache.stormcrawler.persistence.urlbuffer.SimpleURLBuffer");
config.put(Constants.URLFRONTIER_HOST_KEY, connection.getHost());
config.put(Constants.URLFRONTIER_PORT_KEY, connection.getPort());
- config.put(
- "scheduler.class", "com.digitalpebble.stormcrawler.persistence.DefaultScheduler");
+ config.put("scheduler.class", "org.apache.stormcrawler.persistence.DefaultScheduler");
config.put("status.updater.cache.spec", "maximumSize=10000,expireAfterAccess=1h");
config.put("metadata.persist", persistedKey);
config.put("urlfrontier.updater.max.messages", 1);
diff --git a/external/urlfrontier/src/test/java/com/digitalpebble/stormcrawler/urlfrontier/URLFrontierContainer.java b/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainer.java
similarity index 98%
rename from external/urlfrontier/src/test/java/com/digitalpebble/stormcrawler/urlfrontier/URLFrontierContainer.java
rename to external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainer.java
index 5b160567..5475f52d 100644
--- a/external/urlfrontier/src/test/java/com/digitalpebble/stormcrawler/urlfrontier/URLFrontierContainer.java
+++ b/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainer.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.urlfrontier;
+package org.apache.stormcrawler.urlfrontier;
import java.util.ArrayList;
import java.util.List;
diff --git a/external/urlfrontier/src/test/java/com/digitalpebble/stormcrawler/urlfrontier/URLFrontierContainerConfig.java b/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainerConfig.java
similarity index 95%
rename from external/urlfrontier/src/test/java/com/digitalpebble/stormcrawler/urlfrontier/URLFrontierContainerConfig.java
rename to external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainerConfig.java
index 93316ec1..7d5ee16f 100644
--- a/external/urlfrontier/src/test/java/com/digitalpebble/stormcrawler/urlfrontier/URLFrontierContainerConfig.java
+++ b/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainerConfig.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.urlfrontier;
+package org.apache.stormcrawler.urlfrontier;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
diff --git a/external/warc/README.md b/external/warc/README.md
index c51b0647..9b56bbba 100644
--- a/external/warc/README.md
+++ b/external/warc/README.md
@@ -4,7 +4,7 @@ First, you need to add the WARC module to the dependencies of your project.
```
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-warc</artifactId>
<version>${storm-crawler.version}</version>
</dependency>
@@ -59,7 +59,7 @@ If you use Flux, you could add it like so:
```
components:
- id: "WARCFileNameFormat"
- className: "com.digitalpebble.stormcrawler.warc.WARCFileNameFormat"
+ className: "org.apache.stormcrawler.warc.WARCFileNameFormat"
configMethods:
- name: "withPath"
args:
@@ -94,7 +94,7 @@ components:
bolts:
- id: "warc"
- className: "com.digitalpebble.stormcrawler.warc.WARCHdfsBolt"
+ className: "org.apache.stormcrawler.warc.WARCHdfsBolt"
parallelism: 1
configMethods:
- name: "withFileNameFormat"
@@ -149,8 +149,8 @@ Writing complete and valid WARC requires that HTTP headers, IP address and captu
```
http.store.headers: true
- http.protocol.implementation: com.digitalpebble.stormcrawler.protocol.okhttp.HttpProtocol
- https.protocol.implementation: com.digitalpebble.stormcrawler.protocol.okhttp.HttpProtocol
+ http.protocol.implementation: org.apache.stormcrawler.protocol.okhttp.HttpProtocol
+ https.protocol.implementation: org.apache.stormcrawler.protocol.okhttp.HttpProtocol
```
A note on the recording of HTTP requests and responses with StormCrawler and the WARC module:
@@ -209,7 +209,7 @@ Or, if Flux is used:
```
spouts:
- id: "spout"
- className: "com.digitalpebble.stormcrawler.warc.WARCSpout"
+ className: "org.apache.stormcrawler.warc.WARCSpout"
parallelism: 1
constructorArgs:
- "input/"
diff --git a/external/warc/pom.xml b/external/warc/pom.xml
index 9dd0baba..432d0d83 100644
--- a/external/warc/pom.xml
+++ b/external/warc/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-external</artifactId>
<version>2.12-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
@@ -13,7 +13,7 @@
<packaging>jar</packaging>
<name>storm-crawler-warc</name>
- <url>https://github.com/DigitalPebble/storm-crawler/tree/master/external/warc</url>
+ <url>https://github.com/apache/incubator-stormcrawler/tree/master/external/warc</url>
<description>WARC resources for StormCrawler</description>
<properties>
@@ -25,7 +25,7 @@
<!-- Need to set this one explicitly otherwise does not get the right
codec dependencies ?!?!? -->
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-core</artifactId>
<version>${project.version}</version>
</dependency>
@@ -63,7 +63,7 @@
</dependency>
<dependency>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
diff --git a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/FileTimeSizeRotationPolicy.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/FileTimeSizeRotationPolicy.java
similarity index 98%
rename from external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/FileTimeSizeRotationPolicy.java
rename to external/warc/src/main/java/org/apache/stormcrawler/warc/FileTimeSizeRotationPolicy.java
index 6db6240f..cbfec6c5 100644
--- a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/FileTimeSizeRotationPolicy.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/FileTimeSizeRotationPolicy.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.warc;
+package org.apache.stormcrawler.warc;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy;
diff --git a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/GzipHdfsBolt.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/GzipHdfsBolt.java
similarity index 99%
rename from external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/GzipHdfsBolt.java
rename to external/warc/src/main/java/org/apache/stormcrawler/warc/GzipHdfsBolt.java
index 9da2d061..68a17c6d 100644
--- a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/GzipHdfsBolt.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/GzipHdfsBolt.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.warc;
+package org.apache.stormcrawler.warc;
import java.io.IOException;
import java.util.ArrayList;
diff --git a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCFileNameFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCFileNameFormat.java
similarity index 98%
rename from external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCFileNameFormat.java
rename to external/warc/src/main/java/org/apache/stormcrawler/warc/WARCFileNameFormat.java
index 2602cff0..39e4a502 100644
--- a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCFileNameFormat.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCFileNameFormat.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.warc;
+package org.apache.stormcrawler.warc;
import java.text.SimpleDateFormat;
import java.util.Date;
diff --git a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCHdfsBolt.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java
similarity index 95%
rename from external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCHdfsBolt.java
rename to external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java
index 4b6177f6..c0d4bf94 100644
--- a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCHdfsBolt.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java
@@ -12,10 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.warc;
+package org.apache.stormcrawler.warc;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
import java.io.IOException;
import java.time.Instant;
import java.util.HashMap;
@@ -29,6 +27,8 @@ import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
similarity index 96%
rename from external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCRecordFormat.java
rename to external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
index ad4e10d4..29b0c264 100644
--- a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCRecordFormat.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
@@ -12,15 +12,12 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.warc;
+package org.apache.stormcrawler.warc;
-import static com.digitalpebble.stormcrawler.protocol.ProtocolResponse.REQUEST_TIME_KEY;
-import static com.digitalpebble.stormcrawler.protocol.ProtocolResponse.RESPONSE_HEADERS_KEY;
-import static com.digitalpebble.stormcrawler.protocol.ProtocolResponse.RESPONSE_IP_KEY;
+import static org.apache.stormcrawler.protocol.ProtocolResponse.REQUEST_TIME_KEY;
+import static org.apache.stormcrawler.protocol.ProtocolResponse.RESPONSE_HEADERS_KEY;
+import static org.apache.stormcrawler.protocol.ProtocolResponse.RESPONSE_IP_KEY;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@@ -40,6 +37,9 @@ import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.hdfs.bolt.format.RecordFormat;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -293,8 +293,8 @@ public class WARCRecordFormat implements RecordFormat {
/**
* Get the actual fetch time from metadata and format it as required by the WARC-Date field. If
* no fetch time is found in metadata (key {@link
- * com.digitalpebble.stormcrawler.protocol.ProtocolResponse#REQUEST_TIME_KEY REQUEST_TIME_KEY}),
- * the current time is taken.
+ * org.apache.stormcrawler.protocol.ProtocolResponse#REQUEST_TIME_KEY REQUEST_TIME_KEY}), the
+ * current time is taken.
*/
protected String getCaptureTime(Metadata metadata) {
String captureTimeMillis = metadata.getFirstValue(REQUEST_TIME_KEY, this.protocolMDprefix);
diff --git a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCRequestRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java
similarity index 96%
rename from external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCRequestRecordFormat.java
rename to external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java
index f55a23f6..022d26b3 100644
--- a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCRequestRecordFormat.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java
@@ -12,12 +12,11 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.warc;
+package org.apache.stormcrawler.warc;
-import static com.digitalpebble.stormcrawler.protocol.ProtocolResponse.REQUEST_HEADERS_KEY;
-import static com.digitalpebble.stormcrawler.protocol.ProtocolResponse.RESPONSE_IP_KEY;
+import static org.apache.stormcrawler.protocol.ProtocolResponse.REQUEST_HEADERS_KEY;
+import static org.apache.stormcrawler.protocol.ProtocolResponse.RESPONSE_IP_KEY;
-import com.digitalpebble.stormcrawler.Metadata;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@@ -25,6 +24,7 @@ import java.util.UUID;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCSpout.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java
similarity index 97%
rename from external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCSpout.java
rename to external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java
index 3fdff4fe..f7147318 100644
--- a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCSpout.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java
@@ -12,14 +12,8 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.warc;
-
-import com.digitalpebble.stormcrawler.Constants;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.persistence.Status;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
-import com.digitalpebble.stormcrawler.spout.FileSpout;
-import com.digitalpebble.stormcrawler.util.ConfUtils;
+package org.apache.stormcrawler.warc;
+
import java.io.IOException;
import java.net.URL;
import java.nio.ByteBuffer;
@@ -40,6 +34,12 @@ import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.spout.FileSpout;
+import org.apache.stormcrawler.util.ConfUtils;
import org.netpreserve.jwarc.HttpMessage;
import org.netpreserve.jwarc.HttpRequest;
import org.netpreserve.jwarc.HttpResponse;
@@ -57,7 +57,7 @@ import org.slf4j.LoggerFactory;
/**
* Read WARC files from the local files system and emit the WARC captures as tuples into the
- * topology same way as done by {@link com.digitalpebble.stormcrawler.bolt.FetcherBolt}.
+ * topology same way as done by {@link org.apache.stormcrawler.bolt.FetcherBolt}.
*/
public class WARCSpout extends FileSpout {
diff --git a/external/warc/src/test/java/com/digitalpebble/stormcrawler/warc/WARCHdfsBoltTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java
similarity index 97%
rename from external/warc/src/test/java/com/digitalpebble/stormcrawler/warc/WARCHdfsBoltTest.java
rename to external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java
index 8b00a027..dc48c620 100644
--- a/external/warc/src/test/java/com/digitalpebble/stormcrawler/warc/WARCHdfsBoltTest.java
+++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java
@@ -12,17 +12,13 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.warc;
+package org.apache.stormcrawler.warc;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.TestOutputCollector;
-import com.digitalpebble.stormcrawler.TestUtil;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
@@ -36,6 +32,10 @@ import java.util.stream.Stream;
import org.apache.storm.hdfs.bolt.HdfsBolt;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestOutputCollector;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
diff --git a/external/warc/src/test/java/com/digitalpebble/stormcrawler/warc/WARCRecordFormatTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java
similarity index 98%
rename from external/warc/src/test/java/com/digitalpebble/stormcrawler/warc/WARCRecordFormatTest.java
rename to external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java
index 65b73a35..197d68d2 100644
--- a/external/warc/src/test/java/com/digitalpebble/stormcrawler/warc/WARCRecordFormatTest.java
+++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java
@@ -12,7 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.warc;
+package org.apache.stormcrawler.warc;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -20,11 +20,11 @@ import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import com.digitalpebble.stormcrawler.Metadata;
-import com.digitalpebble.stormcrawler.protocol.HttpHeaders;
-import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
import java.nio.charset.StandardCharsets;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.HttpHeaders;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.junit.Test;
public class WARCRecordFormatTest {
diff --git a/external/warc/src/test/java/com/digitalpebble/stormcrawler/warc/WARCSpoutTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCSpoutTest.java
similarity index 93%
rename from external/warc/src/test/java/com/digitalpebble/stormcrawler/warc/WARCSpoutTest.java
rename to external/warc/src/test/java/org/apache/stormcrawler/warc/WARCSpoutTest.java
index 8ffd0fe3..1fb6235b 100644
--- a/external/warc/src/test/java/com/digitalpebble/stormcrawler/warc/WARCSpoutTest.java
+++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCSpoutTest.java
@@ -12,16 +12,16 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.digitalpebble.stormcrawler.warc;
+package org.apache.stormcrawler.warc;
import static org.junit.Assert.assertEquals;
-import com.digitalpebble.stormcrawler.TestOutputCollector;
-import com.digitalpebble.stormcrawler.TestUtil;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.storm.spout.SpoutOutputCollector;
+import org.apache.stormcrawler.TestOutputCollector;
+import org.apache.stormcrawler.TestUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
diff --git a/pom.xml b/pom.xml
index 9cf46ef7..0e168afc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2,7 +2,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.digitalpebble.stormcrawler</groupId>
+ <groupId>org.apache.stormcrawler</groupId>
<artifactId>storm-crawler</artifactId>
<version>2.12-SNAPSHOT</version>
<packaging>pom</packaging>
@@ -19,11 +19,6 @@
</license>
</licenses>
- <organization>
- <name>DigitalPebble Ltd</name>
- <url>http://www.digitalpebble.com</url>
- </organization>
-
<developers>
<developer>
<name>Julien Nioche</name>
@@ -31,46 +26,43 @@
<organization>DigitalPebble</organization>
<organizationUrl>http://www.digitalpebble.com</organizationUrl>
</developer>
- </developers>
-
- <contributors>
- <contributor>
+ <developer>
<name>Sebastian Nagel</name>
<email>sebastian@commoncrawl.org</email>
<organization>CommonCrawl</organization>
<organizationUrl>http://commoncrawl.org</organizationUrl>
- </contributor>
- <contributor>
+ </developer>
+ <developer>
<name>Jorge Luis Betancourt</name>
<email>betancourt.jorge@gmail.com</email>
<organization>trivago</organization>
<organizationUrl>https://www.trivago.com</organizationUrl>
- </contributor>
- <contributor>
+ </developer>
+ <developer>
<name>Richard Zowalla</name>
<email>richard.zowalla@hs-heilbronn.de</email>
<organization>Heilbronn University</organization>
<organizationUrl>https://www.mi.hs-heilbronn.de</organizationUrl>
- </contributor>
- <contributor>
+ </developer>
+ <developer>
<name>Michael Dinzinger</name>
<email>michael.dinzinger@uni-passau.de</email>
<organization>Universität Passau</organization>
<organizationUrl>https://www.fim.uni-passau.de</organizationUrl>
- </contributor>
- </contributors>
+ </developer>
+ </developers>
<scm>
- <connection>scm:git:https://github.com/DigitalPebble/storm-crawler.git</connection>
+ <connection>scm:git:https://github.com/apache/incubator-stormcrawler.git</connection>
<developerConnection>
- scm:git:git@github.com:DigitalPebble/storm-crawler.git</developerConnection>
- <url>https://github.com/DigitalPebble/storm-crawler</url>
+ scm:git:git@github.com:apache/incubator-stormcrawler.git</developerConnection>
+ <url>https://github.com/apache/incubator-stormcrawler</url>
<tag>HEAD</tag>
</scm>
<issueManagement>
<system>GitHub Issues</system>
- <url>https://github.com/DigitalPebble/storm-crawler/issues</url>
+ <url>https://github.com/apache/incubator-stormcrawler/issues</url>
</issueManagement>
<properties>