You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by si...@apache.org on 2026/03/01 16:33:06 UTC
(stormcrawler) 01/02: Fix issues reported by checkstyle
This is an automated email from the ASF dual-hosted git repository.
sigee pushed a commit to branch fix-checkstyle-issues
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
commit 63ebed3727c4ae3aa57bc936f2b273f932a71593
Author: Dávid Szigecsán <si...@apache.org>
AuthorDate: Sun Mar 1 09:24:27 2026 +0100
Fix issues reported by checkstyle
---
.../java/org/apache/stormcrawler/Metadata.java | 35 +++++++++-------
.../apache/stormcrawler/bolt/JSoupParserBolt.java | 5 +--
.../apache/stormcrawler/filtering/URLFilters.java | 30 +++++++-------
.../filtering/regex/FastURLFilter.java | 4 +-
.../stormcrawler/indexing/AbstractIndexerBolt.java | 3 +-
.../apache/stormcrawler/parse/JSoupFilters.java | 24 +++++------
.../apache/stormcrawler/parse/ParseFilters.java | 22 +++++-----
.../stormcrawler/parse/filter/XPathFilter.java | 3 ++
.../protocol/HttpRobotRulesParser.java | 5 +--
.../stormcrawler/protocol/okhttp/HttpProtocol.java | 2 +-
.../java/org/apache/stormcrawler/MetadataTest.java | 1 +
.../stormcrawler/TestMetadataSerialization.java | 1 +
.../apache/stormcrawler/TestOutputCollector.java | 23 ++++++-----
.../java/org/apache/stormcrawler/TestUtil.java | 1 +
.../stormcrawler/bolt/AbstractFetcherBoltTest.java | 1 +
.../stormcrawler/bolt/FeedParserBoltTest.java | 1 +
.../apache/stormcrawler/bolt/FetcherBoltTest.java | 1 +
.../stormcrawler/bolt/JSoupParserBoltTest.java | 1 +
.../stormcrawler/bolt/SimpleFetcherBoltTest.java | 1 +
.../stormcrawler/bolt/SiteMapParserBoltTest.java | 1 +
.../stormcrawler/filtering/BasicURLFilterTest.java | 1 +
.../filtering/BasicURLNormalizerTest.java | 1 +
.../stormcrawler/filtering/FastURLFilterTest.java | 1 +
.../stormcrawler/filtering/HostURLFilterTest.java | 1 +
.../stormcrawler/filtering/MaxDepthFilterTest.java | 1 +
.../filtering/MetadataFilterFromJsonTest.java | 1 +
.../stormcrawler/filtering/MetadataFilterTest.java | 1 +
.../stormcrawler/filtering/RegexFilterTest.java | 1 +
.../ClassInheritingFromAbstractAndInterface.java | 1 +
.../ClassInheritingFromAbstractClassOnly.java | 1 +
.../ClassInheritingFromOpenClass.java | 1 +
.../ClassWithoutValidConstructor.java | 1 +
.../initialisation/FinalClassToInitialize.java | 1 +
.../helper/initialisation/SimpleOpenClass.java | 1 +
.../helper/initialisation/base/AbstractClass.java | 1 +
.../helper/initialisation/base/ITestInterface.java | 1 +
.../OpenClassWithAbstractClassAndInterface.java | 1 +
.../stormcrawler/indexer/BasicIndexingTest.java | 1 +
.../apache/stormcrawler/indexer/DummyIndexer.java | 13 +++---
.../apache/stormcrawler/indexer/IndexerTester.java | 1 +
.../apache/stormcrawler/json/JsoupFilterTest.java | 1 +
.../stormcrawler/jsoup/JSoupFiltersTest.java | 1 +
.../stormcrawler/parse/DuplicateLinksTest.java | 1 +
.../stormcrawler/parse/JSoupTextExtractorTest.java | 1 +
.../apache/stormcrawler/parse/ParsingTester.java | 1 +
.../stormcrawler/parse/StackOverflowTest.java | 5 ++-
.../parse/filter/CSVMetadataFilterTest.java | 1 +
.../parse/filter/CollectionTaggerTest.java | 1 +
.../parse/filter/SubDocumentsFilterTest.java | 1 +
.../parse/filter/SubDocumentsParseFilter.java | 1 +
.../stormcrawler/parse/filter/XPathFilterTest.java | 1 +
.../persistence/AdaptiveSchedulerTest.java | 1 +
.../persistence/DefaultSchedulerTest.java | 1 +
.../stormcrawler/persistence/URLBufferTest.java | 1 +
.../protocol/AbstractProtocolTest.java | 5 ++-
.../protocol/DelegationProtocolTest.java | 1 +
.../stormcrawler/protocol/DummyProtocol.java | 1 +
.../protocol/HttpRobotRulesParserRedirectTest.java | 1 +
.../protocol/HttpRobotRulesParserTest.java | 1 +
.../stormcrawler/proxy/MultiProxyManagerTest.java | 1 +
.../org/apache/stormcrawler/proxy/SCProxyTest.java | 1 +
.../stormcrawler/proxy/SingleProxyManagerTest.java | 1 +
.../apache/stormcrawler/spout/FileSpoutTest.java | 1 +
.../spout/mocks/FileSpoutOutputCollectorMock.java | 1 +
.../spout/mocks/FileSpoutTopologyContextMock.java | 1 +
.../apache/stormcrawler/util/ConfUtilsTest.java | 1 +
.../stormcrawler/util/CookieConverterTest.java | 1 +
.../stormcrawler/util/InitialisationUtilTest.java | 1 +
.../stormcrawler/util/MetadataTransferTest.java | 1 +
.../apache/stormcrawler/util/RefreshTagTest.java | 1 +
.../apache/stormcrawler/util/RobotsTagsTest.java | 1 +
.../stormcrawler/ai/AbstractLLMTextExtractor.java | 1 +
.../stormcrawler/ai/OpenAITextExtractor.java | 1 +
.../ai/listener/LlmResponseListener.java | 1 +
.../stormcrawler/ai/listener/NoOpListener.java | 1 +
.../stormcrawler/ai/OpenAITextExtractorTest.java | 1 +
.../aws/bolt/CloudSearchConstants.java | 1 +
.../aws/bolt/CloudSearchIndexerBolt.java | 15 +++----
.../stormcrawler/aws/bolt/CloudSearchUtils.java | 10 +++--
.../stormcrawler/aws/s3/AbstractS3CacheBolt.java | 1 +
.../apache/stormcrawler/aws/s3/S3CacheChecker.java | 1 +
.../org/apache/stormcrawler/aws/s3/S3Cacher.java | 1 +
.../stormcrawler/aws/s3/S3ContentCacher.java | 3 +-
.../stormcrawler/parse/filter/LanguageID.java | 1 +
.../opensearch/BulkItemResponseToFailedFlag.java | 17 ++++++--
.../apache/stormcrawler/opensearch/Constants.java | 1 +
.../stormcrawler/opensearch/IndexCreation.java | 1 +
.../opensearch/OpenSearchConnection.java | 14 +++----
.../stormcrawler/opensearch/bolt/DeletionBolt.java | 16 ++++++--
.../stormcrawler/opensearch/bolt/IndexerBolt.java | 21 ++++++----
.../opensearch/filtering/JSONURLFilterWrapper.java | 1 +
.../opensearch/metrics/MetricsConsumer.java | 9 ++++-
.../opensearch/metrics/StatusMetricsBolt.java | 1 +
.../parse/filter/JSONResourceWrapper.java | 1 +
.../opensearch/persistence/AbstractSpout.java | 9 +++--
.../opensearch/persistence/AggregationSpout.java | 5 ++-
.../opensearch/persistence/HybridSpout.java | 14 ++++---
.../opensearch/persistence/StatusUpdaterBolt.java | 12 ++++--
.../opensearch/bolt/AbstractOpenSearchTest.java | 1 +
.../opensearch/bolt/IndexerBoltTest.java | 1 +
.../opensearch/bolt/StatusBoltTest.java | 1 +
.../protocol/playwright/HttpProtocol.java | 4 +-
.../protocol/playwright/LocalResourceHandler.java | 8 +++-
.../protocol/playwright/ProtocolTest.java | 1 +
.../protocol/selenium/NavigationFilter.java | 1 +
.../protocol/selenium/NavigationFilters.java | 47 ++++++++++++----------
.../protocol/selenium/RemoteDriverProtocol.java | 5 ++-
.../protocol/selenium/SeleniumProtocol.java | 1 +
.../protocol/selenium/ProtocolTest.java | 1 +
.../org/apache/stormcrawler/solr/Constants.java | 1 +
.../apache/stormcrawler/solr/SolrConnection.java | 9 ++++-
.../stormcrawler/solr/bolt/DeletionBolt.java | 6 ++-
.../apache/stormcrawler/solr/bolt/IndexerBolt.java | 4 +-
.../stormcrawler/solr/metrics/MetricsConsumer.java | 1 +
.../stormcrawler/solr/persistence/SolrSpout.java | 8 ++--
.../solr/persistence/StatusUpdaterBolt.java | 1 +
.../stormcrawler/solr/CloudConnectionTest.java | 1 +
.../stormcrawler/solr/SolrCloudContainerTest.java | 1 +
.../stormcrawler/solr/SolrContainerTest.java | 1 +
.../stormcrawler/solr/bolt/IndexerBoltTest.java | 1 +
.../stormcrawler/solr/persistence/SpoutTest.java | 6 ++-
.../solr/persistence/StatusBoltTest.java | 1 +
.../org/apache/stormcrawler/sql/Constants.java | 1 +
.../org/apache/stormcrawler/sql/IndexerBolt.java | 1 +
.../java/org/apache/stormcrawler/sql/SQLSpout.java | 41 ++++++++++---------
.../java/org/apache/stormcrawler/sql/SQLUtil.java | 1 +
.../apache/stormcrawler/sql/StatusUpdaterBolt.java | 25 +++++-------
.../stormcrawler/sql/metrics/MetricsConsumer.java | 1 +
.../apache/stormcrawler/sql/AbstractSQLTest.java | 1 +
.../apache/stormcrawler/sql/IndexerBoltTest.java | 1 +
.../org/apache/stormcrawler/sql/SQLSpoutTest.java | 9 +++--
.../stormcrawler/sql/StatusUpdaterBoltTest.java | 1 +
.../org/apache/stormcrawler/tika/DOMBuilder.java | 10 ++---
.../org/apache/stormcrawler/tika/ParserBolt.java | 3 +-
.../apache/stormcrawler/tika/RedirectionBolt.java | 1 +
.../stormcrawler/tika/XMLCharacterRecognizer.java | 14 +++++--
.../apache/stormcrawler/tika/ParserBoltTest.java | 1 +
.../apache/stormcrawler/urlfrontier/Constants.java | 1 +
.../urlfrontier/ManagedChannelUtil.java | 1 +
.../org/apache/stormcrawler/urlfrontier/Spout.java | 1 +
.../urlfrontier/StatusUpdaterBolt.java | 5 ++-
.../urlfrontier/StatusUpdaterBoltTest.java | 5 ++-
.../urlfrontier/URLFrontierContainer.java | 5 ++-
.../urlfrontier/URLFrontierContainerConfig.java | 1 +
.../warc/FileTimeSizeRotationPolicy.java | 3 +-
.../org/apache/stormcrawler/warc/GzipHdfsBolt.java | 1 +
.../stormcrawler/warc/MetadataRecordFormat.java | 21 +++++++---
.../stormcrawler/warc/WARCFileNameFormat.java | 1 +
.../org/apache/stormcrawler/warc/WARCHdfsBolt.java | 1 +
.../apache/stormcrawler/warc/WARCRecordFormat.java | 15 ++++---
.../stormcrawler/warc/WARCRequestRecordFormat.java | 6 ++-
.../org/apache/stormcrawler/warc/WARCSpout.java | 36 ++++++++++++-----
.../apache/stormcrawler/warc/WARCHdfsBoltTest.java | 9 ++++-
.../stormcrawler/warc/WARCRecordFormatTest.java | 5 ++-
.../apache/stormcrawler/warc/WARCSpoutTest.java | 1 +
155 files changed, 497 insertions(+), 240 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/Metadata.java b/core/src/main/java/org/apache/stormcrawler/Metadata.java
index 8b5c9eb7..0661908b 100644
--- a/core/src/main/java/org/apache/stormcrawler/Metadata.java
+++ b/core/src/main/java/org/apache/stormcrawler/Metadata.java
@@ -88,7 +88,9 @@ public class Metadata {
}
/**
- * @return the first value for the key or null if it does not exist *
+ * Returns the first non-empty value found for the keys or null if none found.
+ *
+ * @return the first value for the key or null if it does not exist
*/
public String getFirstValue(String key) {
String[] values = getValues(key);
@@ -102,6 +104,8 @@ public class Metadata {
}
/**
+ * Returns the first non-empty value found for the keys or null if none found.
+ *
* @return the first value for the key or null if it does not exist, given a prefix
*/
public String getFirstValue(String key, String prefix) {
@@ -111,6 +115,21 @@ public class Metadata {
return getFirstValue(prefix + key);
}
+ /** Returns the first non-empty value found for the keys or null if none found.
+ *
+ * @return the first value for the key or null if it does not exist, given a prefix
+ */
+ public static String getFirstValue(Metadata md, String... keys) {
+ for (String key : keys) {
+ String val = md.getFirstValue(key);
+ if (StringUtils.isBlank(val)) {
+ continue;
+ }
+ return val;
+ }
+ return null;
+ }
+
public String[] getValues(String key, String prefix) {
if (prefix == null || prefix.length() == 0) {
return getValues(key);
@@ -245,18 +264,6 @@ public class Metadata {
.collect(Collectors.toSet());
}
- /** Returns the first non empty value found for the keys or null if none found. */
- public static String getFirstValue(Metadata md, String... keys) {
- for (String key : keys) {
- String val = md.getFirstValue(key);
- if (StringUtils.isBlank(val)) {
- continue;
- }
- return val;
- }
- return null;
- }
-
/**
* Copies the values arrays for a given key to another metadata object.
*
@@ -288,7 +295,7 @@ public class Metadata {
}
/**
- * Release the lock on a metadata
+ * Release the lock on a metadata.
*
* @since 1.16
*/
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
index 933f41bd..794eb0ed 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
@@ -221,9 +221,8 @@ public class JSoupParserBolt extends StatusEmitterBolt {
if (mimeType.toLowerCase(Locale.ROOT).contains("html")) {
contentTypeIsOk = true;
}
- }
- // go ahead even if no mimetype is available
- else {
+ } else {
+ // go ahead even if no mimetype is available
contentTypeIsOk = true;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java b/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java
index 59f36b6f..daeec876 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java
@@ -59,6 +59,21 @@ public class URLFilters extends URLFilter implements JSONResource {
filters = new URLFilters[0];
}
+ /**
+ * Loads the filters from a JSON configuration file.
+ *
+ * @throws IOException
+ */
+ public URLFilters(Map<String, Object> stormConf, String configFile) throws IOException {
+ this.configFile = configFile;
+ this.stormConf = stormConf;
+ try {
+ loadJSONResources();
+ } catch (Exception e) {
+ throw new IOException("Unable to build JSON object from file", e);
+ }
+ }
+
private String configFile = "urlfilters.json";
private Map<String, Object> stormConf;
@@ -83,21 +98,6 @@ public class URLFilters extends URLFilter implements JSONResource {
return URLFilters.emptyURLFilters;
}
- /**
- * Loads the filters from a JSON configuration file.
- *
- * @throws IOException
- */
- public URLFilters(Map<String, Object> stormConf, String configFile) throws IOException {
- this.configFile = configFile;
- this.stormConf = stormConf;
- try {
- loadJSONResources();
- } catch (Exception e) {
- throw new IOException("Unable to build JSON object from file", e);
- }
- }
-
@Override
public void loadJSONResources(InputStream inputStream)
throws JsonParseException, JsonMappingException, IOException {
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java
index 854b464a..fdf18af5 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java
@@ -284,7 +284,7 @@ class Scope {
GLOBAL,
HOSTNAME,
METADATA
- };
+ }
protected Rule[] rules;
@@ -329,7 +329,7 @@ class Rule {
DENYPATHQUERY,
ALLOWPATH,
ALLOWPATHQUERY
- };
+ }
private Type type;
private Pattern pattern;
diff --git a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
index 91175846..b6165906 100644
--- a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
@@ -105,7 +105,8 @@ public abstract class AbstractIndexerBolt extends BaseRichBolt {
// or an index
if (index != -1 || alias != null) {
throw new RuntimeException(
- "Can't have a mapping for indexer.md.mapping with a glob and index or alias");
+ "Can't have a mapping for indexer.md.mapping with a glob and index or "
+ + "alias");
}
} else {
this.key = key;
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java b/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java
index 8fd8adc3..dfaa61e6 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java
@@ -52,6 +52,17 @@ public class JSoupFilters extends AbstractConfigurable implements JSoupFilter, J
private JSoupFilter[] filters;
+ /** loads the filters from a JSON configuration file. */
+ public JSoupFilters(Map<String, Object> stormConf, String configFile) throws IOException {
+ this.configFile = configFile;
+ this.stormConf = stormConf;
+ try {
+ loadJSONResources();
+ } catch (Exception e) {
+ throw new IOException("Unable to build JSON object from file", e);
+ }
+ }
+
private JSoupFilters() {
filters = new JSoupFilter[0];
}
@@ -80,17 +91,6 @@ public class JSoupFilters extends AbstractConfigurable implements JSoupFilter, J
return JSoupFilters.emptyParseFilter;
}
- /** loads the filters from a JSON configuration file. */
- public JSoupFilters(Map<String, Object> stormConf, String configFile) throws IOException {
- this.configFile = configFile;
- this.stormConf = stormConf;
- try {
- loadJSONResources();
- } catch (Exception e) {
- throw new IOException("Unable to build JSON object from file", e);
- }
- }
-
@Override
public void loadJSONResources(InputStream inputStream) throws IOException {
ObjectMapper mapper = new ObjectMapper();
@@ -125,7 +125,7 @@ public class JSoupFilters extends AbstractConfigurable implements JSoupFilter, J
}
}
- /** * Used for quick testing + debugging */
+ /** * Used for quick testing + debugging. */
public static void main(String[] args) throws IOException, ParseException {
Config conf = new Config();
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java
index 6852d590..45f452a9 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java
@@ -62,6 +62,17 @@ public class ParseFilters extends ParseFilter implements JSONResource {
filters = new ParseFilter[0];
}
+ /** loads the filters from a JSON configuration file. */
+ public ParseFilters(Map<String, Object> stormConf, String configFile) throws IOException {
+ this.configFile = configFile;
+ this.stormConf = stormConf;
+ try {
+ loadJSONResources();
+ } catch (Exception e) {
+ throw new IOException("Unable to build JSON object from file", e);
+ }
+ }
+
private String configFile;
private Map<String, Object> stormConf;
@@ -86,17 +97,6 @@ public class ParseFilters extends ParseFilter implements JSONResource {
return ParseFilters.emptyParseFilter;
}
- /** loads the filters from a JSON configuration file. */
- public ParseFilters(Map<String, Object> stormConf, String configFile) throws IOException {
- this.configFile = configFile;
- this.stormConf = stormConf;
- try {
- loadJSONResources();
- } catch (Exception e) {
- throw new IOException("Unable to build JSON object from file", e);
- }
- }
-
@Override
public void loadJSONResources(InputStream inputStream)
throws JsonParseException, JsonMappingException, IOException {
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java
index 7b2ca320..aa7fecc9 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java
@@ -134,6 +134,9 @@ public class XPathFilter extends ParseFilter {
// the value out of the serialized which isn't used in
// this case
continue;
+ default:
+ throw new IllegalStateException("Unexpected value: "
+ + node.getNodeType());
}
String serializedValue = out.toString();
if (serializedValue.length() > 0) {
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
index 4d347400..bb456a83 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
@@ -174,9 +174,8 @@ public class HttpRobotRulesParser extends RobotRulesParser {
}
} else {
LOG.debug(
- "Robots for {} redirected to {} (not cached for target host because not at root)",
- url,
- redir);
+ "Robots for {} redirected to {} "
+ + "(not cached for target host because not at root)", url, redir);
}
response = http.getProtocolOutput(redir.toString(), Metadata.empty);
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
index e8a14eb9..deae6983 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
@@ -628,7 +628,7 @@ public class HttpProtocol extends AbstractHttpProtocol {
}
}
- public static void main(String args[]) throws Exception {
+ public static void main(String[] args) throws Exception {
org.apache.stormcrawler.protocol.Protocol.main(new HttpProtocol(), args);
}
}
diff --git a/core/src/test/java/org/apache/stormcrawler/MetadataTest.java b/core/src/test/java/org/apache/stormcrawler/MetadataTest.java
index 00bac1a7..b0149a53 100644
--- a/core/src/test/java/org/apache/stormcrawler/MetadataTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/MetadataTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler;
import org.junit.jupiter.api.Assertions;
diff --git a/core/src/test/java/org/apache/stormcrawler/TestMetadataSerialization.java b/core/src/test/java/org/apache/stormcrawler/TestMetadataSerialization.java
index 6bb618d2..e0d587b4 100644
--- a/core/src/test/java/org/apache/stormcrawler/TestMetadataSerialization.java
+++ b/core/src/test/java/org/apache/stormcrawler/TestMetadataSerialization.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --git a/core/src/test/java/org/apache/stormcrawler/TestOutputCollector.java b/core/src/test/java/org/apache/stormcrawler/TestOutputCollector.java
index 6b8b1166..e002a209 100644
--- a/core/src/test/java/org/apache/stormcrawler/TestOutputCollector.java
+++ b/core/src/test/java/org/apache/stormcrawler/TestOutputCollector.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler;
import java.util.ArrayList;
@@ -47,10 +48,21 @@ public class TestOutputCollector implements IOutputCollector, ISpoutOutputCollec
return null;
}
+ @Override
+ public List<Integer> emit(String streamId, List<Object> tuple, Object messageId) {
+ addEmittedTuple(streamId, tuple);
+ return null;
+ }
+
@Override
public void emitDirect(
int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {}
+ @Override
+ public void emitDirect(int taskId, String streamId, List<Object> tuple, Object messageId) {
+ addEmittedTuple(streamId, tuple);
+ }
+
@Override
public void ack(Tuple input) {
acked.add(input);
@@ -77,17 +89,6 @@ public class TestOutputCollector implements IOutputCollector, ISpoutOutputCollec
return failed;
}
- @Override
- public List<Integer> emit(String streamId, List<Object> tuple, Object messageId) {
- addEmittedTuple(streamId, tuple);
- return null;
- }
-
- @Override
- public void emitDirect(int taskId, String streamId, List<Object> tuple, Object messageId) {
- addEmittedTuple(streamId, tuple);
- }
-
private void addEmittedTuple(String streamId, List<Object> tuple) {
emitted.computeIfAbsent(streamId, k -> new ArrayList<>()).add(tuple);
}
diff --git a/core/src/test/java/org/apache/stormcrawler/TestUtil.java b/core/src/test/java/org/apache/stormcrawler/TestUtil.java
index d8358489..c226cfc1 100644
--- a/core/src/test/java/org/apache/stormcrawler/TestUtil.java
+++ b/core/src/test/java/org/apache/stormcrawler/TestUtil.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler;
import static org.mockito.ArgumentMatchers.any;
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
index 2f524cc6..d7066993 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.bolt;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java
index c2862780..a7c2a954 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.bolt;
import java.io.IOException;
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/FetcherBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/FetcherBoltTest.java
index d0bcc97b..8019bcf3 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/FetcherBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/FetcherBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.bolt;
import org.junit.jupiter.api.BeforeEach;
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java
index 696bb943..cca978a5 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.bolt;
import java.io.IOException;
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/SimpleFetcherBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/SimpleFetcherBoltTest.java
index 2a4e7c4f..da212022 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/SimpleFetcherBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/SimpleFetcherBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.bolt;
import org.junit.jupiter.api.BeforeEach;
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java
index 828b19a4..aabd1d2b 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.bolt;
import static org.junit.jupiter.api.Assertions.assertThrows;
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java
index 017b60b8..6402fd06 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.filtering;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
index 65da7630..42f260f4 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.filtering;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java
index 6ae14691..b1bccd0e 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.filtering;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
index 5b183919..b0f00d84 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.filtering;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java
index a76a5c74..46489b66 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.filtering;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterFromJsonTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterFromJsonTest.java
index 85a624b9..62e925de 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterFromJsonTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterFromJsonTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.filtering;
import java.net.MalformedURLException;
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java
index 6a39568b..0d7efb41 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.filtering;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java
index 60482433..06bbfbe6 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.filtering;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
diff --git a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromAbstractAndInterface.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromAbstractAndInterface.java
index 29110361..6a3381d9 100644
--- a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromAbstractAndInterface.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromAbstractAndInterface.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.helper.initialisation;
import org.apache.stormcrawler.helper.initialisation.base.AbstractClass;
diff --git a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromAbstractClassOnly.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromAbstractClassOnly.java
index c488dece..a09802f3 100644
--- a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromAbstractClassOnly.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromAbstractClassOnly.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.helper.initialisation;
import org.apache.stormcrawler.helper.initialisation.base.AbstractClass;
diff --git a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromOpenClass.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromOpenClass.java
index 5764c755..b8674016 100644
--- a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromOpenClass.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassInheritingFromOpenClass.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.helper.initialisation;
import org.apache.stormcrawler.helper.initialisation.base.OpenClassWithAbstractClassAndInterface;
diff --git a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassWithoutValidConstructor.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassWithoutValidConstructor.java
index b49daf71..2185a0b0 100644
--- a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassWithoutValidConstructor.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/ClassWithoutValidConstructor.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.helper.initialisation;
import org.apache.stormcrawler.helper.initialisation.base.AbstractClass;
diff --git a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/FinalClassToInitialize.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/FinalClassToInitialize.java
index 4a610d06..b550735a 100644
--- a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/FinalClassToInitialize.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/FinalClassToInitialize.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.helper.initialisation;
public final class FinalClassToInitialize {}
diff --git a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/SimpleOpenClass.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/SimpleOpenClass.java
index fbb7f426..779b5397 100644
--- a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/SimpleOpenClass.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/SimpleOpenClass.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.helper.initialisation;
public class SimpleOpenClass {}
diff --git a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/AbstractClass.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/AbstractClass.java
index 9a0007c5..29098f9b 100644
--- a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/AbstractClass.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/AbstractClass.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.helper.initialisation.base;
public abstract class AbstractClass {
diff --git a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/ITestInterface.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/ITestInterface.java
index 06fee8a1..9c966712 100644
--- a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/ITestInterface.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/ITestInterface.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.helper.initialisation.base;
public interface ITestInterface {}
diff --git a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/OpenClassWithAbstractClassAndInterface.java b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/OpenClassWithAbstractClassAndInterface.java
index 27d3009e..0e7a2a08 100644
--- a/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/OpenClassWithAbstractClassAndInterface.java
+++ b/core/src/test/java/org/apache/stormcrawler/helper/initialisation/base/OpenClassWithAbstractClassAndInterface.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.helper.initialisation.base;
public class OpenClassWithAbstractClassAndInterface extends AbstractClass
diff --git a/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java b/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java
index 1defed27..cf8cde88 100644
--- a/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.indexer;
import java.util.ArrayList;
diff --git a/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java b/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java
index 55104127..55d5d292 100644
--- a/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java
+++ b/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.indexer;
import java.util.HashMap;
@@ -26,14 +27,14 @@ import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
public class DummyIndexer extends AbstractIndexerBolt {
- OutputCollector _collector;
+ OutputCollector collector;
Map<String, String> fields;
@Override
public void prepare(
Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
super.prepare(conf, context, collector);
- _collector = collector;
+ this.collector = collector;
fields = new HashMap<>();
}
@@ -45,7 +46,7 @@ public class DummyIndexer extends AbstractIndexerBolt {
// tested by the TestOutputCollector
boolean keep = filterDocument(metadata);
if (!keep) {
- _collector.ack(tuple);
+ collector.ack(tuple);
return;
}
@@ -72,11 +73,13 @@ public class DummyIndexer extends AbstractIndexerBolt {
}
}
- _collector.ack(tuple);
+ collector.ack(tuple);
}
private String trimValue(String value) {
- if (value.length() > 100) return value.length() + " chars";
+ if (value.length() > 100) {
+ return value.length() + " chars";
+ }
return value;
}
diff --git a/core/src/test/java/org/apache/stormcrawler/indexer/IndexerTester.java b/core/src/test/java/org/apache/stormcrawler/indexer/IndexerTester.java
index f84bf041..643c1da2 100644
--- a/core/src/test/java/org/apache/stormcrawler/indexer/IndexerTester.java
+++ b/core/src/test/java/org/apache/stormcrawler/indexer/IndexerTester.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.indexer;
import static org.mockito.Mockito.mock;
diff --git a/core/src/test/java/org/apache/stormcrawler/json/JsoupFilterTest.java b/core/src/test/java/org/apache/stormcrawler/json/JsoupFilterTest.java
index e5095d71..d571c9ec 100644
--- a/core/src/test/java/org/apache/stormcrawler/json/JsoupFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/json/JsoupFilterTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.json;
import java.io.IOException;
diff --git a/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java b/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java
index cfd2b8dc..10993526 100644
--- a/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.jsoup;
import java.io.IOException;
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/DuplicateLinksTest.java b/core/src/test/java/org/apache/stormcrawler/parse/DuplicateLinksTest.java
index 5cb81f85..bc7e6202 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/DuplicateLinksTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/DuplicateLinksTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.parse;
import java.io.IOException;
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/JSoupTextExtractorTest.java b/core/src/test/java/org/apache/stormcrawler/parse/JSoupTextExtractorTest.java
index 288cdc8b..eb0f0c44 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/JSoupTextExtractorTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/JSoupTextExtractorTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.parse;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/ParsingTester.java b/core/src/test/java/org/apache/stormcrawler/parse/ParsingTester.java
index b8953e50..9dcc214b 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/ParsingTester.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/ParsingTester.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.parse;
import static org.mockito.Mockito.mock;
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/StackOverflowTest.java b/core/src/test/java/org/apache/stormcrawler/parse/StackOverflowTest.java
index 1f1e9f35..2eb2f997 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/StackOverflowTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/StackOverflowTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.parse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -28,7 +29,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/**
- * @see https://github.com/apache/stormcrawler/pull/653 *
+ * @see <a href="https://github.com/apache/stormcrawler/pull/653">PR #653</a>
*/
class StackOverflowTest extends ParsingTester {
@@ -47,7 +48,7 @@ class StackOverflowTest extends ParsingTester {
}
/**
- * @see https://github.com/apache/stormcrawler/issues/666
+ * @see <a href="https://github.com/apache/stormcrawler/issues/666">Issue #666</a>
*/
@Test
void testNamespaceExtraction() throws IOException {
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/CSVMetadataFilterTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/CSVMetadataFilterTest.java
index e6f3583c..3781fe40 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/filter/CSVMetadataFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/CSVMetadataFilterTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.parse.filter;
import java.io.IOException;
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/CollectionTaggerTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/CollectionTaggerTest.java
index 1f617eda..554a5e74 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/filter/CollectionTaggerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/CollectionTaggerTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.parse.filter;
import com.fasterxml.jackson.databind.node.NullNode;
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsFilterTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsFilterTest.java
index a71e8334..29798f1f 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsFilterTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.parse.filter;
import java.io.IOException;
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsParseFilter.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsParseFilter.java
index 332ba6e6..6e926831 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsParseFilter.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsParseFilter.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.parse.filter;
import java.io.ByteArrayInputStream;
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java
index c05cc3dd..c4a525bc 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.parse.filter;
import java.io.IOException;
diff --git a/core/src/test/java/org/apache/stormcrawler/persistence/AdaptiveSchedulerTest.java b/core/src/test/java/org/apache/stormcrawler/persistence/AdaptiveSchedulerTest.java
index bbdd699a..d754d326 100644
--- a/core/src/test/java/org/apache/stormcrawler/persistence/AdaptiveSchedulerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/persistence/AdaptiveSchedulerTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.persistence;
import java.net.MalformedURLException;
diff --git a/core/src/test/java/org/apache/stormcrawler/persistence/DefaultSchedulerTest.java b/core/src/test/java/org/apache/stormcrawler/persistence/DefaultSchedulerTest.java
index 17f0470d..01d1c6f2 100644
--- a/core/src/test/java/org/apache/stormcrawler/persistence/DefaultSchedulerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/persistence/DefaultSchedulerTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.persistence;
import java.net.MalformedURLException;
diff --git a/core/src/test/java/org/apache/stormcrawler/persistence/URLBufferTest.java b/core/src/test/java/org/apache/stormcrawler/persistence/URLBufferTest.java
index 2d3b243d..e1e5af1c 100644
--- a/core/src/test/java/org/apache/stormcrawler/persistence/URLBufferTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/persistence/URLBufferTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.persistence;
import java.net.MalformedURLException;
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/AbstractProtocolTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/AbstractProtocolTest.java
index 01d3b38d..ae30400a 100644
--- a/core/src/test/java/org/apache/stormcrawler/protocol/AbstractProtocolTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/AbstractProtocolTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.protocol;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
@@ -80,7 +81,9 @@ public abstract class AbstractProtocolTest {
jakarta.servlet.http.HttpServletRequest httpServletRequest,
jakarta.servlet.http.HttpServletResponse response)
throws IOException {
- if (response.isCommitted() || baseRequest.isHandled()) return;
+ if (response.isCommitted() || baseRequest.isHandled()) {
+ return;
+ }
baseRequest.setHandled(true);
final String content = "Success!";
response.setStatus(HttpServletResponse.SC_OK);
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/DelegationProtocolTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/DelegationProtocolTest.java
index f3b586c6..3333b378 100644
--- a/core/src/test/java/org/apache/stormcrawler/protocol/DelegationProtocolTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/DelegationProtocolTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.protocol;
import java.io.FileNotFoundException;
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/DummyProtocol.java b/core/src/test/java/org/apache/stormcrawler/protocol/DummyProtocol.java
index a74664e4..4f289fec 100644
--- a/core/src/test/java/org/apache/stormcrawler/protocol/DummyProtocol.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/DummyProtocol.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.protocol;
import crawlercommons.robots.BaseRobotRules;
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java
index 3f5dfba0..d3e1eccf 100644
--- a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.protocol;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserTest.java
index f314411c..135c3df5 100644
--- a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.protocol;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
diff --git a/core/src/test/java/org/apache/stormcrawler/proxy/MultiProxyManagerTest.java b/core/src/test/java/org/apache/stormcrawler/proxy/MultiProxyManagerTest.java
index 997a9737..c1cd3287 100644
--- a/core/src/test/java/org/apache/stormcrawler/proxy/MultiProxyManagerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/proxy/MultiProxyManagerTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.proxy;
import java.io.FileWriter;
diff --git a/core/src/test/java/org/apache/stormcrawler/proxy/SCProxyTest.java b/core/src/test/java/org/apache/stormcrawler/proxy/SCProxyTest.java
index a0655f74..1cf1e3f5 100644
--- a/core/src/test/java/org/apache/stormcrawler/proxy/SCProxyTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/proxy/SCProxyTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.proxy;
import org.junit.jupiter.api.Assertions;
diff --git a/core/src/test/java/org/apache/stormcrawler/proxy/SingleProxyManagerTest.java b/core/src/test/java/org/apache/stormcrawler/proxy/SingleProxyManagerTest.java
index 3b6bcb3f..9635e518 100644
--- a/core/src/test/java/org/apache/stormcrawler/proxy/SingleProxyManagerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/proxy/SingleProxyManagerTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.proxy;
import java.util.Optional;
diff --git a/core/src/test/java/org/apache/stormcrawler/spout/FileSpoutTest.java b/core/src/test/java/org/apache/stormcrawler/spout/FileSpoutTest.java
index 0c711f03..8d6b03db 100644
--- a/core/src/test/java/org/apache/stormcrawler/spout/FileSpoutTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/spout/FileSpoutTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.spout;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --git a/core/src/test/java/org/apache/stormcrawler/spout/mocks/FileSpoutOutputCollectorMock.java b/core/src/test/java/org/apache/stormcrawler/spout/mocks/FileSpoutOutputCollectorMock.java
index 893e3c22..f09a18fd 100644
--- a/core/src/test/java/org/apache/stormcrawler/spout/mocks/FileSpoutOutputCollectorMock.java
+++ b/core/src/test/java/org/apache/stormcrawler/spout/mocks/FileSpoutOutputCollectorMock.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.spout.mocks;
import java.util.List;
diff --git a/core/src/test/java/org/apache/stormcrawler/spout/mocks/FileSpoutTopologyContextMock.java b/core/src/test/java/org/apache/stormcrawler/spout/mocks/FileSpoutTopologyContextMock.java
index acf48079..9ca138ab 100644
--- a/core/src/test/java/org/apache/stormcrawler/spout/mocks/FileSpoutTopologyContextMock.java
+++ b/core/src/test/java/org/apache/stormcrawler/spout/mocks/FileSpoutTopologyContextMock.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.spout.mocks;
import java.util.List;
diff --git a/core/src/test/java/org/apache/stormcrawler/util/ConfUtilsTest.java b/core/src/test/java/org/apache/stormcrawler/util/ConfUtilsTest.java
index f2ab4c49..3ef5c007 100644
--- a/core/src/test/java/org/apache/stormcrawler/util/ConfUtilsTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/ConfUtilsTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.util;
import java.io.IOException;
diff --git a/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java b/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java
index 1b9cca8d..0c6dfbca 100644
--- a/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.util;
import java.net.MalformedURLException;
diff --git a/core/src/test/java/org/apache/stormcrawler/util/InitialisationUtilTest.java b/core/src/test/java/org/apache/stormcrawler/util/InitialisationUtilTest.java
index 4d9ec858..29351fd7 100644
--- a/core/src/test/java/org/apache/stormcrawler/util/InitialisationUtilTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/InitialisationUtilTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.util;
import org.apache.stormcrawler.helper.initialisation.ClassInheritingFromAbstractAndInterface;
diff --git a/core/src/test/java/org/apache/stormcrawler/util/MetadataTransferTest.java b/core/src/test/java/org/apache/stormcrawler/util/MetadataTransferTest.java
index e4c34666..d28a6d94 100644
--- a/core/src/test/java/org/apache/stormcrawler/util/MetadataTransferTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/MetadataTransferTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.util;
import java.net.MalformedURLException;
diff --git a/core/src/test/java/org/apache/stormcrawler/util/RefreshTagTest.java b/core/src/test/java/org/apache/stormcrawler/util/RefreshTagTest.java
index 5c742489..f6c27f86 100644
--- a/core/src/test/java/org/apache/stormcrawler/util/RefreshTagTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/RefreshTagTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.util;
import java.io.IOException;
diff --git a/core/src/test/java/org/apache/stormcrawler/util/RobotsTagsTest.java b/core/src/test/java/org/apache/stormcrawler/util/RobotsTagsTest.java
index df361e2d..236f0131 100644
--- a/core/src/test/java/org/apache/stormcrawler/util/RobotsTagsTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/RobotsTagsTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.util;
import java.net.MalformedURLException;
diff --git a/external/ai/src/main/java/org/apache/stormcrawler/ai/AbstractLLMTextExtractor.java b/external/ai/src/main/java/org/apache/stormcrawler/ai/AbstractLLMTextExtractor.java
index 3fc360cd..c5ec1a18 100644
--- a/external/ai/src/main/java/org/apache/stormcrawler/ai/AbstractLLMTextExtractor.java
+++ b/external/ai/src/main/java/org/apache/stormcrawler/ai/AbstractLLMTextExtractor.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.ai;
import dev.langchain4j.data.message.SystemMessage;
diff --git a/external/ai/src/main/java/org/apache/stormcrawler/ai/OpenAITextExtractor.java b/external/ai/src/main/java/org/apache/stormcrawler/ai/OpenAITextExtractor.java
index 4a212910..9de21fc4 100644
--- a/external/ai/src/main/java/org/apache/stormcrawler/ai/OpenAITextExtractor.java
+++ b/external/ai/src/main/java/org/apache/stormcrawler/ai/OpenAITextExtractor.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.ai;
import dev.langchain4j.model.chat.ChatModel;
diff --git a/external/ai/src/main/java/org/apache/stormcrawler/ai/listener/LlmResponseListener.java b/external/ai/src/main/java/org/apache/stormcrawler/ai/listener/LlmResponseListener.java
index b0f0d98a..718420bc 100644
--- a/external/ai/src/main/java/org/apache/stormcrawler/ai/listener/LlmResponseListener.java
+++ b/external/ai/src/main/java/org/apache/stormcrawler/ai/listener/LlmResponseListener.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.ai.listener;
/**
diff --git a/external/ai/src/main/java/org/apache/stormcrawler/ai/listener/NoOpListener.java b/external/ai/src/main/java/org/apache/stormcrawler/ai/listener/NoOpListener.java
index 4921ead8..fd04a0d2 100644
--- a/external/ai/src/main/java/org/apache/stormcrawler/ai/listener/NoOpListener.java
+++ b/external/ai/src/main/java/org/apache/stormcrawler/ai/listener/NoOpListener.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.ai.listener;
/**
diff --git a/external/ai/src/test/java/org/apache/stormcrawler/ai/OpenAITextExtractorTest.java b/external/ai/src/test/java/org/apache/stormcrawler/ai/OpenAITextExtractorTest.java
index f65ea9e2..ea3f8368 100644
--- a/external/ai/src/test/java/org/apache/stormcrawler/ai/OpenAITextExtractorTest.java
+++ b/external/ai/src/test/java/org/apache/stormcrawler/ai/OpenAITextExtractorTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.ai;
import static org.junit.jupiter.api.Assertions.assertFalse;
diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchConstants.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchConstants.java
index b5199933..fcf23416 100644
--- a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchConstants.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchConstants.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.aws.bolt;
public interface CloudSearchConstants {
diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
index e958ce4d..2491f516 100644
--- a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.aws.bolt;
import static org.apache.stormcrawler.Constants.StatusStreamName;
@@ -256,9 +257,8 @@ public class CloudSearchIndexerBolt extends AbstractIndexerBolt {
LOG.info("Unparsable date {}", value);
continue;
}
- }
- // normalise strings
- else {
+ } else {
+ // normalise strings
value = CloudSearchUtils.stripNonCharCodepoints(value);
}
@@ -317,13 +317,14 @@ public class CloudSearchIndexerBolt extends AbstractIndexerBolt {
// can add it to the buffer without overflowing?
if (currentDocLength + 2 + currentBufferLength < MAX_SIZE_BATCH_BYTES) {
- if (numDocsInBatch != 0) buffer.append(',');
+ if (numDocsInBatch != 0) {
+ buffer.append(',');
+ }
buffer.append(currentDoc);
this.unacked.add(tuple);
numDocsInBatch++;
- }
- // flush the previous batch and create a new one with this doc
- else {
+ } else {
+ // flush the previous batch and create a new one with this doc
sendBatch();
buffer.append(currentDoc);
this.unacked.add(tuple);
diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchUtils.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchUtils.java
index e35916cc..06fb26d6 100644
--- a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchUtils.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchUtils.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.aws.bolt;
import java.nio.charset.StandardCharsets;
@@ -28,7 +29,7 @@ public class CloudSearchUtils {
private static MessageDigest digester;
private static final Pattern INVALID_XML_CHARS =
- Pattern.compile("[^\\u0009\\u000A\\u000D\\u0020-\\uD7FF\\uE000-\\uFFFD]");
+ Pattern.compile("[^\\t\\n\\r -\\uD7FF\\uE000-\\uFFFD]");
static {
try {
@@ -76,9 +77,12 @@ public class CloudSearchUtils {
public static String cleanFieldName(String name) {
String lowercase = name.toLowerCase(Locale.ROOT);
lowercase = lowercase.replaceAll("[^a-z_0-9]", "_");
- if (lowercase.length() < 3 || lowercase.length() > 64)
+ if (lowercase.length() < 3 || lowercase.length() > 64) {
throw new RuntimeException("Field name must be between 3 and 64 chars : " + lowercase);
- if (lowercase.equals("score")) throw new RuntimeException("Field name must be score");
+ }
+ if (lowercase.equals("score")) {
+ throw new RuntimeException("Field name must be score");
+ }
return lowercase;
}
}
diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/AbstractS3CacheBolt.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/AbstractS3CacheBolt.java
index c449aade..94ab205e 100644
--- a/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/AbstractS3CacheBolt.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/AbstractS3CacheBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.aws.s3;
import com.amazonaws.ClientConfiguration;
diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3CacheChecker.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3CacheChecker.java
index 422933d7..52bc447f 100644
--- a/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3CacheChecker.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3CacheChecker.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.aws.s3;
import com.amazonaws.services.s3.model.AmazonS3Exception;
diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3Cacher.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3Cacher.java
index 253c264c..d8b0a271 100644
--- a/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3Cacher.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3Cacher.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.aws.s3;
import com.amazonaws.services.s3.model.AmazonS3Exception;
diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3ContentCacher.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3ContentCacher.java
index 1d5dffb6..c47250af 100644
--- a/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3ContentCacher.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/s3/S3ContentCacher.java
@@ -14,11 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.aws.s3;
import org.apache.stormcrawler.Metadata;
-/** Caches byte[] content into S3 */
+/** Caches byte[] content into S3. */
public class S3ContentCacher extends S3Cacher {
@Override
diff --git a/external/langid/src/main/java/org/apache/stormcrawler/parse/filter/LanguageID.java b/external/langid/src/main/java/org/apache/stormcrawler/parse/filter/LanguageID.java
index 377a0723..6d43a5a5 100644
--- a/external/langid/src/main/java/org/apache/stormcrawler/parse/filter/LanguageID.java
+++ b/external/langid/src/main/java/org/apache/stormcrawler/parse/filter/LanguageID.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.parse.filter;
import com.fasterxml.jackson.databind.JsonNode;
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
index cfceefb0..e4eec09e 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch;
import java.io.IOException;
@@ -39,13 +40,21 @@ public final class BulkItemResponseToFailedFlag {
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof BulkItemResponseToFailedFlag)) return false;
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof BulkItemResponseToFailedFlag)) {
+ return false;
+ }
BulkItemResponseToFailedFlag that = (BulkItemResponseToFailedFlag) o;
- if (failed != that.failed) return false;
- if (!response.equals(that.response)) return false;
+ if (failed != that.failed) {
+ return false;
+ }
+ if (!response.equals(that.response)) {
+ return false;
+ }
return id.equals(that.id);
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/Constants.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/Constants.java
index 38938732..8c0cbc98 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/Constants.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/Constants.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch;
public interface Constants {
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
index 4251ac3b..180a1074 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch;
import com.google.common.base.Charsets;
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
index 82b0f081..d1a1dbde 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch;
import static org.opensearch.client.RestClientBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS;
@@ -77,10 +78,6 @@ public final class OpenSearchConnection {
return client;
}
- public void addToProcessor(final DocWriteRequest<?> request) {
- processor.add(request);
- }
-
public static RestHighLevelClient getClient(Map<String, Object> stormConf, String boltType) {
final String dottedType = boltType + ".";
@@ -194,9 +191,8 @@ public final class OpenSearchConnection {
requestConfigBuilder ->
requestConfigBuilder
.setConnectTimeout(connectTimeout)
- .setSocketTimeout(socketTimeout) // Timeout when waiting
- // for data
- );
+ .setSocketTimeout(socketTimeout) // Timeout when waiting for data
+ );
// TODO check if this has gone somewhere else
// int maxRetryTimeout = ConfUtils.getInt(stormConf, Constants.PARAMPREFIX +
@@ -234,6 +230,10 @@ public final class OpenSearchConnection {
return new RestHighLevelClient(builder);
}
+ public void addToProcessor(final DocWriteRequest<?> request) {
+ processor.add(request);
+ }
+
/**
* Creates a connection with a default listener. The values for bolt type are
* [indexer,status,metrics]
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
index 2fee97fe..c67b9095 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.bolt;
import com.github.benmanes.caffeine.cache.Cache;
@@ -107,7 +108,9 @@ public class DeletionBolt extends BaseRichBolt
public void onRemoval(
@Nullable String key, @Nullable List<Tuple> value, @NotNull RemovalCause cause) {
- if (!cause.wasEvicted()) return;
+ if (!cause.wasEvicted()) {
+ return;
+ }
if (value != null) {
LOG.error("Purged from waitAck {} with {} values", key, value.size());
for (Tuple t : value) {
@@ -121,7 +124,9 @@ public class DeletionBolt extends BaseRichBolt
@Override
public void cleanup() {
- if (connection != null) connection.close();
+ if (connection != null) {
+ connection.close();
+ }
}
@Override
@@ -233,8 +238,11 @@ public class DeletionBolt extends BaseRichBolt
if (tmp == null) {
tmp = buwff;
}
- if (buwff.failed) ctFailed++;
- else tmp = buwff;
+ if (buwff.failed) {
+ ctFailed++;
+ } else {
+ tmp = buwff;
+ }
}
if (ctFailed != bulkItemsWithFailedFlag.size()) {
LOG.warn(
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
index 7619afc7..04de31ca 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.bolt;
import static org.apache.stormcrawler.Constants.StatusStreamName;
@@ -155,7 +156,9 @@ public class IndexerBolt extends AbstractIndexerBolt
public void onRemoval(
@Nullable String key, @Nullable List<Tuple> value, @NotNull RemovalCause cause) {
- if (!cause.wasEvicted()) return;
+ if (!cause.wasEvicted()) {
+ return;
+ }
if (value != null) {
LOG.error("Purged from waitAck {} with {} values", key, value.size());
for (Tuple t : value) {
@@ -169,7 +172,9 @@ public class IndexerBolt extends AbstractIndexerBolt
@Override
public void cleanup() {
- if (connection != null) connection.close();
+ if (connection != null) {
+ connection.close();
+ }
}
@Override
@@ -349,8 +354,11 @@ public class IndexerBolt extends AbstractIndexerBolt
if (tmp == null) {
tmp = buwff;
}
- if (buwff.failed) ctFailed++;
- else tmp = buwff;
+ if (buwff.failed) {
+ ctFailed++;
+ } else {
+ tmp = buwff;
+ }
}
if (ctFailed != bulkItemsWithFailedFlag.size()) {
LOG.warn(
@@ -402,9 +410,8 @@ public class IndexerBolt extends AbstractIndexerBolt
t,
new Values(url, metadata, Status.ERROR));
_collector.ack(t);
- }
- // otherwise just fail it
- else {
+ } else {
+ // otherwise just fail it
_collector.fail(t);
}
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
index 693b00df..900223fa 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.filtering;
import com.fasterxml.jackson.databind.JsonNode;
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
index 2675e750..6b9ccf4c 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.metrics;
import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -95,7 +96,9 @@ public class MetricsConsumer implements IMetricsConsumer {
@Override
public void cleanup() {
- if (connection != null) connection.close();
+ if (connection != null) {
+ connection.close();
+ }
}
@Override
@@ -130,7 +133,9 @@ public class MetricsConsumer implements IMetricsConsumer {
* @return elastic index name
*/
private String getIndexName(Date timestamp) {
- if (dateFormat == null) return indexName;
+ if (dateFormat == null) {
+ return indexName;
+ }
StringBuilder sb = new StringBuilder(indexName);
sb.append("-").append(dateFormat.format(timestamp));
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
index 6d5729d6..56edf696 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.metrics;
import java.util.HashMap;
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
index bffb43c1..e475afb2 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.parse.filter;
import com.fasterxml.jackson.databind.JsonNode;
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
index 37bcd096..43b0e428 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.persistence;
import java.io.IOException;
@@ -201,9 +202,8 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
// single value
if (mdValObj instanceof String) {
metadata.addValue(key, (String) mdValObj);
- }
- // multi valued
- else {
+ } else {
+ // multi valued
metadata.addValues(key, (List<String>) mdValObj);
}
}
@@ -225,11 +225,12 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
@Override
public void close() {
- if (client != null)
+ if (client != null) {
try {
client.close();
} catch (IOException e) {
LOG.error("Exception caught when closing client", e);
}
+ }
}
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
index 810a5dd8..2eb97102 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.persistence;
import static org.opensearch.index.query.QueryBuilders.boolQuery;
@@ -276,7 +277,9 @@ public class AggregationSpout extends AbstractSpout implements ActionListener<Se
sortValuesForKey(key, lastHit.getSortValues());
}
- if (hitsForThisBucket > 0) numBuckets++;
+ if (hitsForThisBucket > 0) {
+ numBuckets++;
+ }
numhits += hitsForThisBucket;
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
index 99b62aa7..551153f5 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.persistence;
import static org.opensearch.index.query.QueryBuilders.boolQuery;
@@ -144,8 +145,8 @@ public class HybridSpout extends AggregationSpout implements EmptyQueueListener
client.searchAsync(request, RequestOptions.DEFAULT, hrl);
}
+ /** Overrides the handling of responses for aggregations. */
@Override
- /** Overrides the handling of responses for aggregations */
public void onResponse(SearchResponse response) {
// delete all entries from the searchAfterCache when
// we get the results from the aggregation spouts
@@ -153,13 +154,15 @@ public class HybridSpout extends AggregationSpout implements EmptyQueueListener
super.onResponse(response);
}
+ /** The aggregation kindly told us where to start from. */
@Override
- /** The aggregation kindly told us where to start from * */
protected void sortValuesForKey(String key, Object[] sortValues) {
- if (sortValues != null && sortValues.length > 0) this.searchAfterCache.put(key, sortValues);
+ if (sortValues != null && sortValues.length > 0) {
+ this.searchAfterCache.put(key, sortValues);
+ }
}
- /** Handling of results for a specific queue * */
+ /** Handling of results for a specific queue. */
class HostResultListener implements ActionListener<SearchResponse> {
@Override
@@ -185,8 +188,9 @@ public class HybridSpout extends AggregationSpout implements EmptyQueueListener
}
Object key_as_object = sourceAsMap.get(pfield);
if (key_as_object instanceof List) {
- if (((List<String>) (key_as_object)).size() == 1)
+ if (((List<String>) (key_as_object)).size() == 1) {
key = ((List<String>) key_as_object).get(0);
+ }
} else {
key = key_as_object.toString();
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
index 85ce0c58..bd178f7d 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.persistence;
import com.github.benmanes.caffeine.cache.Cache;
@@ -307,7 +308,9 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt
@Override
public void onRemoval(
@Nullable String key, @Nullable List<Tuple> value, @NotNull RemovalCause cause) {
- if (!cause.wasEvicted()) return;
+ if (!cause.wasEvicted()) {
+ return;
+ }
LOG.error("Purged from waitAck {} with {} values", key, value.size());
for (Tuple t : value) {
eventCounter.scope("purged").incrBy(1);
@@ -385,8 +388,11 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt
if (tmp == null) {
tmp = buwff;
}
- if (buwff.failed) ctFailed++;
- else tmp = buwff;
+ if (buwff.failed) {
+ ctFailed++;
+ } else {
+ tmp = buwff;
+ }
}
if (ctFailed != bulkItemsWithFailedFlag.size()) {
LOG.warn(
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
index 54b9f82c..e9c72b33 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.bolt;
import org.junit.jupiter.api.AfterEach;
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java
index cf6203e1..2cc863fa 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.bolt;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
index b95b5838..f1fddb83 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.opensearch.bolt;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
index 55544167..ef1f2148 100644
--- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
+++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
@@ -280,7 +280,9 @@ public class HttpProtocol extends AbstractHttpProtocol {
/** Returns a proxy object if required * */
private Proxy getProxy(String proxyserver, String proxyuser, String proxypwd) {
- if (proxyserver == null) return null;
+ if (proxyserver == null) {
+ return null;
+ }
Proxy proxy = new Proxy(proxyserver);
if (proxyuser != null) {
diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/LocalResourceHandler.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/LocalResourceHandler.java
index 7c3cf8d9..ed37691e 100644
--- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/LocalResourceHandler.java
+++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/LocalResourceHandler.java
@@ -36,10 +36,14 @@ public class LocalResourceHandler extends AbstractHandler {
jakarta.servlet.http.HttpServletRequest httpServletRequest,
jakarta.servlet.http.HttpServletResponse response)
throws IOException {
- if (response.isCommitted() || baseRequest.isHandled()) return;
+ if (response.isCommitted() || baseRequest.isHandled()) {
+ return;
+ }
// does the URL have a file element?
- if (s.length() < 2) return;
+ if (s.length() < 2) {
+ return;
+ }
baseRequest.setHandled(true);
diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ProtocolTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ProtocolTest.java
index a40f0e27..f3800ea1 100644
--- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ProtocolTest.java
+++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ProtocolTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.protocol.playwright;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
diff --git a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilter.java b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilter.java
index bd68ab2c..eeafc00e 100644
--- a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilter.java
+++ b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilter.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.protocol.selenium;
import org.apache.stormcrawler.Metadata;
diff --git a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilters.java b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilters.java
index dab1fc26..30abd482 100644
--- a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilters.java
+++ b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/NavigationFilters.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.protocol.selenium;
import com.fasterxml.jackson.databind.JsonNode;
@@ -33,7 +34,7 @@ import org.openqa.selenium.remote.RemoteWebDriver;
import org.slf4j.LoggerFactory;
/**
- * Wrapper for the NavigationFilter defined in a JSON configuration
+ * Wrapper for the NavigationFilter defined in a JSON configuration.
*
* @see org.apache.stormcrawler.util.Configurable#createConfiguredInstance(Class, Class, Map,
* JsonNode) for more information.
@@ -50,11 +51,33 @@ public class NavigationFilters extends NavigationFilter {
filters = new NavigationFilter[0];
}
+ /**
+ * loads the filters from a JSON configuration file
+ *
+ * @throws IOException
+ */
+ public NavigationFilters(@NotNull Map<String, Object> stormConf, @NotNull String configFile)
+ throws IOException {
+ // load the JSON configFile
+ // build a JSON object out of it
+ JsonNode confNode;
+ try (InputStream confStream = getClass().getClassLoader().getResourceAsStream(configFile)) {
+ ObjectMapper mapper = new ObjectMapper();
+ confNode = mapper.readValue(confStream, JsonNode.class);
+ } catch (Exception e) {
+ throw new IOException("Unable to build JSON object from file", e);
+ }
+
+ configure(stormConf, confNode);
+ }
+
public @Nullable ProtocolResponse filter(
@NotNull RemoteWebDriver driver, @NotNull Metadata metadata) {
for (NavigationFilter filter : filters) {
ProtocolResponse response = filter.filter(driver, metadata);
- if (response != null) return response;
+ if (response != null) {
+ return response;
+ }
}
return null;
}
@@ -79,26 +102,6 @@ public class NavigationFilters extends NavigationFilter {
return NavigationFilters.emptyNavigationFilters;
}
- /**
- * loads the filters from a JSON configuration file
- *
- * @throws IOException
- */
- public NavigationFilters(@NotNull Map<String, Object> stormConf, @NotNull String configFile)
- throws IOException {
- // load the JSON configFile
- // build a JSON object out of it
- JsonNode confNode;
- try (InputStream confStream = getClass().getClassLoader().getResourceAsStream(configFile)) {
- ObjectMapper mapper = new ObjectMapper();
- confNode = mapper.readValue(confStream, JsonNode.class);
- } catch (Exception e) {
- throw new IOException("Unable to build JSON object from file", e);
- }
-
- configure(stormConf, confNode);
- }
-
@Override
public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filtersConf) {
List<NavigationFilter> filterLists =
diff --git a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
index 219f4b0a..109dbbb1 100644
--- a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
+++ b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.protocol.selenium;
import java.net.URL;
@@ -36,7 +37,9 @@ import org.openqa.selenium.remote.RemoteWebDriver;
public class RemoteDriverProtocol extends SeleniumProtocol {
private void substituteUserAgent(Map<String, Object> keyvals, final String userAgentString) {
- if (keyvals == null) return;
+ if (keyvals == null) {
+ return;
+ }
for (Entry<String, Object> entry : keyvals.entrySet()) {
Object val = entry.getValue();
diff --git a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java
index 6f9b1993..f4830054 100644
--- a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java
+++ b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.protocol.selenium;
import java.nio.charset.StandardCharsets;
diff --git a/external/selenium/src/test/java/org/apache/stormcrawler/protocol/selenium/ProtocolTest.java b/external/selenium/src/test/java/org/apache/stormcrawler/protocol/selenium/ProtocolTest.java
index a3794f5b..5e03e658 100644
--- a/external/selenium/src/test/java/org/apache/stormcrawler/protocol/selenium/ProtocolTest.java
+++ b/external/selenium/src/test/java/org/apache/stormcrawler/protocol/selenium/ProtocolTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.protocol.selenium;
import static org.awaitility.Awaitility.await;
diff --git a/external/solr/src/main/java/org/apache/stormcrawler/solr/Constants.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/Constants.java
index 57d58b54..43ee9baa 100644
--- a/external/solr/src/main/java/org/apache/stormcrawler/solr/Constants.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/Constants.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr;
public interface Constants {
diff --git a/external/solr/src/main/java/org/apache/stormcrawler/solr/SolrConnection.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/SolrConnection.java
index 1cf317eb..0219a198 100644
--- a/external/solr/src/main/java/org/apache/stormcrawler/solr/SolrConnection.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/SolrConnection.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr;
import java.io.IOException;
@@ -102,7 +103,9 @@ public class SolrConnection {
private void flushAllUpdates(boolean force) {
synchronized (lock) {
- if (!force && System.currentTimeMillis() - lastUpdate < noUpdateThreshold) return;
+ if (!force && System.currentTimeMillis() - lastUpdate < noUpdateThreshold) {
+ return;
+ }
CloudHttp2SolrClient cloudHttp2SolrClient = (CloudHttp2SolrClient) client;
DocCollection col = cloudHttp2SolrClient.getClusterState().getCollection(collection);
@@ -110,7 +113,9 @@ public class SolrConnection {
// Flush all slices
for (var entry : updateQueues.entrySet()) {
List<Update> waitingUpdates = entry.getValue();
- if (waitingUpdates.isEmpty()) continue;
+ if (waitingUpdates.isEmpty()) {
+ continue;
+ }
Slice slice = col.getSlice(entry.getKey());
Replica leader = slice.getLeader();
diff --git a/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/DeletionBolt.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/DeletionBolt.java
index eace96cb..de733e33 100644
--- a/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/DeletionBolt.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/DeletionBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr.bolt;
import java.io.IOException;
@@ -32,7 +33,7 @@ import org.slf4j.LoggerFactory;
* Deletion bolt for Apache SOLR.
* Must be connected to the 'deletion' stream of the status updater bolt.
* <pre>
- * builder.setBolt("deleter", new DeletionBolt()).localOrShuffleGrouping("status", Constants.DELETION_STREAM_NAME);
+ * builder.setBolt("deleter", new DeletionBolt()).localOrShuffleGrouping("status", Constants.DELETION_STREAM_NAME);
* </pre>
**/
@SuppressWarnings("serial")
@@ -60,12 +61,13 @@ public class DeletionBolt extends BaseRichBolt {
@Override
public void cleanup() {
- if (connection != null)
+ if (connection != null) {
try {
connection.close();
} catch (IOException | SolrServerException e) {
LOG.warn("Failed to close connection", e);
}
+ }
}
@Override
diff --git a/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/IndexerBolt.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/IndexerBolt.java
index c2aef39b..2b10dc83 100644
--- a/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/IndexerBolt.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/IndexerBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr.bolt;
import static org.apache.stormcrawler.Constants.StatusStreamName;
@@ -65,12 +66,13 @@ public class IndexerBolt extends AbstractIndexerBolt {
@Override
public void cleanup() {
- if (connection != null)
+ if (connection != null) {
try {
connection.close();
} catch (Exception e) {
LOG.error("Can't close connection to Solr: {}", e);
}
+ }
}
@Override
diff --git a/external/solr/src/main/java/org/apache/stormcrawler/solr/metrics/MetricsConsumer.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/metrics/MetricsConsumer.java
index 916ebdd8..561e32b9 100644
--- a/external/solr/src/main/java/org/apache/stormcrawler/solr/metrics/MetricsConsumer.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/metrics/MetricsConsumer.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr.metrics;
import java.text.DateFormat;
diff --git a/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/SolrSpout.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/SolrSpout.java
index 117a763d..10b437c5 100644
--- a/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/SolrSpout.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/SolrSpout.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr.persistence;
import java.time.Instant;
@@ -136,10 +137,9 @@ public class SolrSpout extends AbstractQueryingSpout {
lastNextFetchDate = Instant.now();
lastStartOffset = 0;
lastTimeResetToNow = Instant.now();
- }
- // reset the value for next fetch date if the previous one is too
- // old
- else if (resetFetchDateAfterNSecs != -1) {
+ } else if (resetFetchDateAfterNSecs != -1) {
+ // reset the value for next fetch date if the previous one is too
+ // old
Instant changeNeededOn =
Instant.ofEpochMilli(
lastTimeResetToNow.toEpochMilli() + (resetFetchDateAfterNSecs * 1000));
diff --git a/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/StatusUpdaterBolt.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/StatusUpdaterBolt.java
index a80d2e84..e3df9bac 100644
--- a/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/StatusUpdaterBolt.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/StatusUpdaterBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr.persistence;
import java.util.Date;
diff --git a/external/solr/src/test/java/org/apache/stormcrawler/solr/CloudConnectionTest.java b/external/solr/src/test/java/org/apache/stormcrawler/solr/CloudConnectionTest.java
index a2037b16..743b7972 100644
--- a/external/solr/src/test/java/org/apache/stormcrawler/solr/CloudConnectionTest.java
+++ b/external/solr/src/test/java/org/apache/stormcrawler/solr/CloudConnectionTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --git a/external/solr/src/test/java/org/apache/stormcrawler/solr/SolrCloudContainerTest.java b/external/solr/src/test/java/org/apache/stormcrawler/solr/SolrCloudContainerTest.java
index fade61fc..7c07a11c 100644
--- a/external/solr/src/test/java/org/apache/stormcrawler/solr/SolrCloudContainerTest.java
+++ b/external/solr/src/test/java/org/apache/stormcrawler/solr/SolrCloudContainerTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr;
import java.io.File;
diff --git a/external/solr/src/test/java/org/apache/stormcrawler/solr/SolrContainerTest.java b/external/solr/src/test/java/org/apache/stormcrawler/solr/SolrContainerTest.java
index fe3b0942..12690775 100644
--- a/external/solr/src/test/java/org/apache/stormcrawler/solr/SolrContainerTest.java
+++ b/external/solr/src/test/java/org/apache/stormcrawler/solr/SolrContainerTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr;
import java.io.File;
diff --git a/external/solr/src/test/java/org/apache/stormcrawler/solr/bolt/IndexerBoltTest.java b/external/solr/src/test/java/org/apache/stormcrawler/solr/bolt/IndexerBoltTest.java
index db3d5415..dd896dbb 100644
--- a/external/solr/src/test/java/org/apache/stormcrawler/solr/bolt/IndexerBoltTest.java
+++ b/external/solr/src/test/java/org/apache/stormcrawler/solr/bolt/IndexerBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr.bolt;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --git a/external/solr/src/test/java/org/apache/stormcrawler/solr/persistence/SpoutTest.java b/external/solr/src/test/java/org/apache/stormcrawler/solr/persistence/SpoutTest.java
index 3346e64f..ecdf1f66 100644
--- a/external/solr/src/test/java/org/apache/stormcrawler/solr/persistence/SpoutTest.java
+++ b/external/solr/src/test/java/org/apache/stormcrawler/solr/persistence/SpoutTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr.persistence;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -22,7 +23,10 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
-import java.util.*;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
diff --git a/external/solr/src/test/java/org/apache/stormcrawler/solr/persistence/StatusBoltTest.java b/external/solr/src/test/java/org/apache/stormcrawler/solr/persistence/StatusBoltTest.java
index f659622b..5570043b 100644
--- a/external/solr/src/test/java/org/apache/stormcrawler/solr/persistence/StatusBoltTest.java
+++ b/external/solr/src/test/java/org/apache/stormcrawler/solr/persistence/StatusBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.solr.persistence;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --git a/external/sql/src/main/java/org/apache/stormcrawler/sql/Constants.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/Constants.java
index 5fc709b0..c82fb31c 100644
--- a/external/sql/src/main/java/org/apache/stormcrawler/sql/Constants.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/Constants.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.sql;
public class Constants {
diff --git a/external/sql/src/main/java/org/apache/stormcrawler/sql/IndexerBolt.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/IndexerBolt.java
index d9b0db0c..2ae45ae7 100644
--- a/external/sql/src/main/java/org/apache/stormcrawler/sql/IndexerBolt.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/IndexerBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.sql;
import static org.apache.stormcrawler.Constants.StatusStreamName;
diff --git a/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLSpout.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLSpout.java
index 9f2f0638..5456a970 100644
--- a/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLSpout.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLSpout.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.sql;
import java.nio.ByteBuffer;
@@ -45,30 +46,30 @@ public class SQLSpout extends AbstractQueryingSpout {
private static final Scheme SCHEME = new StringTabScheme();
private static final String BASE_SQL =
- """
- SELECT *
- FROM (
- SELECT
- rank() OVER (PARTITION BY host ORDER BY nextfetchdate DESC, url) AS ranking,
- url,
- metadata,
- nextfetchdate
- FROM %s
- WHERE nextfetchdate <= ? %s
- ) AS urls_ranks
- WHERE urls_ranks.ranking <= ?
- ORDER BY ranking %s
- """;
+ """
+ SELECT *
+ FROM (
+ SELECT
+ rank() OVER (PARTITION BY host ORDER BY nextfetchdate DESC, url) AS ranking,
+ url,
+ metadata,
+ nextfetchdate
+ FROM %s
+ WHERE nextfetchdate <= ? %s
+ ) AS urls_ranks
+ WHERE urls_ranks.ranking <= ?
+ ORDER BY ranking %s
+ """;
private static final String BUCKET_CLAUSE =
- """
- AND bucket = ?
- """;
+ """
+ AND bucket = ?
+ """;
private static final String LIMIT_CLAUSE =
- """
- LIMIT ?
- """;
+ """
+ LIMIT ?
+ """;
private static final String URL_COLUMN = "url";
private static final String METADATA_COLUMN = "metadata";
diff --git a/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLUtil.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLUtil.java
index c6c197a6..f759b8d0 100644
--- a/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLUtil.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLUtil.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.sql;
import java.sql.Connection;
diff --git a/external/sql/src/main/java/org/apache/stormcrawler/sql/StatusUpdaterBolt.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/StatusUpdaterBolt.java
index 27dc2719..7ebf2899 100644
--- a/external/sql/src/main/java/org/apache/stormcrawler/sql/StatusUpdaterBolt.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/StatusUpdaterBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.sql;
import static org.apache.stormcrawler.sql.SQLUtil.closeResource;
@@ -108,22 +109,18 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt {
""";
final String updateQuery =
- String.format(
- Locale.ROOT,
- """
- REPLACE INTO %s %s
- """,
- tableName,
- baseColumns);
+ String.format(
+ Locale.ROOT,
+ "REPLACE INTO %s %s",
+ tableName,
+ baseColumns);
final String insertQuery =
- String.format(
- Locale.ROOT,
- """
- INSERT IGNORE INTO %s %s
- """,
- tableName,
- baseColumns);
+ String.format(
+ Locale.ROOT,
+ "INSERT IGNORE INTO %s %s",
+ tableName,
+ baseColumns);
try {
updatePreparedStmt = connection.prepareStatement(updateQuery);
diff --git a/external/sql/src/main/java/org/apache/stormcrawler/sql/metrics/MetricsConsumer.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/metrics/MetricsConsumer.java
index 0062f154..c9554e8a 100644
--- a/external/sql/src/main/java/org/apache/stormcrawler/sql/metrics/MetricsConsumer.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/metrics/MetricsConsumer.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.sql.metrics;
import java.sql.Connection;
diff --git a/external/sql/src/test/java/org/apache/stormcrawler/sql/AbstractSQLTest.java b/external/sql/src/test/java/org/apache/stormcrawler/sql/AbstractSQLTest.java
index bf454cf1..b1670430 100644
--- a/external/sql/src/test/java/org/apache/stormcrawler/sql/AbstractSQLTest.java
+++ b/external/sql/src/test/java/org/apache/stormcrawler/sql/AbstractSQLTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.sql;
import java.sql.Connection;
diff --git a/external/sql/src/test/java/org/apache/stormcrawler/sql/IndexerBoltTest.java b/external/sql/src/test/java/org/apache/stormcrawler/sql/IndexerBoltTest.java
index 5a34a6b7..b84029ec 100644
--- a/external/sql/src/test/java/org/apache/stormcrawler/sql/IndexerBoltTest.java
+++ b/external/sql/src/test/java/org/apache/stormcrawler/sql/IndexerBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.sql;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --git a/external/sql/src/test/java/org/apache/stormcrawler/sql/SQLSpoutTest.java b/external/sql/src/test/java/org/apache/stormcrawler/sql/SQLSpoutTest.java
index 9b65f2e2..a91fbcac 100644
--- a/external/sql/src/test/java/org/apache/stormcrawler/sql/SQLSpoutTest.java
+++ b/external/sql/src/test/java/org/apache/stormcrawler/sql/SQLSpoutTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.sql;
import static org.apache.stormcrawler.TestUtil.getMockedTopologyContextWithBucket;
@@ -173,10 +174,10 @@ class SQLSpoutTest extends AbstractSQLTest {
private void insertTestURL(String url, int bucket, String host, Instant time) throws Exception {
String sql =
- """
- INSERT INTO urls (url, status, nextfetchdate, metadata, bucket, host)
- VALUES (?, ?, ?, ?, ?, ?)
- """;
+ """
+ INSERT INTO urls (url, status, nextfetchdate, metadata, bucket, host)
+ VALUES (?, ?, ?, ?, ?, ?)
+ """;
try (PreparedStatement ps = testConnection.prepareStatement(sql)) {
ps.setString(1, url);
diff --git a/external/sql/src/test/java/org/apache/stormcrawler/sql/StatusUpdaterBoltTest.java b/external/sql/src/test/java/org/apache/stormcrawler/sql/StatusUpdaterBoltTest.java
index 4108150f..62855fee 100644
--- a/external/sql/src/test/java/org/apache/stormcrawler/sql/StatusUpdaterBoltTest.java
+++ b/external/sql/src/test/java/org/apache/stormcrawler/sql/StatusUpdaterBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.sql;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/DOMBuilder.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/DOMBuilder.java
index f7e48d36..a5c62588 100644
--- a/external/tika/src/main/java/org/apache/stormcrawler/tika/DOMBuilder.java
+++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/DOMBuilder.java
@@ -344,7 +344,7 @@ public class DOMBuilder implements ContentHandler, LexicalHandler {
* @see org.xml.sax.Locator
*/
@Override
- public void characters(char ch[], int start, int length) throws org.xml.sax.SAXException {
+ public void characters(char[] ch, int start, int length) throws org.xml.sax.SAXException {
if (isOutsideDocElem() && XMLCharacterRecognizer.isWhiteSpace(ch, start, length)) {
return; // avoid DOM006 Hierarchy request error
}
@@ -375,7 +375,7 @@ public class DOMBuilder implements ContentHandler, LexicalHandler {
* @param start Index to start of characters in the array
* @param length Number of characters in the array
*/
- public void charactersRaw(char ch[], int start, int length) throws org.xml.sax.SAXException {
+ public void charactersRaw(char[] ch, int start, int length) throws org.xml.sax.SAXException {
if (isOutsideDocElem() && XMLCharacterRecognizer.isWhiteSpace(ch, start, length)) {
return; // avoid DOM006 Hierarchy request error
}
@@ -443,7 +443,7 @@ public class DOMBuilder implements ContentHandler, LexicalHandler {
* @see #characters
*/
@Override
- public void ignorableWhitespace(char ch[], int start, int length)
+ public void ignorableWhitespace(char[] ch, int start, int length)
throws org.xml.sax.SAXException {
if (isOutsideDocElem()) {
return; // avoid DOM006 Hierarchy request error
@@ -493,7 +493,7 @@ public class DOMBuilder implements ContentHandler, LexicalHandler {
* @param length The number of characters to use from the array.
*/
@Override
- public void comment(char ch[], int start, int length) throws org.xml.sax.SAXException {
+ public void comment(char[] ch, int start, int length) throws org.xml.sax.SAXException {
// tagsoup sometimes submits invalid values here
if (ch == null || start < 0 || length >= (ch.length - start) || length < 0) {
return;
@@ -544,7 +544,7 @@ public class DOMBuilder implements ContentHandler, LexicalHandler {
* @see #ignorableWhitespace
* @see org.xml.sax.Locator
*/
- public void cdata(char ch[], int start, int length) {
+ public void cdata(char[] ch, int start, int length) {
if (isOutsideDocElem() && XMLCharacterRecognizer.isWhiteSpace(ch, start, length)) {
return; // avoid DOM006 Hierarchy request error
}
diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
index 09dea259..d7a40526 100644
--- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
+++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.tika;
import static org.apache.stormcrawler.Constants.StatusStreamName;
@@ -70,7 +71,7 @@ import org.slf4j.LoggerFactory;
import org.w3c.dom.DocumentFragment;
import org.xml.sax.ContentHandler;
-/** Uses Tika to parse the output of a fetch and extract text + metadata */
+/** Uses Tika to parse the output of a fetch and extract text + metadata. */
public class ParserBolt extends BaseRichBolt {
private Tika tika;
diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/RedirectionBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/RedirectionBolt.java
index 2e6779d6..96914680 100644
--- a/external/tika/src/main/java/org/apache/stormcrawler/tika/RedirectionBolt.java
+++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/RedirectionBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.tika;
import java.util.Map;
diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/XMLCharacterRecognizer.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/XMLCharacterRecognizer.java
index 2292f8c9..da079010 100644
--- a/external/tika/src/main/java/org/apache/stormcrawler/tika/XMLCharacterRecognizer.java
+++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/XMLCharacterRecognizer.java
@@ -52,12 +52,14 @@ class XMLCharacterRecognizer {
* @param length Number of characters in the array
* @return True if the characters in the array are XML whitespace; otherwise, false.
*/
- static boolean isWhiteSpace(char ch[], int start, int length) {
+ static boolean isWhiteSpace(char[] ch, int start, int length) {
int end = start + length;
for (int s = start; s < end; s++) {
- if (!isWhiteSpace(ch[s])) return false;
+ if (!isWhiteSpace(ch[s])) {
+ return false;
+ }
}
return true;
@@ -74,7 +76,9 @@ class XMLCharacterRecognizer {
int n = buf.length();
for (int i = 0; i < n; i++) {
- if (!isWhiteSpace(buf.charAt(i))) return false;
+ if (!isWhiteSpace(buf.charAt(i))) {
+ return false;
+ }
}
return true;
@@ -92,7 +96,9 @@ class XMLCharacterRecognizer {
int n = s.length();
for (int i = 0; i < n; i++) {
- if (!isWhiteSpace(s.charAt(i))) return false;
+ if (!isWhiteSpace(s.charAt(i))) {
+ return false;
+ }
}
}
diff --git a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java
index 5b2bf3cc..fc4e5e33 100644
--- a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java
+++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.tika;
import static org.mockito.Mockito.mock;
diff --git a/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Constants.java b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Constants.java
index 241e45c6..d747aafe 100644
--- a/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Constants.java
+++ b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Constants.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.urlfrontier;
public final class Constants {
diff --git a/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/ManagedChannelUtil.java b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/ManagedChannelUtil.java
index 8ca34747..35e12736 100644
--- a/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/ManagedChannelUtil.java
+++ b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/ManagedChannelUtil.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.urlfrontier;
import static org.apache.stormcrawler.urlfrontier.Constants.URLFRONTIER_DEFAULT_PORT;
diff --git a/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Spout.java b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Spout.java
index 100f8cbb..6aa9ef7f 100644
--- a/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Spout.java
+++ b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/Spout.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.urlfrontier;
import static org.apache.stormcrawler.urlfrontier.Constants.URLFRONTIER_ADDRESS_KEY;
diff --git a/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBolt.java b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBolt.java
index d05df852..87c8fa04 100644
--- a/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBolt.java
+++ b/external/urlfrontier/src/main/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.urlfrontier;
import static org.apache.stormcrawler.urlfrontier.Constants.URLFRONTIER_ADDRESS_KEY;
@@ -380,7 +381,9 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt
String[] vals = metadata.getValues(k);
if (vals != null) {
Builder builder = StringList.newBuilder();
- for (String v : vals) builder.addValues(v);
+ for (String v : vals) {
+ builder.addValues(v);
+ }
mdCopy.put(k, builder.build());
}
}
diff --git a/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBoltTest.java b/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBoltTest.java
index 2a4112b1..c10b67ed 100644
--- a/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBoltTest.java
+++ b/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/StatusUpdaterBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.urlfrontier;
import static org.mockito.Mockito.mock;
@@ -71,7 +72,9 @@ class StatusUpdaterBoltTest {
void before() {
String image = "crawlercommons/url-frontier";
String version = System.getProperty("urlfrontier-version");
- if (version != null) image += ":" + version;
+ if (version != null) {
+ image += ":" + version;
+ }
urlFrontierContainer = new URLFrontierContainer(image);
urlFrontierContainer.start();
bolt = new StatusUpdaterBolt();
diff --git a/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainer.java b/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainer.java
index d42e0060..b5ecc3b4 100644
--- a/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainer.java
+++ b/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainer.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.urlfrontier;
import java.util.ArrayList;
@@ -101,7 +102,9 @@ public class URLFrontierContainer extends GenericContainer<URLFrontierContainer>
@Nullable
public ConnectionInfo.Prometheus getPrometheusConnection() {
- if (prometheusPort == -1) return null;
+ if (prometheusPort == -1) {
+ return null;
+ }
return new ConnectionInfo.Prometheus(getHost(), getMappedPort(prometheusPort));
}
diff --git a/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainerConfig.java b/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainerConfig.java
index f08b3507..c62e94c7 100644
--- a/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainerConfig.java
+++ b/external/urlfrontier/src/test/java/org/apache/stormcrawler/urlfrontier/URLFrontierContainerConfig.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.urlfrontier;
import org.jetbrains.annotations.NotNull;
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/FileTimeSizeRotationPolicy.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/FileTimeSizeRotationPolicy.java
index cbfec6c5..e8f7753c 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/FileTimeSizeRotationPolicy.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/FileTimeSizeRotationPolicy.java
@@ -12,6 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.warc;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
@@ -20,7 +21,7 @@ import org.apache.storm.tuple.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/** Rotates a file based on size or optionally time, whichever occurs first * */
+/** Rotates a file based on size or optionally time, whichever occurs first. * */
public class FileTimeSizeRotationPolicy implements FileRotationPolicy {
private static final Logger LOG = LoggerFactory.getLogger(FileSizeRotationPolicy.class);
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/GzipHdfsBolt.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/GzipHdfsBolt.java
index 1537f10b..7b75cf73 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/GzipHdfsBolt.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/GzipHdfsBolt.java
@@ -12,6 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.warc;
import java.io.IOException;
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java
index fbd47634..9c2a531d 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java
@@ -12,6 +12,7 @@
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.warc;
import java.nio.ByteBuffer;
@@ -61,23 +62,33 @@ public class MetadataRecordFormat extends WARCRecordFormat {
String url = tuple.getStringByField("url");
Metadata metadata = (Metadata) tuple.getValueByField("metadata");
- if (metadata == null) return new byte[0];
- if (url == null) return new byte[0];
+ if (metadata == null) {
+ return new byte[0];
+ }
+ if (url == null) {
+ return new byte[0];
+ }
final StringBuilder payload = new StringBuilder();
// get the metadata key / values to save in the WARCs
for (String key : metadataKeys) {
final String[] values = metadata.getValues(key);
- if (values == null || values.length == 0) continue;
+ if (values == null || values.length == 0) {
+ continue;
+ }
for (String value : values) {
- if (StringUtils.isBlank(value)) continue;
+ if (StringUtils.isBlank(value)) {
+ continue;
+ }
payload.append(key).append(": ").append(value).append(CRLF);
}
}
// no payload? don't bother
- if (payload.length() == 0) return new byte[0];
+ if (payload.length() == 0) {
+ return new byte[0];
+ }
final byte[] metadata_representation = payload.toString().getBytes(StandardCharsets.UTF_8);
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCFileNameFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCFileNameFormat.java
index 319ef1cb..4f41a185 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCFileNameFormat.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCFileNameFormat.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.warc;
import java.text.SimpleDateFormat;
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java
index 3251590f..8bb1937d 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.warc;
import java.io.IOException;
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
index 66882616..4555fac3 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.warc;
import static org.apache.stormcrawler.protocol.ProtocolResponse.REQUEST_TIME_KEY;
@@ -141,7 +142,9 @@ public class WARCRecordFormat implements RecordFormat {
// http://bibnum.bnf.fr/warc/WARC_ISO_28500_version1_latestdraft.pdf
for (Entry<String, String> entry : fields.entrySet()) {
String key = entry.getKey();
- if (key.startsWith("WARC-")) continue;
+ if (key.startsWith("WARC-")) {
+ continue;
+ }
fieldsBuffer.append(key).append(": ").append(entry.getValue()).append(CRLF);
}
@@ -170,7 +173,10 @@ public class WARCRecordFormat implements RecordFormat {
* @return safe HTTP response header
*/
public static String fixHttpHeaders(String headers, int contentLength) {
- int start = 0, lineEnd = 0, last = 0, trailingCrLf = 0;
+ int start = 0;
+ int lineEnd = 0;
+ int last = 0;
+ int trailingCrLf = 0;
final StringBuilder replacement = new StringBuilder();
while (start < headers.length()) {
lineEnd = headers.indexOf(CRLF, start);
@@ -393,9 +399,8 @@ public class WARCRecordFormat implements RecordFormat {
// provide a ContentType if type response
if (WARCTypeValue.equals(WARC_TYPE_RESPONSE)) {
buffer.append("Content-Type: application/http; msgtype=response").append(CRLF);
- }
- // for resources just use the content type provided by the server if any
- else {
+ } else {
+ // for resources just use the content type provided by the server if any
String ct = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMDprefix);
if (StringUtils.isBlank(ct)) {
ct = "application/octet-stream";
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java
index 57dd2633..1b4833fb 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.warc;
import static org.apache.stormcrawler.protocol.ProtocolResponse.REQUEST_HEADERS_KEY;
@@ -112,7 +113,10 @@ public class WARCRequestRecordFormat extends WARCRecordFormat {
}
private static String fixHttpHeaders(String headers) {
- int start = 0, lineEnd = 0, last = 0, trailingCrLf = 0;
+ int start = 0;
+ int lineEnd = 0;
+ int last = 0;
+ int trailingCrLf = 0;
final StringBuilder replacement = new StringBuilder();
while (start < headers.length()) {
boolean valid = true;
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java
index fb2e5a1c..afde9c85 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.warc;
import java.io.IOException;
@@ -126,7 +127,9 @@ public class WARCSpout extends FileSpout {
byte[] head = buffer.removeFirst();
List<Object> fields = scheme.deserialize(ByteBuffer.wrap(head));
warcFileInProgress = (String) fields.get(0);
- if (warcFileInProgress == null) return;
+ if (warcFileInProgress == null) {
+ return;
+ }
LOG.info("Reading WARC file {}", warcFileInProgress);
ReadableByteChannel warcChannel = null;
@@ -215,9 +218,15 @@ public class WARCSpout extends FileSpout {
}
private boolean isHttpResponse(Optional<WarcRecord> record) {
- if (!record.isPresent()) return false;
- if (!(record.get() instanceof WarcResponse)) return false;
- if (record.get().contentType().equals(MediaType.HTTP_RESPONSE)) return true;
+ if (!record.isPresent()) {
+ return false;
+ }
+ if (!(record.get() instanceof WarcResponse)) {
+ return false;
+ }
+ if (record.get().contentType().equals(MediaType.HTTP_RESPONSE)) {
+ return true;
+ }
return false;
}
@@ -287,10 +296,13 @@ public class WARCSpout extends FileSpout {
}
// dynamically growing list of buffers for large content of unknown size
ArrayList<ByteBuffer> bufs = new ArrayList<>();
- int r, read = 0;
+ int r;
+ int read = 0;
while (read < maxContentSize) {
try {
- if ((r = body.read(buf)) < 0) break; // eof
+ if ((r = body.read(buf)) < 0) {
+ break; // eof
+ }
} catch (ParsingException e) {
LOG.error("Failed to read chunked content of {}: {}", record.target(), e);
/*
@@ -417,7 +429,9 @@ public class WARCSpout extends FileSpout {
@Override
public void nextTuple() {
- if (!active) return;
+ if (!active) {
+ return;
+ }
if (buffer.isEmpty()) {
try {
@@ -432,7 +446,9 @@ public class WARCSpout extends FileSpout {
return;
}
- if (!record.isPresent()) nextRecord();
+ if (!record.isPresent()) {
+ nextRecord();
+ }
while (record.isPresent() && !isHttpResponse(record)) {
String warcType = record.get().type();
@@ -462,7 +478,9 @@ public class WARCSpout extends FileSpout {
nextRecord();
}
- if (!record.isPresent()) return;
+ if (!record.isPresent()) {
+ return;
+ }
eventCounter.scope("warc_http_response_record").incr();
WarcResponse w = (WarcResponse) record.get();
diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java
index 2a09b5d3..0dbac9bb 100644
--- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java
+++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.warc;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -160,8 +161,12 @@ class WARCHdfsBoltTest {
private static WarcRecord readWARCrecord(WarcRecord record) {
try {
// need to read the HTTP header, so that it gets parsed
- if (record instanceof WarcResponse) ((WarcResponse) record).http();
- if (record instanceof WarcRequest) ((WarcRequest) record).http();
+ if (record instanceof WarcResponse) {
+ ((WarcResponse) record).http();
+ }
+ if (record instanceof WarcRequest) {
+ ((WarcRequest) record).http();
+ }
} catch (IOException e) {
}
return record;
diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java
index 3470522f..c02fc8e3 100644
--- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java
+++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java
@@ -14,9 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.warc;
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCSpoutTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCSpoutTest.java
index 4427bc51..95ce95a7 100644
--- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCSpoutTest.java
+++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCSpoutTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.stormcrawler.warc;
import static org.junit.jupiter.api.Assertions.assertEquals;