You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2025/07/22 10:00:13 UTC
(stormcrawler) branch main updated: Fix java 5 language level issues (#1609)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
The following commit(s) were added to refs/heads/main by this push:
new 4ab7afd1 Fix java 5 language level issues (#1609)
4ab7afd1 is described below
commit 4ab7afd1087a93f95dca5bbd0c057c0ebbb30ec3
Author: Dávid Szigecsán <si...@gmail.com>
AuthorDate: Tue Jul 22 12:00:09 2025 +0200
Fix java 5 language level issues (#1609)
* Terminate raw use of parameterized class
* Remove unnecessary (un)boxing
* Simplify loops
* Fix formatting
* Remove unneeded supression
* Terminate raw use of parameterized class part 2
* Simplify loops part 2
* Replace StringBuffer with StringBuilder
---
.../apache/stormcrawler/ConfigurableTopology.java | 3 ++-
.../java/org/apache/stormcrawler/Metadata.java | 5 +----
.../org/apache/stormcrawler/bolt/FetcherBolt.java | 10 +++------
.../stormcrawler/bolt/SiteMapParserBolt.java | 10 ++-------
.../stormcrawler/bolt/URLPartitionerBolt.java | 2 +-
.../filtering/metadata/MetadataFilter.java | 5 +----
.../filtering/regex/RegexURLNormalizer.java | 6 +----
.../stormcrawler/indexing/StdOutIndexer.java | 9 +++-----
.../apache/stormcrawler/jsoup/LinkParseFilter.java | 4 ++--
.../org/apache/stormcrawler/jsoup/XPathFilter.java | 7 ++----
.../CommaSeparatedToMultivaluedMetadata.java | 6 ++---
.../parse/filter/DomainParseFilter.java | 2 +-
.../stormcrawler/parse/filter/LinkParseFilter.java | 6 ++---
.../stormcrawler/parse/filter/XPathFilter.java | 4 +---
.../stormcrawler/persistence/DefaultScheduler.java | 5 +----
.../persistence/urlbuffer/PriorityURLBuffer.java | 7 +-----
.../protocol/selenium/RemoteDriverProtocol.java | 7 ++----
.../org/apache/stormcrawler/util/ConfUtils.java | 26 ++++++++++++----------
.../apache/stormcrawler/util/PerSecondReducer.java | 2 +-
.../apache/stormcrawler/util/URLPartitioner.java | 2 +-
.../stormcrawler/util/URLStreamGrouping.java | 2 +-
.../stormcrawler/TestMetadataSerialization.java | 2 +-
.../java/org/apache/stormcrawler/TestUtil.java | 2 +-
.../stormcrawler/bolt/AbstractFetcherBoltTest.java | 4 ++--
.../stormcrawler/bolt/FeedParserBoltTest.java | 4 ++--
.../stormcrawler/bolt/JSoupParserBoltTest.java | 12 +++++-----
.../stormcrawler/indexer/BasicIndexingTest.java | 24 ++++++++++----------
.../apache/stormcrawler/indexer/DummyIndexer.java | 5 +----
.../apache/stormcrawler/indexer/IndexerTester.java | 2 +-
.../apache/stormcrawler/json/JsoupFilterTest.java | 2 +-
.../stormcrawler/jsoup/JSoupFiltersTest.java | 2 +-
.../stormcrawler/parse/DuplicateLinksTest.java | 2 +-
.../apache/stormcrawler/parse/ParsingTester.java | 4 ++--
.../parse/filter/SubDocumentsFilterTest.java | 2 +-
.../aws/bolt/CloudSearchIndexerBolt.java | 5 +----
.../opensearch/metrics/MetricsConsumer.java | 9 +++-----
.../opensearch/persistence/AbstractSpout.java | 5 +----
.../opensearch/persistence/HybridSpout.java | 4 ++--
.../opensearch/bolt/AbstractOpenSearchTest.java | 4 ++--
.../apache/stormcrawler/solr/bolt/IndexerBolt.java | 5 +----
.../stormcrawler/solr/metrics/MetricsConsumer.java | 7 ++----
.../stormcrawler/solr/persistence/SolrSpout.java | 12 +++-------
.../solr/persistence/StatusUpdaterBolt.java | 5 +----
.../org/apache/stormcrawler/sql/IndexerBolt.java | 6 ++---
.../java/org/apache/stormcrawler/sql/SQLUtil.java | 10 ++++-----
.../stormcrawler/sql/metrics/MetricsConsumer.java | 7 ++----
.../apache/stormcrawler/tika/ParserBoltTest.java | 4 ++--
.../apache/stormcrawler/warc/WARCRecordFormat.java | 7 ++----
48 files changed, 107 insertions(+), 180 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/ConfigurableTopology.java b/core/src/main/java/org/apache/stormcrawler/ConfigurableTopology.java
index bde9c517..06219ae8 100644
--- a/core/src/main/java/org/apache/stormcrawler/ConfigurableTopology.java
+++ b/core/src/main/java/org/apache/stormcrawler/ConfigurableTopology.java
@@ -36,7 +36,8 @@ public abstract class ConfigurableTopology {
public static void start(ConfigurableTopology topology, String args[]) {
// loads the default configuration file
- Map defaultSCConfig = Utils.findAndReadConfigFile("crawler-default.yaml", false);
+ Map<String, Object> defaultSCConfig =
+ Utils.findAndReadConfigFile("crawler-default.yaml", false);
topology.conf.putAll(ConfUtils.extractConfigElement(defaultSCConfig));
String[] remainingArgs = topology.parse(args);
diff --git a/core/src/main/java/org/apache/stormcrawler/Metadata.java b/core/src/main/java/org/apache/stormcrawler/Metadata.java
index 6c63e9e8..7cff857d 100644
--- a/core/src/main/java/org/apache/stormcrawler/Metadata.java
+++ b/core/src/main/java/org/apache/stormcrawler/Metadata.java
@@ -23,7 +23,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.ConcurrentModificationException;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
@@ -194,9 +193,7 @@ public class Metadata {
public String toString(String prefix) {
StringBuilder sb = new StringBuilder();
if (prefix == null) prefix = "";
- Iterator<Entry<String, String[]>> iter = md.entrySet().iterator();
- while (iter.hasNext()) {
- Entry<String, String[]> entry = iter.next();
+ for (Entry<String, String[]> entry : md.entrySet()) {
for (String val : entry.getValue()) {
sb.append(prefix).append(entry.getKey()).append(": ").append(val).append("\n");
}
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
index c7f361ad..36acb0e7 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
@@ -992,17 +992,13 @@ public class FetcherBolt extends StatusEmitterBolt {
StringBuilder sb = new StringBuilder();
synchronized (fetchQueues.queues) {
sb.append("\nNum queues : ").append(fetchQueues.queues.size());
- Iterator<Entry<String, FetchItemQueue>> iterator =
- fetchQueues.queues.entrySet().iterator();
- while (iterator.hasNext()) {
- Entry<String, FetchItemQueue> entry = iterator.next();
+ for (Entry<String, FetchItemQueue> entry : fetchQueues.queues.entrySet()) {
sb.append("\nQueue ID : ").append(entry.getKey());
FetchItemQueue fiq = entry.getValue();
sb.append("\t size : ").append(fiq.getQueueSize());
sb.append("\t in progress : ").append(fiq.getInProgressSize());
- Iterator<FetchItem> urlsIter = fiq.queue.iterator();
- while (urlsIter.hasNext()) {
- sb.append("\n\t").append(urlsIter.next().url);
+ for (FetchItem fetchItem : fiq.queue) {
+ sb.append("\n\t").append(fetchItem.url);
}
}
LOG.info("Dumping queue content {}", sb.toString());
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
index 8f0f4afd..d7f5ee80 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
@@ -37,7 +37,6 @@ import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
-import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -208,9 +207,7 @@ public class SiteMapParserBolt extends StatusEmitterBolt {
// keep the subsitemaps as outlinks
// they will be fetched and parsed in the following steps
- Iterator<AbstractSiteMap> iter = subsitemaps.iterator();
- while (iter.hasNext()) {
- AbstractSiteMap asm = iter.next();
+ for (AbstractSiteMap asm : subsitemaps) {
String target = asm.getUrl().toExternalForm();
Date lastModified = asm.getLastModified();
@@ -261,10 +258,7 @@ public class SiteMapParserBolt extends StatusEmitterBolt {
SiteMap sm = (SiteMap) siteMap;
// TODO see what we can do with the LastModified info
Collection<SiteMapURL> sitemapURLs = sm.getSiteMapUrls();
- Iterator<SiteMapURL> iter = sitemapURLs.iterator();
- while (iter.hasNext()) {
- SiteMapURL smurl = iter.next();
-
+ for (SiteMapURL smurl : sitemapURLs) {
// TODO handle priority in metadata
double priority = smurl.getPriority();
// TODO convert the frequency into a numerical value and handle
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java
index 8849e2b7..d38fa831 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java
@@ -163,7 +163,7 @@ public class URLPartitionerBolt extends BaseRichBolt {
final int MAX_ENTRIES = 500;
cache =
- new LinkedHashMap(MAX_ENTRIES + 1, .75F, true) {
+ new LinkedHashMap<>(MAX_ENTRIES + 1, .75F, true) {
// This method is called just after a new entry has been added
@Override
public boolean removeEldestEntry(Map.Entry eldest) {
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/metadata/MetadataFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/metadata/MetadataFilter.java
index 9aa2598c..955411a7 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/metadata/MetadataFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/metadata/MetadataFilter.java
@@ -18,7 +18,6 @@ package org.apache.stormcrawler.filtering.metadata;
import com.fasterxml.jackson.databind.JsonNode;
import java.net.URL;
-import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map;
import java.util.Map.Entry;
@@ -54,9 +53,7 @@ public class MetadataFilter extends URLFilter {
return urlToFilter;
}
// check whether any of the metadata can be found in the source
- Iterator<String[]> iter = mdFilters.iterator();
- while (iter.hasNext()) {
- String[] kv = iter.next();
+ for (String[] kv : mdFilters) {
String[] vals = sourceMetadata.getValues(kv[0]);
if (vals == null) {
continue;
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLNormalizer.java
index 4679f8e1..275e27c2 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLNormalizer.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/regex/RegexURLNormalizer.java
@@ -28,7 +28,6 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
@@ -101,10 +100,7 @@ public class RegexURLNormalizer extends URLFilter {
public @Nullable String filter(
@Nullable URL sourceUrl, @Nullable Metadata sourceMetadata, @NotNull String urlString) {
- Iterator<Rule> i = rules.iterator();
- while (i.hasNext()) {
- Rule r = i.next();
-
+ for (Rule r : rules) {
Matcher matcher = r.pattern.matcher(urlString);
urlString = matcher.replaceAll(r.substitution);
diff --git a/core/src/main/java/org/apache/stormcrawler/indexing/StdOutIndexer.java b/core/src/main/java/org/apache/stormcrawler/indexing/StdOutIndexer.java
index 34c58acd..dfb946cf 100644
--- a/core/src/main/java/org/apache/stormcrawler/indexing/StdOutIndexer.java
+++ b/core/src/main/java/org/apache/stormcrawler/indexing/StdOutIndexer.java
@@ -18,7 +18,6 @@ package org.apache.stormcrawler.indexing;
import static org.apache.stormcrawler.Constants.StatusStreamName;
-import java.util.Iterator;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.task.OutputCollector;
@@ -75,12 +74,10 @@ public class StdOutIndexer extends AbstractIndexerBolt {
// which metadata to display?
Map<String, String[]> keyVals = filterMetadata(metadata);
- Iterator<String> iterator = keyVals.keySet().iterator();
- while (iterator.hasNext()) {
- String fieldName = iterator.next();
- String[] values = keyVals.get(fieldName);
+ for (Map.Entry<String, String[]> entry : keyVals.entrySet()) {
+ String[] values = entry.getValue();
for (String value : values) {
- System.out.println(fieldName + "\t" + trimValue(value));
+ System.out.println(entry.getKey() + "\t" + trimValue(value));
}
}
diff --git a/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java b/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
index daa57eae..2a6f0bdc 100644
--- a/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
@@ -119,11 +119,11 @@ public class LinkParseFilter extends XPathFilter {
}
}
- parse.setOutlinks(new ArrayList(dedup.values()));
+ parse.setOutlinks(new ArrayList<>(dedup.values()));
}
@Override
- public void configure(@NotNull Map stormConf, @NotNull JsonNode filterParams) {
+ public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filterParams) {
super.configure(stormConf, filterParams);
this.metadataTransfer = MetadataTransfer.getInstance(stormConf);
this.urlFilters = URLFilters.fromConf(stormConf);
diff --git a/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java b/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java
index 73c2f98f..a60d194d 100644
--- a/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java
@@ -63,9 +63,8 @@ public class XPathFilter extends AbstractConfigurable implements JSoupFilter {
}
}
- @SuppressWarnings("rawtypes")
@Override
- public void configure(@NotNull Map stormConf, @NotNull JsonNode filterParams) {
+ public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filterParams) {
super.configure(stormConf, filterParams);
java.util.Iterator<Entry<String, JsonNode>> iter = filterParams.fields();
while (iter.hasNext()) {
@@ -101,9 +100,7 @@ public class XPathFilter extends AbstractConfigurable implements JSoupFilter {
Metadata metadata = parseData.getMetadata();
// applies the XPATH expression in the order in which they are produced
- java.util.Iterator<List<LabelledExpression>> iter = expressions.values().iterator();
- while (iter.hasNext()) {
- List<LabelledExpression> leList = iter.next();
+ for (List<LabelledExpression> leList : expressions.values()) {
for (LabelledExpression le : leList) {
try {
List<String> values = le.evaluate(doc);
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java
index c70805c7..306f1bd6 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java
@@ -18,7 +18,6 @@ package org.apache.stormcrawler.parse.filter;
import com.fasterxml.jackson.databind.JsonNode;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import org.apache.stormcrawler.Metadata;
@@ -42,9 +41,8 @@ public class CommaSeparatedToMultivaluedMetadata extends ParseFilter {
return;
}
if (node.isArray()) {
- Iterator<JsonNode> iter = node.iterator();
- while (iter.hasNext()) {
- keys.add(iter.next().asText());
+ for (JsonNode jsonNode : node) {
+ keys.add(jsonNode.asText());
}
} else {
keys.add(node.asText());
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java
index f7a260d5..ce32d9e5 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java
@@ -48,7 +48,7 @@ public class DomainParseFilter extends ParseFilter {
}
partitioner = new URLPartitioner();
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put(Constants.PARTITION_MODEParamName, partitionMode);
partitioner.configure(config);
}
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
index d3f4017d..1d6f9d12 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
@@ -81,9 +81,7 @@ public class LinkParseFilter extends XPathFilter {
}
// applies the XPATH expression in the order in which they are produced
- java.util.Iterator<List<LabelledExpression>> iter = expressions.values().iterator();
- while (iter.hasNext()) {
- List<LabelledExpression> leList = iter.next();
+ for (List<LabelledExpression> leList : expressions.values()) {
for (LabelledExpression le : leList) {
try {
List<String> values = le.evaluate(doc);
@@ -121,7 +119,7 @@ public class LinkParseFilter extends XPathFilter {
}
}
- parse.setOutlinks(new ArrayList(dedup.values()));
+ parse.setOutlinks(new ArrayList<>(dedup.values()));
}
@Override
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java
index b6eae078..f2ddf10e 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java
@@ -159,9 +159,7 @@ public class XPathFilter extends ParseFilter {
Metadata metadata = parseData.getMetadata();
// applies the XPATH expression in the order in which they are produced
- java.util.Iterator<List<LabelledExpression>> iter = expressions.values().iterator();
- while (iter.hasNext()) {
- List<LabelledExpression> leList = iter.next();
+ for (List<LabelledExpression> leList : expressions.values()) {
for (LabelledExpression le : leList) {
try {
List<String> values = le.evaluate(doc);
diff --git a/core/src/main/java/org/apache/stormcrawler/persistence/DefaultScheduler.java b/core/src/main/java/org/apache/stormcrawler/persistence/DefaultScheduler.java
index 8ec6d926..b21fdbdb 100644
--- a/core/src/main/java/org/apache/stormcrawler/persistence/DefaultScheduler.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/DefaultScheduler.java
@@ -19,7 +19,6 @@ package org.apache.stormcrawler.persistence;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
@@ -67,9 +66,7 @@ public class DefaultScheduler extends Scheduler {
// e.g. fetchInterval.FETCH_ERROR.isFeed=true
Map<String, CustomInterval> intervals = new HashMap<>();
Pattern pattern = Pattern.compile("^fetchInterval(\\..+?)?\\.(.+)=(.+)");
- Iterator<String> keyIter = stormConf.keySet().iterator();
- while (keyIter.hasNext()) {
- String key = keyIter.next();
+ for (String key : stormConf.keySet()) {
Matcher m = pattern.matcher(key);
if (!m.matches()) {
continue;
diff --git a/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/PriorityURLBuffer.java b/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/PriorityURLBuffer.java
index 9748538b..40700f39 100644
--- a/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/PriorityURLBuffer.java
+++ b/core/src/main/java/org/apache/stormcrawler/persistence/urlbuffer/PriorityURLBuffer.java
@@ -19,7 +19,6 @@ package org.apache.stormcrawler.persistence.urlbuffer;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -69,11 +68,7 @@ public class PriorityURLBuffer extends SimpleURLBuffer {
List<QueueCount> sorted = new ArrayList<>();
- // populate a sorted set with key - queues
- Iterator<Entry<String, Queue<URLMetadata>>> i = queues.entrySet().iterator();
-
- while (i.hasNext()) {
- Entry<String, Queue<URLMetadata>> entry = i.next();
+ for (Entry<String, Queue<URLMetadata>> entry : queues.entrySet()) {
String name = entry.getKey();
int ackNum = ackCount.getOrDefault(name, new AtomicInteger(0)).get();
QueueCount qc = new QueueCount(name, ackNum);
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
index 75b03bdf..219f4b0a 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
@@ -19,7 +19,6 @@ package org.apache.stormcrawler.protocol.selenium;
import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -39,9 +38,7 @@ public class RemoteDriverProtocol extends SeleniumProtocol {
private void substituteUserAgent(Map<String, Object> keyvals, final String userAgentString) {
if (keyvals == null) return;
- Iterator<Entry<String, Object>> iter = keyvals.entrySet().iterator();
- while (iter.hasNext()) {
- Entry<String, Object> entry = iter.next();
+ for (Entry<String, Object> entry : keyvals.entrySet()) {
Object val = entry.getValue();
// substitute variable $useragent for the real value
if (val instanceof String && val.toString().contains("$useragent")) {
@@ -50,7 +47,7 @@ public class RemoteDriverProtocol extends SeleniumProtocol {
} else if (val instanceof Map<?, ?>) {
substituteUserAgent((Map<String, Object>) val, userAgentString);
} else if (val instanceof List<?>) {
- List newList = new ArrayList<String>();
+ List<String> newList = new ArrayList<>();
((List<String>) val)
.forEach(
s -> {
diff --git a/core/src/main/java/org/apache/stormcrawler/util/ConfUtils.java b/core/src/main/java/org/apache/stormcrawler/util/ConfUtils.java
index e175101a..8cf34b89 100644
--- a/core/src/main/java/org/apache/stormcrawler/util/ConfUtils.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/ConfUtils.java
@@ -83,7 +83,7 @@ public class ConfUtils {
boolean defaultValue) {
Object ret = conf.get(prefix + optional + suffix);
if (ret != null) {
- return ((Boolean) ret).booleanValue();
+ return (Boolean) ret;
}
return getBoolean(conf, prefix + suffix, defaultValue);
}
@@ -93,7 +93,7 @@ public class ConfUtils {
if (ret == null) {
ret = defaultValue;
}
- return ((Boolean) ret).booleanValue();
+ return (Boolean) ret;
}
/**
@@ -143,7 +143,7 @@ public class ConfUtils {
* Return one or more Strings regardless of whether they are represented as a single String or a
* list in the config or an empty List if no value could be found for that key.
*/
- public static List<String> loadListFromConf(String paramKey, Map stormConf) {
+ public static List<String> loadListFromConf(String paramKey, Map<String, Object> stormConf) {
Object obj = stormConf.get(paramKey);
List<String> list = new LinkedList<>();
@@ -169,7 +169,10 @@ public class ConfUtils {
* @return List of String values
*/
public static List<String> loadListFromConf(
- final String prefix, final String optional, final String suffix, Map stormConf) {
+ final String prefix,
+ final String optional,
+ final String suffix,
+ Map<String, Object> stormConf) {
List<String> list = loadListFromConf(prefix + optional + suffix, stormConf);
if (!list.isEmpty()) return list;
return loadListFromConf(prefix + suffix, stormConf);
@@ -177,13 +180,12 @@ public class ConfUtils {
public static Config loadConf(String resource, Config conf) throws FileNotFoundException {
Yaml yaml = new Yaml();
- Map ret =
- (Map)
- yaml.load(
- new InputStreamReader(
- new FileInputStream(resource), Charset.defaultCharset()));
+ Map<String, Object> ret =
+ yaml.load(
+ new InputStreamReader(
+ new FileInputStream(resource), Charset.defaultCharset()));
if (ret == null) {
- ret = new HashMap();
+ ret = new HashMap<>();
}
// contains a single config element ?
else {
@@ -194,11 +196,11 @@ public class ConfUtils {
}
/** If the config consists of a single key 'config', its values are used instead */
- public static Map extractConfigElement(Map conf) {
+ public static Map<String, Object> extractConfigElement(Map<String, Object> conf) {
if (conf.size() == 1) {
Object confNode = conf.get("config");
if (confNode != null && confNode instanceof Map) {
- conf = (Map) confNode;
+ conf = (Map<String, Object>) confNode;
}
}
return conf;
diff --git a/core/src/main/java/org/apache/stormcrawler/util/PerSecondReducer.java b/core/src/main/java/org/apache/stormcrawler/util/PerSecondReducer.java
index 706ef71f..dc5aa4f6 100644
--- a/core/src/main/java/org/apache/stormcrawler/util/PerSecondReducer.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/PerSecondReducer.java
@@ -49,7 +49,7 @@ public class PerSecondReducer implements IReducer<TimeReducerState> {
double msec = System.currentTimeMillis() - accumulator.started;
if (msec == 0) return 0;
double permsec = accumulator.sum / msec;
- return Double.valueOf(permsec * 1000d);
+ return permsec * 1000d;
}
}
diff --git a/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java b/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java
index 41192cd0..c59fc74f 100644
--- a/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java
@@ -100,7 +100,7 @@ public class URLPartitioner {
return getPartition(url, metadata, mode);
}
- public void configure(Map stormConf) {
+ public void configure(Map<String, Object> stormConf) {
mode =
ConfUtils.getString(
diff --git a/core/src/main/java/org/apache/stormcrawler/util/URLStreamGrouping.java b/core/src/main/java/org/apache/stormcrawler/util/URLStreamGrouping.java
index a95d36ed..a29ed2fe 100644
--- a/core/src/main/java/org/apache/stormcrawler/util/URLStreamGrouping.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/URLStreamGrouping.java
@@ -71,7 +71,7 @@ public class URLStreamGrouping implements CustomStreamGrouping, Serializable {
this.targetTask = targetTasks;
partitioner = new URLPartitioner();
if (StringUtils.isNotBlank(partitionMode)) {
- Map<String, String> conf = new HashMap<>();
+ Map<String, Object> conf = new HashMap<>();
conf.put(Constants.PARTITION_MODEParamName, partitionMode);
partitioner.configure(conf);
}
diff --git a/core/src/test/java/org/apache/stormcrawler/TestMetadataSerialization.java b/core/src/test/java/org/apache/stormcrawler/TestMetadataSerialization.java
index 8fdf879d..6bb618d2 100644
--- a/core/src/test/java/org/apache/stormcrawler/TestMetadataSerialization.java
+++ b/core/src/test/java/org/apache/stormcrawler/TestMetadataSerialization.java
@@ -31,7 +31,7 @@ class TestMetadataSerialization {
@Test
void testSerialization() throws IOException {
- Map conf = Utils.readDefaultConfig();
+ Map<String, Object> conf = Utils.readDefaultConfig();
Config.registerSerialization(conf, Metadata.class);
KryoValuesSerializer kvs = new KryoValuesSerializer(conf);
Metadata md = new Metadata();
diff --git a/core/src/test/java/org/apache/stormcrawler/TestUtil.java b/core/src/test/java/org/apache/stormcrawler/TestUtil.java
index 1ecc11aa..033255c4 100644
--- a/core/src/test/java/org/apache/stormcrawler/TestUtil.java
+++ b/core/src/test/java/org/apache/stormcrawler/TestUtil.java
@@ -76,7 +76,7 @@ public class TestUtil {
});
when(tuple.getValueByField(anyString()))
.thenAnswer(
- new Answer() {
+ new Answer<>() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
return tupleValues.get(invocation.getArgument(0, String.class));
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
index d65a34d1..2f524cc6 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
@@ -53,7 +53,7 @@ abstract class AbstractFetcherBoltTest {
@Test
void testDodgyURL() throws IOException {
TestOutputCollector output = new TestOutputCollector();
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put("http.agent.name", "this_is_only_a_test");
bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
Tuple tuple = mock(Tuple.class);
@@ -75,7 +75,7 @@ abstract class AbstractFetcherBoltTest {
void test304(WireMockRuntimeInfo wmRuntimeInfo) {
stubFor(get(urlMatching(".+")).willReturn(aResponse().withStatus(304)));
TestOutputCollector output = new TestOutputCollector();
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put("http.agent.name", "this_is_only_a_test");
bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
Tuple tuple = mock(Tuple.class);
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java
index 29ff5a4d..c2862780 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java
@@ -57,7 +57,7 @@ class FeedParserBoltTest extends ParsingTester {
@Test
void testFeedParsingNoMT() throws IOException {
- Map parserConfig = new HashMap();
+ Map<String, Object> parserConfig = new HashMap<>();
parserConfig.put("feed.sniffContent", true);
parserConfig.put("parsefilters.config.file", "test.parsefilters.json");
parserConfig.put(ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, "http.");
@@ -72,7 +72,7 @@ class FeedParserBoltTest extends ParsingTester {
@Test
void testFeedParsingDetextBytes() throws IOException {
- Map parserConfig = new HashMap();
+ Map<String, Object> parserConfig = new HashMap<>();
parserConfig.put("feed.sniffContent", true);
parserConfig.put("parsefilters.config.file", "test.parsefilters.json");
bolt.prepare(
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java
index 80969e5f..696bb943 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java
@@ -44,7 +44,7 @@ class JSoupParserBoltTest extends ParsingTester {
*
* <META HTTP-EQUIV="Pragma" CONTENT="no-cache">
*/
- Map stormConf = new HashMap();
+ Map<String, Object> stormConf = new HashMap<>();
public static String[] tests = {
"<html><head><title>test page</title>"
@@ -118,7 +118,7 @@ class JSoupParserBoltTest extends ParsingTester {
/** Checks that content in script is not included in the text representation */
void testNoScriptInText() throws IOException {
bolt.prepare(
- new HashMap(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+ new HashMap<>(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html");
List<Object> parsedTuple = output.getEmitted().remove(0);
// check in the metadata that the values match
@@ -132,7 +132,7 @@ class JSoupParserBoltTest extends ParsingTester {
/** Checks that individual links marked as rel="nofollow" are not followed */
void testNoFollowOutlinks() throws IOException {
bolt.prepare(
- new HashMap(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+ new HashMap<>(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html");
List<List<Object>> statusTuples = output.getEmitted(Constants.StatusStreamName);
Assertions.assertEquals(25, statusTuples.size());
@@ -141,7 +141,7 @@ class JSoupParserBoltTest extends ParsingTester {
@Test
void testHTTPRobots() throws IOException {
bolt.prepare(
- new HashMap(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+ new HashMap<>(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
Metadata metadata = new Metadata();
metadata.setValues("X-Robots-Tag", new String[] {"noindex", "nofollow"});
parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html", metadata);
@@ -167,7 +167,7 @@ class JSoupParserBoltTest extends ParsingTester {
@Test
void testRobotsMetaProcessor() throws IOException {
bolt.prepare(
- new HashMap(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+ new HashMap<>(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
for (int i = 0; i < tests.length; i++) {
byte[] bytes = tests[i].getBytes(StandardCharsets.UTF_8);
parse("https://stormcrawler.apache.org", bytes, new Metadata());
@@ -194,7 +194,7 @@ class JSoupParserBoltTest extends ParsingTester {
@Test
void testHTMLRedir() throws IOException {
bolt.prepare(
- new HashMap(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+ new HashMap<>(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
parse("http://www.somesite.com", "redir.html");
List<List<Object>> statusTuples = output.getEmitted(Constants.StatusStreamName);
// one for the redir + one for the discovered
diff --git a/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java b/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java
index 3bed3937..1defed27 100644
--- a/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java
@@ -39,7 +39,7 @@ class BasicIndexingTest extends IndexerTester {
@Test
void testEmptyCanonicalURL() throws Exception {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put(AbstractIndexerBolt.urlFieldParamName, "url");
config.put(AbstractIndexerBolt.canonicalMetadataParamName, "canonical");
prepareIndexerBolt(config);
@@ -51,7 +51,7 @@ class BasicIndexingTest extends IndexerTester {
@Test
void testCanonicalURL() throws Exception {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put(AbstractIndexerBolt.urlFieldParamName, "url");
config.put(AbstractIndexerBolt.canonicalMetadataParamName, "canonical");
Metadata metadata = new Metadata();
@@ -67,7 +67,7 @@ class BasicIndexingTest extends IndexerTester {
@Test
void testRelativeCanonicalURL() throws Exception {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put(AbstractIndexerBolt.urlFieldParamName, "url");
config.put(AbstractIndexerBolt.canonicalMetadataParamName, "canonical");
Metadata metadata = new Metadata();
@@ -83,7 +83,7 @@ class BasicIndexingTest extends IndexerTester {
@Test
void testBadCanonicalURL() throws Exception {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put(AbstractIndexerBolt.urlFieldParamName, "url");
config.put(AbstractIndexerBolt.canonicalMetadataParamName, "canonical");
Metadata metadata = new Metadata();
@@ -99,7 +99,7 @@ class BasicIndexingTest extends IndexerTester {
@Test
void testOtherHostCanonicalURL() throws Exception {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put(AbstractIndexerBolt.urlFieldParamName, "url");
config.put(AbstractIndexerBolt.canonicalMetadataParamName, "canonical");
Metadata metadata = new Metadata();
@@ -115,7 +115,7 @@ class BasicIndexingTest extends IndexerTester {
@Test
void testMissingCanonicalParamConfiguration() throws Exception {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put(AbstractIndexerBolt.urlFieldParamName, "url");
Metadata metadata = new Metadata();
metadata.setValue("canonical", "https://stormcrawler.apache.org/");
@@ -130,7 +130,7 @@ class BasicIndexingTest extends IndexerTester {
@Test
void testFilterDocumentWithMetadata() throws Exception {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put(AbstractIndexerBolt.urlFieldParamName, "url");
config.put(AbstractIndexerBolt.metadataFilterParamName, "key1=value1");
Metadata metadata = new Metadata();
@@ -146,7 +146,7 @@ class BasicIndexingTest extends IndexerTester {
@Test
void testFilterDocumentWithoutMetadata() throws Exception {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put(AbstractIndexerBolt.urlFieldParamName, "url");
config.put(AbstractIndexerBolt.metadataFilterParamName, "key1=value1");
prepareIndexerBolt(config);
@@ -160,9 +160,9 @@ class BasicIndexingTest extends IndexerTester {
@Test
void testFilterMetadata() throws Exception {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put(AbstractIndexerBolt.urlFieldParamName, "url");
- final List vector = new ArrayList();
+ final List<String> vector = new ArrayList<>();
vector.add("parse.title=title");
vector.add("parse.keywords=keywords");
config.put(AbstractIndexerBolt.metadata2fieldParamName, vector);
@@ -181,7 +181,7 @@ class BasicIndexingTest extends IndexerTester {
@Test
void testEmptyFilterMetadata() throws Exception {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put(AbstractIndexerBolt.urlFieldParamName, "url");
prepareIndexerBolt(config);
Metadata metadata = new Metadata();
@@ -198,7 +198,7 @@ class BasicIndexingTest extends IndexerTester {
@Test
void testGlobFilterMetadata() throws Exception {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put(AbstractIndexerBolt.urlFieldParamName, "url");
List<String> listKV = new ArrayList<>();
listKV.add("parse.*");
diff --git a/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java b/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java
index 9326df6f..0200855d 100644
--- a/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java
+++ b/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java
@@ -17,7 +17,6 @@
package org.apache.stormcrawler.indexer;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.task.OutputCollector;
@@ -66,9 +65,7 @@ public class DummyIndexer extends AbstractIndexerBolt {
// which metadata to display?
Map<String, String[]> keyVals = filterMetadata(metadata);
- Iterator<String> iterator = keyVals.keySet().iterator();
- while (iterator.hasNext()) {
- String fieldName = iterator.next();
+ for (String fieldName : keyVals.keySet()) {
String[] values = keyVals.get(fieldName);
for (String value : values) {
fields.put(fieldName, value);
diff --git a/core/src/test/java/org/apache/stormcrawler/indexer/IndexerTester.java b/core/src/test/java/org/apache/stormcrawler/indexer/IndexerTester.java
index 284f967c..f84bf041 100644
--- a/core/src/test/java/org/apache/stormcrawler/indexer/IndexerTester.java
+++ b/core/src/test/java/org/apache/stormcrawler/indexer/IndexerTester.java
@@ -49,7 +49,7 @@ public class IndexerTester {
output = null;
}
- protected void prepareIndexerBolt(Map config) {
+ protected void prepareIndexerBolt(Map<String, Object> config) {
bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
}
diff --git a/core/src/test/java/org/apache/stormcrawler/json/JsoupFilterTest.java b/core/src/test/java/org/apache/stormcrawler/json/JsoupFilterTest.java
index bb77f167..e5095d71 100644
--- a/core/src/test/java/org/apache/stormcrawler/json/JsoupFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/json/JsoupFilterTest.java
@@ -36,7 +36,7 @@ class JsoupFilterTest extends ParsingTester {
setupParserBolt(bolt);
}
- protected void prepareParserBolt(String configFile, Map parserConfig) {
+ protected void prepareParserBolt(String configFile, Map<String, Object> parserConfig) {
parserConfig.put("jsoup.filters.config.file", configFile);
bolt.prepare(
parserConfig, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
diff --git a/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java b/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java
index fb1b2e42..cfd2b8dc 100644
--- a/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java
@@ -37,7 +37,7 @@ class JSoupFiltersTest extends ParsingTester {
setupParserBolt(bolt);
}
- protected void prepareParserBolt(String configFile, Map parserConfig) {
+ protected void prepareParserBolt(String configFile, Map<String, Object> parserConfig) {
parserConfig.put("jsoup.filters.config.file", configFile);
bolt.prepare(
parserConfig, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/DuplicateLinksTest.java b/core/src/test/java/org/apache/stormcrawler/parse/DuplicateLinksTest.java
index e105c835..5cb81f85 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/DuplicateLinksTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/DuplicateLinksTest.java
@@ -39,7 +39,7 @@ class DuplicateLinksTest extends ParsingTester {
@Test
void testSitemapSubdocuments() throws IOException {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
// generate a dummy config file
config.put("urlfilters.config.file", "basicurlnormalizer.json");
bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/ParsingTester.java b/core/src/test/java/org/apache/stormcrawler/parse/ParsingTester.java
index e4f8adbe..b8953e50 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/ParsingTester.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/ParsingTester.java
@@ -52,10 +52,10 @@ public class ParsingTester {
}
protected void prepareParserBolt(String configFile) {
- prepareParserBolt(configFile, new HashMap());
+ prepareParserBolt(configFile, new HashMap<>());
}
- protected void prepareParserBolt(String configFile, Map parserConfig) {
+ protected void prepareParserBolt(String configFile, Map<String, Object> parserConfig) {
parserConfig.put("parsefilters.config.file", configFile);
bolt.prepare(
parserConfig, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsFilterTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsFilterTest.java
index efb9f737..a71e8334 100644
--- a/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsFilterTest.java
@@ -36,7 +36,7 @@ class SubDocumentsFilterTest extends ParsingTester {
@Test
void testSitemapSubdocuments() throws IOException {
- Map config = new HashMap();
+ Map<String, Object> config = new HashMap<>();
config.put("detect.mimetype", false);
prepareParserBolt("test.subdocfilter.json", config);
Metadata metadata = new Metadata();
diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
index 03618df7..676cd2ce 100644
--- a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
@@ -44,7 +44,6 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -142,9 +141,7 @@ public class CloudSearchIndexerBolt extends AbstractIndexerBolt {
// retrieve the domain name
DescribeDomainsResult domains = cl.describeDomains(new DescribeDomainsRequest());
- Iterator<DomainStatus> dsiter = domains.getDomainStatusList().iterator();
- while (dsiter.hasNext()) {
- DomainStatus ds = dsiter.next();
+ for (DomainStatus ds : domains.getDomainStatusList()) {
if (ds.getDocService().getEndpoint().equals(endpoint)) {
domainName = ds.getDomainName();
break;
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
index 2fc7058f..2675e750 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
-import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
@@ -69,7 +68,7 @@ public class MetricsConsumer implements IMetricsConsumer {
@Override
public void prepare(
- Map stormConf,
+ Map<String, Object> stormConf,
Object registrationArgument,
TopologyContext context,
IErrorReporter errorReporter) {
@@ -112,14 +111,12 @@ public class MetricsConsumer implements IMetricsConsumer {
if (value instanceof Number) {
indexDataPoint(taskInfo, now, nameprefix, ((Number) value).doubleValue());
} else if (value instanceof Map) {
- Iterator<Entry> keyValiter = ((Map) value).entrySet().iterator();
- while (keyValiter.hasNext()) {
- Entry entry = keyValiter.next();
+ for (Entry<String, Object> entry : ((Map<String, Object>) value).entrySet()) {
String newnameprefix = nameprefix + "." + entry.getKey();
handleDataPoints(taskInfo, newnameprefix, entry.getValue(), now);
}
} else if (value instanceof Collection) {
- for (Object collectionObj : (Collection) value) {
+ for (Object collectionObj : (Collection<Object>) value) {
handleDataPoints(taskInfo, nameprefix, collectionObj, now);
}
} else {
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
index 8f2e853d..37bcd096 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
@@ -19,7 +19,6 @@ package org.apache.stormcrawler.opensearch.persistence;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -194,9 +193,7 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
Map<String, List<String>> mdAsMap = (Map<String, List<String>>) keyValues.get("metadata");
Metadata metadata = new Metadata();
if (mdAsMap != null) {
- Iterator<Entry<String, List<String>>> mdIter = mdAsMap.entrySet().iterator();
- while (mdIter.hasNext()) {
- Entry<String, List<String>> mdEntry = mdIter.next();
+ for (Entry<String, List<String>> mdEntry : mdAsMap.entrySet()) {
String key = mdEntry.getKey();
// periods are not allowed - replace with %2E
key = key.replaceAll("%2E", "\\.");
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
index 08759fa9..60d47515 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
@@ -185,8 +185,8 @@ public class HybridSpout extends AggregationSpout implements EmptyQueueListener
}
Object key_as_object = sourceAsMap.get(pfield);
if (key_as_object instanceof List) {
- if (((List) (key_as_object)).size() == 1)
- key = (String) ((List) key_as_object).get(0);
+ if (((List<String>) (key_as_object)).size() == 1)
+ key = ((List<String>) key_as_object).get(0);
} else {
key = key_as_object.toString();
}
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
index d7e305c8..1132dd43 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
@@ -29,8 +29,8 @@ public abstract class AbstractOpenSearchTest {
public static final String PASSWORD = "This1sAPassw0rd";
- protected GenericContainer opensearchContainer =
- new GenericContainer(
+ protected GenericContainer<?> opensearchContainer =
+ new GenericContainer<>(
DockerImageName.parse(
"opensearchproject/opensearch:" + OPENSEARCH_VERSION))
.withExposedPorts(9200)
diff --git a/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/IndexerBolt.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/IndexerBolt.java
index 3fa93289..25eaffda 100644
--- a/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/IndexerBolt.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/IndexerBolt.java
@@ -18,7 +18,6 @@ package org.apache.stormcrawler.solr.bolt;
import static org.apache.stormcrawler.Constants.StatusStreamName;
-import java.util.Iterator;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.common.SolrInputDocument;
@@ -113,9 +112,7 @@ public class IndexerBolt extends AbstractIndexerBolt {
// select which metadata to index
Map<String, String[]> keyVals = filterMetadata(metadata);
- Iterator<String> iterator = keyVals.keySet().iterator();
- while (iterator.hasNext()) {
- String fieldName = iterator.next();
+ for (String fieldName : keyVals.keySet()) {
String[] values = keyVals.get(fieldName);
for (String value : values) {
if (!ignoreEmptyFields() || StringUtils.isNotBlank(value)) {
diff --git a/external/solr/src/main/java/org/apache/stormcrawler/solr/metrics/MetricsConsumer.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/metrics/MetricsConsumer.java
index 6fd6161e..916ebdd8 100644
--- a/external/solr/src/main/java/org/apache/stormcrawler/solr/metrics/MetricsConsumer.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/metrics/MetricsConsumer.java
@@ -20,7 +20,6 @@ import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
-import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
@@ -80,14 +79,12 @@ public class MetricsConsumer implements IMetricsConsumer {
if (value instanceof Number) {
indexDataPoint(taskInfo, now, nameprefix, ((Number) value).doubleValue());
} else if (value instanceof Map) {
- Iterator<Entry> keyValiter = ((Map) value).entrySet().iterator();
- while (keyValiter.hasNext()) {
- Entry entry = keyValiter.next();
+ for (Entry<String, Object> entry : ((Map<String, Object>) value).entrySet()) {
String newnameprefix = nameprefix + "." + entry.getKey();
handleDataPoints(taskInfo, newnameprefix, entry.getValue(), now);
}
} else if (value instanceof Collection) {
- for (Object collectionObj : (Collection) value) {
+ for (Object collectionObj : (Collection<Object>) value) {
handleDataPoints(taskInfo, nameprefix, collectionObj, now);
}
} else {
diff --git a/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/SolrSpout.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/SolrSpout.java
index eb0283d9..35ccb2f4 100644
--- a/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/SolrSpout.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/SolrSpout.java
@@ -18,7 +18,6 @@ package org.apache.stormcrawler.solr.persistence;
import java.time.Instant;
import java.util.Collection;
-import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
@@ -253,18 +252,13 @@ public class SolrSpout extends AbstractQueryingSpout {
Metadata metadata = new Metadata();
- Iterator<String> keyIterators = doc.getFieldNames().iterator();
- while (keyIterators.hasNext()) {
- String key = keyIterators.next();
-
+ for (String key : doc.getFieldNames()) {
if (key.startsWith(prefix)) {
Collection<Object> values = doc.getFieldValues(key);
key = key.substring(prefix.length());
- Iterator<Object> valueIterator = values.iterator();
- while (valueIterator.hasNext()) {
- String value = (String) valueIterator.next();
- metadata.addValue(key, value);
+ for (Object value : values) {
+ metadata.addValue(key, (String) value);
}
}
}
diff --git a/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/StatusUpdaterBolt.java b/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/StatusUpdaterBolt.java
index f332fdcd..61ff78ca 100644
--- a/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/StatusUpdaterBolt.java
+++ b/external/solr/src/main/java/org/apache/stormcrawler/solr/persistence/StatusUpdaterBolt.java
@@ -17,7 +17,6 @@
package org.apache.stormcrawler.solr.persistence;
import java.util.Date;
-import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
@@ -106,9 +105,7 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt {
doc.setField("status", status.name());
- Iterator<String> keyIterator = metadata.keySet().iterator();
- while (keyIterator.hasNext()) {
- String key = keyIterator.next();
+ for (String key : metadata.keySet()) {
String[] values = metadata.getValues(key);
doc.setField(String.format(Locale.ROOT, "%s.%s", mdPrefix, key), values);
}
diff --git a/external/sql/src/main/java/org/apache/stormcrawler/sql/IndexerBolt.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/IndexerBolt.java
index bc384223..9e7add9e 100644
--- a/external/sql/src/main/java/org/apache/stormcrawler/sql/IndexerBolt.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/IndexerBolt.java
@@ -50,7 +50,7 @@ public class IndexerBolt extends AbstractIndexerBolt {
private String tableName;
- private Map conf;
+ private Map<String, Object> conf;
@Override
public void prepare(
@@ -99,8 +99,8 @@ public class IndexerBolt extends AbstractIndexerBolt {
Object[] keys = keyVals.keySet().toArray();
- for (int i = 0; i < keys.length; i++) {
- query.append(", ").append((String) keys[i]);
+ for (Object o : keys) {
+ query.append(", ").append((String) o);
}
query.append(") values(?");
diff --git a/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLUtil.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLUtil.java
index dcdb6090..ad75f243 100644
--- a/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLUtil.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/SQLUtil.java
@@ -27,16 +27,16 @@ public class SQLUtil {
private SQLUtil() {}
- public static Connection getConnection(Map stormConf) throws SQLException {
+ public static Connection getConnection(Map<String, Object> stormConf) throws SQLException {
// SQL connection details
- Map<String, String> sqlConf = (Map) stormConf.get("sql.connection");
+ Map<String, Object> sqlConf = (Map<String, Object>) stormConf.get("sql.connection");
if (sqlConf == null) {
throw new RuntimeException(
"Missing SQL connection config, add a section 'sql.connection' to the configuration");
}
- String url = sqlConf.get("url");
+ String url = (String) sqlConf.get("url");
if (url == null) {
throw new RuntimeException(
"Missing SQL url, add an entry 'url' to the section 'sql.connection' of the configuration");
@@ -44,8 +44,8 @@ public class SQLUtil {
Properties props = new Properties();
- for (Entry<String, String> entry : sqlConf.entrySet()) {
- props.setProperty(entry.getKey(), entry.getValue());
+ for (Entry<String, Object> entry : sqlConf.entrySet()) {
+ props.setProperty(entry.getKey(), (String) entry.getValue());
}
return DriverManager.getConnection(url, props);
diff --git a/external/sql/src/main/java/org/apache/stormcrawler/sql/metrics/MetricsConsumer.java b/external/sql/src/main/java/org/apache/stormcrawler/sql/metrics/MetricsConsumer.java
index 013a4974..0062f154 100644
--- a/external/sql/src/main/java/org/apache/stormcrawler/sql/metrics/MetricsConsumer.java
+++ b/external/sql/src/main/java/org/apache/stormcrawler/sql/metrics/MetricsConsumer.java
@@ -22,7 +22,6 @@ import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.Collection;
-import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.storm.metric.api.IMetricsConsumer;
@@ -92,14 +91,12 @@ public class MetricsConsumer implements IMetricsConsumer {
LOG.error("Exception while indexing datapoint", e);
}
} else if (value instanceof Map) {
- Iterator<Entry> keyValiter = ((Map) value).entrySet().iterator();
- while (keyValiter.hasNext()) {
- Entry entry = keyValiter.next();
+ for (Entry<String, Object> entry : ((Map<String, Object>) value).entrySet()) {
String newnameprefix = nameprefix + "." + entry.getKey();
handleDataPoints(preparedStmt, taskInfo, newnameprefix, entry.getValue(), now);
}
} else if (value instanceof Collection) {
- for (Object collectionObj : (Collection) value) {
+ for (Object collectionObj : (Collection<Object>) value) {
handleDataPoints(preparedStmt, taskInfo, nameprefix, collectionObj, now);
}
} else {
diff --git a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java
index 7d9717f0..5b2bf3cc 100644
--- a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java
+++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTest.java
@@ -51,7 +51,7 @@ class ParserBoltTest extends ParsingTester {
* @see https://issues.apache.org/jira/browse/TIKA-2096
*/
void testRecursiveDoc() throws IOException {
- Map conf = new HashMap();
+ Map<String, Object> conf = new HashMap<>();
conf.put("parser.extract.embedded", true);
bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
parse(
@@ -77,7 +77,7 @@ class ParserBoltTest extends ParsingTester {
* @see https://github.com/apache/stormcrawler/issues/712
*/
void testMimeTypeWhileList() throws IOException {
- Map conf = new HashMap();
+ Map<String, Object> conf = new HashMap<>();
conf.put("parser.mimetype.whitelist", "application/.+word.*");
conf.put(ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, "http.");
bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
index a013f6ec..6a4ffb0c 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
@@ -28,7 +28,6 @@ import java.time.DateTimeException;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
-import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
@@ -112,7 +111,7 @@ public class WARCRecordFormat implements RecordFormat {
/** Generates a WARC info entry which can be stored at the beginning of each WARC file. */
public static byte[] generateWARCInfo(Map<String, String> fields) {
- StringBuffer buffer = new StringBuffer();
+ StringBuilder buffer = new StringBuilder();
buffer.append(WARC_VERSION);
buffer.append(CRLF);
@@ -140,9 +139,7 @@ public class WARCRecordFormat implements RecordFormat {
// add WARC fields
// http://bibnum.bnf.fr/warc/WARC_ISO_28500_version1_latestdraft.pdf
- Iterator<Entry<String, String>> iter = fields.entrySet().iterator();
- while (iter.hasNext()) {
- Entry<String, String> entry = iter.next();
+ for (Entry<String, String> entry : fields.entrySet()) {
String key = entry.getKey();
if (key.startsWith("WARC-")) continue;
fieldsBuffer.append(key).append(": ").append(entry.getValue()).append(CRLF);