You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ma...@apache.org on 2012/03/08 14:52:55 UTC
svn commit: r1298394 -
/nutch/trunk/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/DomainBlacklistURLFilter.java
Author: markus
Date: Thu Mar 8 13:52:54 2012
New Revision: 1298394
URL: http://svn.apache.org/viewvc?rev=1298394&view=rev
Log:
NUTCH-1305 Domain(blacklist)URLFilter to trim entries
Modified:
nutch/trunk/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/DomainBlacklistURLFilter.java
Modified: nutch/trunk/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/DomainBlacklistURLFilter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/DomainBlacklistURLFilter.java?rev=1298394&r1=1298393&r2=1298394&view=diff
==============================================================================
--- nutch/trunk/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/DomainBlacklistURLFilter.java (original)
+++ nutch/trunk/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/DomainBlacklistURLFilter.java Thu Mar 8 13:52:54 2012
@@ -80,7 +80,7 @@ public class DomainBlacklistURLFilter
while ((line = reader.readLine()) != null) {
if (StringUtils.isNotBlank(line) && !line.startsWith("#")) {
// add non-blank lines and non-commented lines
- domainSet.add(StringUtils.lowerCase(line));
+ domainSet.add(StringUtils.lowerCase(line.trim()));
}
}
}
@@ -120,12 +120,12 @@ public class DomainBlacklistURLFilter
break;
}
}
-
+
// handle blank non empty input
if (attributeFile != null && attributeFile.trim().equals("")) {
attributeFile = null;
}
-
+
if (attributeFile != null) {
if (LOG.isInfoEnabled()) {
LOG.info("Attribute \"file\" is defined for plugin " + pluginName
@@ -170,9 +170,7 @@ public class DomainBlacklistURLFilter
}
public String filter(String url) {
-
try {
-
// match for suffix, domain, and host in that order. more general will
// override more specific
String domain = URLUtil.getDomainName(url).toLowerCase().trim();
@@ -182,7 +180,7 @@ public class DomainBlacklistURLFilter
if (domainSuffix != null) {
suffix = domainSuffix.getDomain();
}
-
+
if (domainSet.contains(suffix) || domainSet.contains(domain)
|| domainSet.contains(host)) {
// Matches, filter!
@@ -193,7 +191,7 @@ public class DomainBlacklistURLFilter
return url;
}
catch (Exception e) {
-
+
// if an error happens, allow the url to pass
LOG.error("Could not apply filter on url: " + url + "\n"
+ org.apache.hadoop.util.StringUtils.stringifyException(e));