You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ma...@apache.org on 2012/03/08 14:52:55 UTC

svn commit: r1298394 - /nutch/trunk/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/DomainBlacklistURLFilter.java

Author: markus
Date: Thu Mar  8 13:52:54 2012
New Revision: 1298394

URL: http://svn.apache.org/viewvc?rev=1298394&view=rev
Log:
NUTCH-1305 Domain(blacklist)URLFilter to trim entries

Modified:
    nutch/trunk/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/DomainBlacklistURLFilter.java

Modified: nutch/trunk/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/DomainBlacklistURLFilter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/DomainBlacklistURLFilter.java?rev=1298394&r1=1298393&r2=1298394&view=diff
==============================================================================
--- nutch/trunk/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/DomainBlacklistURLFilter.java (original)
+++ nutch/trunk/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/DomainBlacklistURLFilter.java Thu Mar  8 13:52:54 2012
@@ -80,7 +80,7 @@ public class DomainBlacklistURLFilter
     while ((line = reader.readLine()) != null) {
       if (StringUtils.isNotBlank(line) && !line.startsWith("#")) {
         // add non-blank lines and non-commented lines
-        domainSet.add(StringUtils.lowerCase(line));
+        domainSet.add(StringUtils.lowerCase(line.trim()));
       }
     }
   }
@@ -120,12 +120,12 @@ public class DomainBlacklistURLFilter
         break;
       }
     }
-    
+
     // handle blank non empty input
     if (attributeFile != null && attributeFile.trim().equals("")) {
       attributeFile = null;
     }
-    
+
     if (attributeFile != null) {
       if (LOG.isInfoEnabled()) {
         LOG.info("Attribute \"file\" is defined for plugin " + pluginName
@@ -170,9 +170,7 @@ public class DomainBlacklistURLFilter
   }
 
   public String filter(String url) {
-
     try {
-
       // match for suffix, domain, and host in that order.  more general will
       // override more specific
       String domain = URLUtil.getDomainName(url).toLowerCase().trim();
@@ -182,7 +180,7 @@ public class DomainBlacklistURLFilter
       if (domainSuffix != null) {
         suffix = domainSuffix.getDomain();
       }
-      
+
       if (domainSet.contains(suffix) || domainSet.contains(domain)
         || domainSet.contains(host)) {
         // Matches, filter!
@@ -193,7 +191,7 @@ public class DomainBlacklistURLFilter
       return url;
     }
     catch (Exception e) {
-      
+
       // if an error happens, allow the url to pass
       LOG.error("Could not apply filter on url: " + url + "\n"
         + org.apache.hadoop.util.StringUtils.stringifyException(e));