You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ma...@apache.org on 2016/02/23 11:38:32 UTC
svn commit: r1731836 - in /nutch/trunk: CHANGES.txt conf/nutch-default.xml
src/java/org/apache/nutch/fetcher/FetcherThread.java
src/java/org/apache/nutch/parse/ParseOutputFormat.java
Author: markus
Date: Tue Feb 23 10:38:31 2016
New Revision: 1731836
URL: http://svn.apache.org/viewvc?rev=1731836&view=rev
Log:
NUTCH-2221 Introduce db.ignore.internal.links to FetcherThread
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/conf/nutch-default.xml
nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherThread.java
nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1731836&r1=1731835&r2=1731836&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue Feb 23 10:38:31 2016
@@ -10,6 +10,8 @@ in the release announcement and keep it
Nutch Change Log
+* NUTCH-2221 Introduce db.ignore.internal.links to FetcherThread (markus)
+
* NUTCH-2220 Rename db.* options used only by the linkdb to linkdb.* (markus)
* NUTCH-2228 Plugin index-replace unit test broken on Java 8 (snagel via markus)
Modified: nutch/trunk/conf/nutch-default.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=1731836&r1=1731835&r2=1731836&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Tue Feb 23 10:38:31 2016
@@ -538,6 +538,16 @@
</property>
<property>
+ <name>db.ignore.internal.links</name>
+ <value>false</value>
+ <description>If true, outlinks leading from a page to internal hosts or domain
+ will be ignored. This is an effective way to limit the crawl to include
+ only initially injected hosts, without creating complex URLFilters.
+ See 'db.ignore.external.links.mode'.
+ </description>
+</property>
+
+<property>
<name>db.ignore.external.links</name>
<value>false</value>
<description>If true, outlinks leading from a page to external hosts or domain
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherThread.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherThread.java?rev=1731836&r1=1731835&r2=1731836&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherThread.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherThread.java Tue Feb 23 10:38:31 2016
@@ -84,6 +84,7 @@ public class FetcherThread extends Threa
private String reprUrl;
private boolean redirecting;
private int redirectCount;
+ private boolean ignoreInternalLinks;
private boolean ignoreExternalLinks;
private String ignoreExternalLinksMode;
@@ -174,6 +175,7 @@ public class FetcherThread extends Threa
maxOutlinks = (maxOutlinksPerPage < 0) ? Integer.MAX_VALUE
: maxOutlinksPerPage;
interval = conf.getInt("db.fetch.interval.default", 2592000);
+ ignoreInternalLinks = conf.getBoolean("db.ignore.internal.links", false);
ignoreExternalLinks = conf.getBoolean("db.ignore.external.links", false);
ignoreExternalLinksMode = conf.get("db.ignore.external.links.mode", "byHost");
maxOutlinkDepth = conf.getInt("fetcher.follow.outlinks.depth", -1);
@@ -428,10 +430,10 @@ public class FetcherThread extends Threa
newUrl = normalizers.normalize(newUrl, URLNormalizers.SCOPE_FETCHER);
newUrl = urlFilters.filter(newUrl);
- if (ignoreExternalLinks) {
- try {
- String origHost = new URL(urlString).getHost().toLowerCase();
- String newHost = new URL(newUrl).getHost().toLowerCase();
+ try {
+ String origHost = new URL(urlString).getHost().toLowerCase();
+ String newHost = new URL(newUrl).getHost().toLowerCase();
+ if (ignoreExternalLinks) {
if (!origHost.equals(newHost)) {
if (LOG.isDebugEnabled()) {
LOG.debug(" - ignoring redirect " + redirType + " from "
@@ -440,10 +442,20 @@ public class FetcherThread extends Threa
}
return null;
}
- } catch (MalformedURLException e) {
}
- }
-
+
+ if (ignoreInternalLinks) {
+ if (origHost.equals(newHost)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" - ignoring redirect " + redirType + " from "
+ + urlString + " to " + newUrl
+ + " because internal links are ignored");
+ }
+ return null;
+ }
+ }
+ } catch (MalformedURLException e) { }
+
if (newUrl != null && !newUrl.equals(urlString)) {
reprUrl = URLUtil.chooseRepr(reprUrl, newUrl, temp);
url = new Text(newUrl);
@@ -621,7 +633,7 @@ public class FetcherThread extends Threa
// collect outlinks for subsequent db update
Outlink[] links = parseData.getOutlinks();
int outlinksToStore = Math.min(maxOutlinks, links.length);
- if (ignoreExternalLinks) {
+ if (ignoreExternalLinks || ignoreInternalLinks) {
URL originURL = new URL(url.toString());
// based on domain?
if ("bydomain".equalsIgnoreCase(ignoreExternalLinksMode)) {
@@ -648,7 +660,7 @@ public class FetcherThread extends Threa
String toUrl = links[i].getToUrl();
toUrl = ParseOutputFormat.filterNormalize(url.toString(), toUrl,
- origin, ignoreExternalLinks, ignoreExternalLinksMode, urlFilters, normalizers);
+ origin, ignoreInternalLinks, ignoreExternalLinks, ignoreExternalLinksMode, urlFilters, normalizers);
if (toUrl == null) {
continue;
}
Modified: nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java?rev=1731836&r1=1731835&r2=1731836&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java Tue Feb 23 10:38:31 2016
@@ -102,6 +102,8 @@ public class ParseOutputFormat implement
this.scfilters = new ScoringFilters(job);
final int interval = job.getInt("db.fetch.interval.default", 2592000);
+ final boolean ignoreInternalLinks = job.getBoolean(
+ "db.ignore.internal.links", false);
final boolean ignoreExternalLinks = job.getBoolean(
"db.ignore.external.links", false);
final String ignoreExternalLinksMode = job.get(
@@ -189,7 +191,7 @@ public class ParseOutputFormat implement
crawlOut.append(key, parseMDCrawlDatum);
// need to determine origin (once for all outlinks)
- if (ignoreExternalLinks) {
+ if (ignoreExternalLinks || ignoreInternalLinks) {
URL originURL = new URL(fromUrl.toString());
// based on domain?
if ("bydomain".equalsIgnoreCase(ignoreExternalLinksMode)) {
@@ -207,7 +209,7 @@ public class ParseOutputFormat implement
String newUrl = pstatus.getMessage();
int refreshTime = Integer.valueOf(pstatus.getArgs()[1]);
newUrl = filterNormalize(fromUrl, newUrl, origin,
- ignoreExternalLinks, ignoreExternalLinksMode, filters, normalizers,
+ ignoreInternalLinks, ignoreExternalLinks, ignoreExternalLinksMode, filters, normalizers,
URLNormalizers.SCOPE_FETCHER);
if (newUrl != null) {
@@ -238,7 +240,7 @@ public class ParseOutputFormat implement
// Only normalize and filter if fetcher.parse = false
if (!isParsing) {
toUrl = ParseOutputFormat.filterNormalize(fromUrl, toUrl, origin,
- ignoreExternalLinks, ignoreExternalLinksMode, filters, normalizers);
+ ignoreInternalLinks, ignoreExternalLinks, ignoreExternalLinksMode, filters, normalizers);
if (toUrl == null) {
continue;
}
@@ -316,37 +318,52 @@ public class ParseOutputFormat implement
}
public static String filterNormalize(String fromUrl, String toUrl,
- String fromHost, boolean ignoreExternalLinks,
+ String fromHost, boolean ignoreInternalLinks, boolean ignoreExternalLinks,
String ignoreExternalLinksMode, URLFilters filters,
URLNormalizers normalizers) {
- return filterNormalize(fromUrl, toUrl, fromHost, ignoreExternalLinks,
+ return filterNormalize(fromUrl, toUrl, fromHost, ignoreInternalLinks, ignoreExternalLinks,
ignoreExternalLinksMode, filters, normalizers,
URLNormalizers.SCOPE_OUTLINK);
}
public static String filterNormalize(String fromUrl, String toUrl,
- String origin, boolean ignoreExternalLinks, String ignoreExternalLinksMode, URLFilters filters,
+ String origin, boolean ignoreInternalLinks, boolean ignoreExternalLinks, String ignoreExternalLinksMode, URLFilters filters,
URLNormalizers normalizers, String urlNormalizerScope) {
// ignore links to self (or anchors within the page)
if (fromUrl.equals(toUrl)) {
return null;
}
- if (ignoreExternalLinks) {
+ if (ignoreExternalLinks || ignoreInternalLinks) {
URL targetURL = null;
try {
targetURL = new URL(toUrl);
} catch (MalformedURLException e1) {
return null; // skip it
}
- if ("bydomain".equalsIgnoreCase(ignoreExternalLinksMode)) {
- String toDomain = URLUtil.getDomainName(targetURL).toLowerCase();
- if (toDomain == null || !toDomain.equals(origin)) {
- return null; // skip it
- }
- } else {
- String toHost = targetURL.getHost().toLowerCase();
- if (toHost == null || !toHost.equals(origin)) {
- return null; // skip it
+ if (ignoreExternalLinks) {
+ if ("bydomain".equalsIgnoreCase(ignoreExternalLinksMode)) {
+ String toDomain = URLUtil.getDomainName(targetURL).toLowerCase();
+ if (toDomain == null || !toDomain.equals(origin)) {
+ return null; // skip it
+ }
+ } else {
+ String toHost = targetURL.getHost().toLowerCase();
+ if (toHost == null || !toHost.equals(origin)) {
+ return null; // skip it
+ }
+ }
+ }
+ if (ignoreInternalLinks) {
+ if ("bydomain".equalsIgnoreCase(ignoreExternalLinksMode)) {
+ String toDomain = URLUtil.getDomainName(targetURL).toLowerCase();
+ if (toDomain == null || toDomain.equals(origin)) {
+ return null; // skip it
+ }
+ } else {
+ String toHost = targetURL.getHost().toLowerCase();
+ if (toHost == null || toHost.equals(origin)) {
+ return null; // skip it
+ }
}
}
}