You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ab...@apache.org on 2006/07/20 00:39:54 UTC
svn commit: r423643 - in /lucene/nutch/trunk/src:
java/org/apache/nutch/crawl/ java/org/apache/nutch/scoring/
plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/
Author: ab
Date: Wed Jul 19 15:39:53 2006
New Revision: 423643
URL: http://svn.apache.org/viewvc?rev=423643&view=rev
Log:
Fix a deficiency in the scoring API (NUTCH-321).
Modified:
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java
lucene/nutch/trunk/src/java/org/apache/nutch/scoring/ScoringFilter.java
lucene/nutch/trunk/src/java/org/apache/nutch/scoring/ScoringFilters.java
lucene/nutch/trunk/src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java?rev=423643&r1=423642&r2=423643&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java Wed Jul 19 15:39:53 2006
@@ -149,7 +149,7 @@
}
try {
- scfilters.updateDbScore((UTF8)key, result, linked);
+ scfilters.updateDbScore((UTF8)key, old, result, linked);
} catch (Exception e) {
if (LOG.isWarnEnabled()) {
LOG.warn("Couldn't update score, key=" + key + ": " + e);
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/scoring/ScoringFilter.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/scoring/ScoringFilter.java?rev=423643&r1=423642&r2=423643&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/scoring/ScoringFilter.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/scoring/ScoringFilter.java Wed Jul 19 15:39:53 2006
@@ -111,12 +111,18 @@
* initial value of the original CrawlDatum, and also score values contributed by
* inlinked pages.
* @param url url of the page
- * @param datum original datum, with original score. Filters will update it in-place.
+ * @param old original datum, with original score. May be null if this is a newly
+ * discovered page. If not null, filters should use score values from this parameter
+ * as the starting values - the {@param datum} parameter may contain values that are
+ * no longer valid, if other updates occured between generation and this update.
+ * @param datum the new datum, with the original score saved at the time when
+ * fetchlist was generated. Filters should update this in-place, and it will be saved in
+ * the crawldb.
* @param inlinked (partial) list of CrawlDatum-s (with their scores) from
* links pointing to this page, found in the current update batch.
* @throws ScoringFilterException
*/
- public void updateDbScore(UTF8 url, CrawlDatum datum, List inlinked) throws ScoringFilterException;
+ public void updateDbScore(UTF8 url, CrawlDatum old, CrawlDatum datum, List inlinked) throws ScoringFilterException;
/**
* This method calculates a Lucene document boost.
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/scoring/ScoringFilters.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/scoring/ScoringFilters.java?rev=423643&r1=423642&r2=423643&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/scoring/ScoringFilters.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/scoring/ScoringFilters.java Wed Jul 19 15:39:53 2006
@@ -100,9 +100,9 @@
}
/** Calculate updated page score during CrawlDb.update(). */
- public void updateDbScore(UTF8 url, CrawlDatum datum, List inlinked) throws ScoringFilterException {
+ public void updateDbScore(UTF8 url, CrawlDatum old, CrawlDatum datum, List inlinked) throws ScoringFilterException {
for (int i = 0; i < this.filters.length; i++) {
- this.filters[i].updateDbScore(url, datum, inlinked);
+ this.filters[i].updateDbScore(url, old, datum, inlinked);
}
}
Modified: lucene/nutch/trunk/src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java?rev=423643&r1=423642&r2=423643&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java Wed Jul 19 15:39:53 2006
@@ -77,13 +77,13 @@
}
/** Increase the score by a sum of inlinked scores. */
- public void updateDbScore(UTF8 url, CrawlDatum datum, List inlinked) throws ScoringFilterException {
+ public void updateDbScore(UTF8 url, CrawlDatum old, CrawlDatum datum, List inlinked) throws ScoringFilterException {
float adjust = 0.0f;
for (int i = 0; i < inlinked.size(); i++) {
CrawlDatum linked = (CrawlDatum)inlinked.get(i);
adjust += linked.getScore();
}
- datum.setScore(datum.getScore() + adjust);
+ datum.setScore(old.getScore() + adjust);
}
/** Store a float value of CrawlDatum.getScore() under Fetcher.SCORE_KEY. */