You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ku...@apache.org on 2008/12/04 22:16:43 UTC
svn commit: r723441 [2/2] - in /lucene/nutch/trunk: ./
src/java/org/apache/nutch/scoring/
src/java/org/apache/nutch/scoring/webgraph/
src/java/org/apache/nutch/tools/compat/ src/java/org/apache/nutch/util/
src/plugin/ src/plugin/scoring-link/ src/plugi...
Added: lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java?rev=723441&view=auto
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java (added)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java Thu Dec 4 13:16:42 2008
@@ -0,0 +1,226 @@
+package org.apache.nutch.scoring.webgraph;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Random;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.ObjectWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapFileOutputFormat;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.CrawlDb;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.NutchJob;
+
+/**
+ * Updates the score from the WebGraph node database into the crawl database.
+ * Any score that is not in the node database is set to the clear score in the
+ * crawl database.
+ */
+public class ScoreUpdater
+ extends Configured
+ implements Tool, Mapper<Text, Writable, Text, ObjectWritable>,
+ Reducer<Text, ObjectWritable, Text, CrawlDatum> {
+
+ public static final Log LOG = LogFactory.getLog(ScoreUpdater.class);
+
+ private JobConf conf;
+ private float clearScore = 0.0f;
+
+ public void configure(JobConf conf) {
+ this.conf = conf;
+ clearScore = conf.getFloat("link.score.updater.clear.score", 0.0f);
+ }
+
+ /**
+ * Changes input into ObjectWritables.
+ */
+ public void map(Text key, Writable value,
+ OutputCollector<Text, ObjectWritable> output, Reporter reporter)
+ throws IOException {
+
+ ObjectWritable objWrite = new ObjectWritable();
+ objWrite.set(value);
+ output.collect(key, objWrite);
+ }
+
+ /**
+ * Creates new CrawlDatum objects with the updated score from the NodeDb or
+ * with a cleared score.
+ */
+ public void reduce(Text key, Iterator<ObjectWritable> values,
+ OutputCollector<Text, CrawlDatum> output, Reporter reporter)
+ throws IOException {
+
+ String url = key.toString();
+ Node node = null;
+ CrawlDatum datum = null;
+
+ // set the node and the crawl datum, should be one of each unless no node
+ // for url in the crawldb
+ while (values.hasNext()) {
+ ObjectWritable next = values.next();
+ Object value = next.get();
+ if (value instanceof Node) {
+ node = (Node)value;
+ }
+ else if (value instanceof CrawlDatum) {
+ datum = (CrawlDatum)value;
+ }
+ }
+
+ // datum should never be null, could happen if somehow the url was
+ // normalized or changed after being pulled from the crawldb
+ if (datum != null) {
+
+ if (node != null) {
+
+ // set the inlink score in the nodedb
+ float inlinkScore = node.getInlinkScore();
+ datum.setScore(inlinkScore);
+ LOG.debug(url + ": setting to score " + inlinkScore);
+ }
+ else {
+
+ // clear out the score in the crawldb
+ datum.setScore(clearScore);
+ LOG.debug(url + ": setting to clear score of " + clearScore);
+ }
+
+ output.collect(key, datum);
+ }
+ else {
+ LOG.debug(url + ": no datum");
+ }
+ }
+
+ public void close() {
+ }
+
+ /**
+ * Updates the inlink score in the web graph node databsae into the crawl
+ * database.
+ *
+ * @param crawlDb The crawl database to update
+ * @param webGraphDb The webgraph database to use.
+ *
+ * @throws IOException If an error occurs while updating the scores.
+ */
+ public void update(Path crawlDb, Path webGraphDb)
+ throws IOException {
+
+ Configuration conf = getConf();
+ FileSystem fs = FileSystem.get(conf);
+
+ // create a temporary crawldb with the new scores
+ LOG.info("Running crawldb update " + crawlDb);
+ Path nodeDb = new Path(webGraphDb, WebGraph.NODE_DIR);
+ Path crawlDbCurrent = new Path(crawlDb, CrawlDb.CURRENT_NAME);
+ Path newCrawlDb = new Path(crawlDb,
+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
+
+ // run the updater job outputting to the temp crawl database
+ JobConf updater = new NutchJob(conf);
+ updater.setJobName("Update CrawlDb from WebGraph");
+ FileInputFormat.addInputPath(updater, crawlDbCurrent);
+ FileInputFormat.addInputPath(updater, nodeDb);
+ FileOutputFormat.setOutputPath(updater, newCrawlDb);
+ updater.setInputFormat(SequenceFileInputFormat.class);
+ updater.setMapperClass(ScoreUpdater.class);
+ updater.setReducerClass(ScoreUpdater.class);
+ updater.setMapOutputKeyClass(Text.class);
+ updater.setMapOutputValueClass(ObjectWritable.class);
+ updater.setOutputKeyClass(Text.class);
+ updater.setOutputValueClass(CrawlDatum.class);
+ updater.setOutputFormat(MapFileOutputFormat.class);
+
+ try {
+ JobClient.runJob(updater);
+ }
+ catch (IOException e) {
+ LOG.error(StringUtils.stringifyException(e));
+
+ // remove the temp crawldb on error
+ if (fs.exists(newCrawlDb)) {
+ fs.delete(newCrawlDb, true);
+ }
+ throw e;
+ }
+
+ // install the temp crawl database
+ LOG.info("Installing new crawldb " + crawlDb);
+ CrawlDb.install(updater, crawlDb);
+ }
+
+ public static void main(String[] args)
+ throws Exception {
+ int res = ToolRunner.run(NutchConfiguration.create(), new ScoreUpdater(),
+ args);
+ System.exit(res);
+ }
+
+ /**
+ * Runs the ScoreUpdater tool.
+ */
+ public int run(String[] args)
+ throws Exception {
+
+ Options options = new Options();
+ Option helpOpts = OptionBuilder.withArgName("help").withDescription(
+ "show this help message").create("help");
+ Option crawlDbOpts = OptionBuilder.withArgName("crawldb").hasArg().withDescription(
+ "the crawldb to use").create("crawldb");
+ Option webGraphOpts = OptionBuilder.withArgName("webgraphdb").hasArg().withDescription(
+ "the webgraphdb to use").create("webgraphdb");
+ options.addOption(helpOpts);
+ options.addOption(crawlDbOpts);
+ options.addOption(webGraphOpts);
+
+ CommandLineParser parser = new GnuParser();
+ try {
+
+ CommandLine line = parser.parse(options, args);
+ if (line.hasOption("help") || !line.hasOption("webgraphdb")
+ || !line.hasOption("crawldb")) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("ScoreUpdater", options);
+ return -1;
+ }
+
+ String crawlDb = line.getOptionValue("crawldb");
+ String webGraphDb = line.getOptionValue("webgraphdb");
+ update(new Path(crawlDb), new Path(webGraphDb));
+ return 0;
+ }
+ catch (Exception e) {
+ LOG.fatal("ScoreUpdater: " + StringUtils.stringifyException(e));
+ return -1;
+ }
+ }
+}
Added: lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java?rev=723441&view=auto
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java (added)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java Thu Dec 4 13:16:42 2008
@@ -0,0 +1,629 @@
+package org.apache.nutch.scoring.webgraph;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapFileOutputFormat;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.net.URLNormalizers;
+import org.apache.nutch.parse.Outlink;
+import org.apache.nutch.parse.ParseData;
+import org.apache.nutch.util.FSUtils;
+import org.apache.nutch.util.LockUtil;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.NutchJob;
+import org.apache.nutch.util.URLUtil;
+
+/**
+ * Creates three databases, one for inlinks, one for outlinks, and a node
+ * database that holds the number of in and outlinks to a url and the current
+ * score for the url.
+ *
+ * The score is set by an analysis program such as LinkRank. The WebGraph is an
+ * update-able database. Outlinks are stored by their fetch time or by the
+ * current system time if no fetch time is available. Only the most recent
+ * version of outlinks for a given url is stored. As more crawls are executed
+ * and the WebGraph updated, newer Outlinks will replace older Outlinks. This
+ * allows the WebGraph to adapt to changes in the link structure of the web.
+ *
+ * The Inlink database is created from the Outlink database and is regenerated
+ * when the WebGraph is updated. The Node database is created from both the
+ * Inlink and Outlink databases. Because the Node database is overwritten when
+ * the WebGraph is updated and because the Node database holds current scores
+ * for urls it is recommended that a crawl-cyle (one or more full crawls) fully
+ * complete before the WebGraph is updated and some type of analysis, such as
+ * LinkRank, is run to update scores in the Node database in a stable fashion.
+ */
+public class WebGraph
+ extends Configured
+ implements Tool {
+
+ public static final Log LOG = LogFactory.getLog(WebGraph.class);
+ public static final String LOCK_NAME = ".locked";
+ public static final String INLINK_DIR = "inlinks";
+ public static final String OUTLINK_DIR = "outlinks";
+ public static final String NODE_DIR = "nodes";
+
+ /**
+ * The OutlinkDb creates a database of all outlinks. Outlinks to internal urls
+ * by domain and host can be ignored. The number of Outlinks out to a given
+ * page or domain can also be limited.
+ */
+ public static class OutlinkDb
+ extends Configured
+ implements Mapper<Text, Writable, Text, LinkDatum>,
+ Reducer<Text, LinkDatum, Text, LinkDatum> {
+
+ // ignoring internal domains, internal hosts
+ private boolean ignoreDomain = true;
+ private boolean ignoreHost = true;
+
+ // limiting urls out to a page or to a domain
+ private boolean limitPages = true;
+ private boolean limitDomains = true;
+
+ // url normalizers and job configuration
+ private URLNormalizers urlNormalizers;
+ private JobConf conf;
+
+ /**
+ * Normalizes and trims extra whitespace from the given url.
+ *
+ * @param url The url to normalize.
+ *
+ * @return The normalized url.
+ */
+ private String normalizeUrl(String url) {
+
+ String normalized = null;
+ if (urlNormalizers != null) {
+ try {
+
+ // normalize and trim the url
+ normalized = urlNormalizers.normalize(url,
+ URLNormalizers.SCOPE_DEFAULT);
+ normalized = normalized.trim();
+ }
+ catch (Exception e) {
+ LOG.warn("Skipping " + url + ":" + e);
+ normalized = null;
+ }
+ }
+ return normalized;
+ }
+
+ /**
+ * Returns the fetch time from the parse data or the current system time if
+ * the fetch time doesn't exist.
+ *
+ * @param data The parse data.
+ *
+ * @return The fetch time as a long.
+ */
+ private long getFetchTime(ParseData data) {
+
+ // default to current system time
+ long fetchTime = System.currentTimeMillis();
+ String fetchTimeStr = data.getContentMeta().get(Nutch.FETCH_TIME_KEY);
+ try {
+
+ // get the fetch time from the parse data
+ fetchTime = Long.parseLong(fetchTimeStr);
+ }
+ catch (Exception e) {
+ fetchTime = System.currentTimeMillis();
+ }
+ return fetchTime;
+ }
+
+ /**
+ * Default constructor.
+ */
+ public OutlinkDb() {
+ }
+
+ /**
+ * Configurable constructor.
+ */
+ public OutlinkDb(Configuration conf) {
+ setConf(conf);
+ }
+
+ /**
+ * Configures the OutlinkDb job. Sets up internal links and link limiting.
+ */
+ public void configure(JobConf conf) {
+ this.conf = conf;
+ ignoreHost = conf.getBoolean("link.ignore.internal.host", true);
+ ignoreDomain = conf.getBoolean("link.ignore.internal.domain", true);
+ limitPages = conf.getBoolean("link.ignore.limit.page", true);
+ limitDomains = conf.getBoolean("link.ignore.limit.domain", true);
+ urlNormalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_DEFAULT);
+ }
+
+ /**
+ * Passes through existing LinkDatum objects from an existing OutlinkDb and
+ * maps out new LinkDatum objects from new crawls ParseData.
+ */
+ public void map(Text key, Writable value,
+ OutputCollector<Text, LinkDatum> output, Reporter reporter)
+ throws IOException {
+
+ // normalize url, stop processing if null
+ String url = normalizeUrl(key.toString());
+ if (url == null) {
+ return;
+ }
+
+ if (value instanceof ParseData) {
+
+ // get the parse data and the outlinks from the parse data, along with
+ // the fetch time for those links
+ ParseData data = (ParseData)value;
+ long fetchTime = getFetchTime(data);
+ Outlink[] outlinkAr = data.getOutlinks();
+ Map<String, String> outlinkMap = new LinkedHashMap<String, String>();
+
+ // normalize urls and put into map
+ if (outlinkAr != null && outlinkAr.length > 0) {
+ for (int i = 0; i < outlinkAr.length; i++) {
+ Outlink outlink = outlinkAr[i];
+ String toUrl = normalizeUrl(outlink.getToUrl());
+
+ // only put into map if the url doesn't already exist in the map or
+ // if it does and the anchor for that link is null, will replace if
+ // url is existing
+ boolean existingUrl = outlinkMap.containsKey(toUrl);
+ if (toUrl != null
+ && (!existingUrl || (existingUrl && outlinkMap.get(toUrl) == null))) {
+ outlinkMap.put(toUrl, outlink.getAnchor());
+ }
+ }
+ }
+
+ // collect the outlinks under the fetch time
+ for (String outlinkUrl : outlinkMap.keySet()) {
+ String anchor = outlinkMap.get(outlinkUrl);
+ LinkDatum datum = new LinkDatum(outlinkUrl, anchor, fetchTime);
+ output.collect(key, datum);
+ }
+ }
+ else if (value instanceof LinkDatum) {
+
+ // collect existing outlinks from existing OutlinkDb
+ output.collect(key, (LinkDatum)value);
+ }
+ }
+
+ public void reduce(Text key, Iterator<LinkDatum> values,
+ OutputCollector<Text, LinkDatum> output, Reporter reporter)
+ throws IOException {
+
+ // aggregate all outlinks, get the most recent timestamp for a fetch
+ // which should be the timestamp for all of the most recent outlinks
+ long mostRecent = 0L;
+ List<LinkDatum> outlinkList = new ArrayList<LinkDatum>();
+ while (values.hasNext()) {
+
+ // loop through, change out most recent timestamp if needed
+ LinkDatum next = values.next();
+ long timestamp = next.getTimestamp();
+ if (mostRecent == 0L || mostRecent < timestamp) {
+ mostRecent = timestamp;
+ }
+ outlinkList.add((LinkDatum)WritableUtils.clone(next, conf));
+ }
+
+ // get the url, domain, and host for the url
+ String url = key.toString();
+ String domain = URLUtil.getDomainName(url);
+ String host = URLUtil.getHost(url);
+
+ // setup checking sets for domains and pages
+ Set<String> domains = new HashSet<String>();
+ Set<String> pages = new HashSet<String>();
+
+ // loop through the link datums
+ for (LinkDatum datum : outlinkList) {
+
+ // get the url, host, domain, and page for each outlink
+ String toUrl = datum.getUrl();
+ String toDomain = URLUtil.getDomainName(toUrl);
+ String toHost = URLUtil.getHost(toUrl);
+ String toPage = URLUtil.getPage(toUrl);
+ datum.setLinkType(LinkDatum.OUTLINK);
+
+ // outlinks must be the most recent and conform to internal url and
+ // limiting rules, if it does collect it
+ if (datum.getTimestamp() == mostRecent
+ && (!limitPages || (limitPages && !pages.contains(toPage)))
+ && (!limitDomains || (limitDomains && !domains.contains(toDomain)))
+ && (!ignoreHost || (ignoreHost && !toHost.equalsIgnoreCase(host)))
+ && (!ignoreDomain || (ignoreDomain && !toDomain.equalsIgnoreCase(domain)))) {
+ output.collect(key, datum);
+ pages.add(toPage);
+ domains.add(toDomain);
+ }
+ }
+ }
+
+ public void close() {
+ }
+ }
+
+ /**
+ * The InlinkDb creates a database of Inlinks. Inlinks are inverted from the
+ * OutlinkDb LinkDatum objects and are regenerated each time the WebGraph is
+ * updated.
+ */
+ private static class InlinkDb
+ extends Configured
+ implements Mapper<Text, LinkDatum, Text, LinkDatum> {
+
+ private JobConf conf;
+ private long timestamp;
+
+ /**
+ * Default constructor.
+ */
+ public InlinkDb() {
+ }
+
+ /**
+ * Configurable constructor.
+ */
+ public InlinkDb(Configuration conf) {
+ setConf(conf);
+ }
+
+ /**
+ * Configures job. Sets timestamp for all Inlink LinkDatum objects to the
+ * current system time.
+ */
+ public void configure(JobConf conf) {
+ this.conf = conf;
+ timestamp = System.currentTimeMillis();
+ }
+
+ public void close() {
+ }
+
+ /**
+ * Inverts the Outlink LinkDatum objects into new LinkDatum objects with a
+ * new system timestamp, type and to and from url switched.
+ */
+ public void map(Text key, LinkDatum datum,
+ OutputCollector<Text, LinkDatum> output, Reporter reporter)
+ throws IOException {
+
+ // get the to and from url and the anchor
+ String fromUrl = key.toString();
+ String toUrl = datum.getUrl();
+ String anchor = datum.getAnchor();
+
+ // flip the from and to url and set the new link type
+ LinkDatum inlink = new LinkDatum(fromUrl, anchor, timestamp);
+ inlink.setLinkType(LinkDatum.INLINK);
+ output.collect(new Text(toUrl), inlink);
+ }
+ }
+
+ /**
+ * Creates the Node database which consists of the number of in and outlinks
+ * for each url and a score slot for analysis programs such as LinkRank.
+ */
+ private static class NodeDb
+ extends Configured
+ implements Reducer<Text, LinkDatum, Text, Node> {
+
+ private JobConf conf;
+
+ /**
+ * Default constructor.
+ */
+ public NodeDb() {
+ }
+
+ /**
+ * Configurable constructor.
+ */
+ public NodeDb(Configuration conf) {
+ setConf(conf);
+ }
+
+ /**
+ * Configures job.
+ */
+ public void configure(JobConf conf) {
+ this.conf = conf;
+ }
+
+ public void close() {
+ }
+
+ /**
+ * Counts the number of inlinks and outlinks for each url and sets a default
+ * score of 0.0 for each url (node) in the webgraph.
+ */
+ public void reduce(Text key, Iterator<LinkDatum> values,
+ OutputCollector<Text, Node> output, Reporter reporter)
+ throws IOException {
+
+ Node node = new Node();
+ int numInlinks = 0;
+ int numOutlinks = 0;
+
+ // loop through counting number of in and out links
+ while (values.hasNext()) {
+ LinkDatum next = values.next();
+ if (next.getLinkType() == LinkDatum.INLINK) {
+ numInlinks++;
+ }
+ else if (next.getLinkType() == LinkDatum.OUTLINK) {
+ numOutlinks++;
+ }
+ }
+
+ // set the in and outlinks and a default score of 0
+ node.setNumInlinks(numInlinks);
+ node.setNumOutlinks(numOutlinks);
+ node.setInlinkScore(0.0f);
+ output.collect(key, node);
+ }
+ }
+
+ /**
+ * Creates the three different WebGraph databases, Outlinks, Inlinks, and
+ * Node. If a current WebGraph exists then it is updated, if it doesn't exist
+ * then a new WebGraph database is created.
+ *
+ * @param webGraphDb The WebGraph to create or update.
+ * @param segments The array of segments used to update the WebGraph. Newer
+ * segments and fetch times will overwrite older segments.
+ *
+ * @throws IOException If an error occurs while processing the WebGraph.
+ */
+ public void createWebGraph(Path webGraphDb, Path[] segments)
+ throws IOException {
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("WebGraphDb: starting");
+ LOG.info("WebGraphDb: webgraphdb: " + webGraphDb);
+ }
+
+ Configuration conf = getConf();
+ FileSystem fs = FileSystem.get(conf);
+
+ // lock an existing webgraphdb to prevent multiple simultaneous updates
+ Path lock = new Path(webGraphDb, LOCK_NAME);
+ boolean webGraphDbExists = fs.exists(webGraphDb);
+ if (webGraphDbExists) {
+ LockUtil.createLockFile(fs, lock, false);
+ }
+ else {
+
+ // if the webgraph doesn't exist, create it
+ fs.mkdirs(webGraphDb);
+ }
+
+ // outlink and temp outlink database paths
+ Path outlinkDb = new Path(webGraphDb, OUTLINK_DIR);
+ Path tempOutlinkDb = new Path(outlinkDb + "-"
+ + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
+ JobConf outlinkJob = new NutchJob(conf);
+ outlinkJob.setJobName("Outlinkdb: " + outlinkDb);
+
+ // get the parse data for all segments
+ if (segments != null) {
+ for (int i = 0; i < segments.length; i++) {
+ Path parseData = new Path(segments[i], ParseData.DIR_NAME);
+ if (fs.exists(parseData)) {
+ LOG.info("OutlinkDb: adding input: " + parseData);
+ FileInputFormat.addInputPath(outlinkJob, parseData);
+ }
+ }
+ }
+
+ // add the existing webgraph
+ if (webGraphDbExists) {
+ LOG.info("OutlinkDb: adding input: " + outlinkDb);
+ FileInputFormat.addInputPath(outlinkJob, outlinkDb);
+ }
+
+ outlinkJob.setInputFormat(SequenceFileInputFormat.class);
+ outlinkJob.setMapperClass(OutlinkDb.class);
+ outlinkJob.setReducerClass(OutlinkDb.class);
+ outlinkJob.setMapOutputKeyClass(Text.class);
+ outlinkJob.setMapOutputValueClass(LinkDatum.class);
+ outlinkJob.setOutputKeyClass(Text.class);
+ outlinkJob.setOutputValueClass(LinkDatum.class);
+ FileOutputFormat.setOutputPath(outlinkJob, tempOutlinkDb);
+ outlinkJob.setOutputFormat(MapFileOutputFormat.class);
+
+ // run the outlinkdb job and replace any old outlinkdb with the new one
+ try {
+ LOG.info("OutlinkDb: running");
+ JobClient.runJob(outlinkJob);
+ LOG.info("OutlinkDb: installing " + outlinkDb);
+ FSUtils.replace(fs, outlinkDb, tempOutlinkDb, true);
+ LOG.info("OutlinkDb: finished");
+ }
+ catch (IOException e) {
+
+ // remove lock file and and temporary directory if an error occurs
+ LockUtil.removeLockFile(fs, lock);
+ if (fs.exists(tempOutlinkDb)) {
+ fs.delete(tempOutlinkDb, true);
+ }
+ LOG.error(StringUtils.stringifyException(e));
+ throw e;
+ }
+
+ // inlink and temp link database paths
+ Path inlinkDb = new Path(webGraphDb, INLINK_DIR);
+ Path tempInlinkDb = new Path(inlinkDb + "-"
+ + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
+
+ JobConf inlinkJob = new NutchJob(conf);
+ inlinkJob.setJobName("Inlinkdb " + inlinkDb);
+ LOG.info("InlinkDb: adding input: " + outlinkDb);
+ FileInputFormat.addInputPath(inlinkJob, outlinkDb);
+ inlinkJob.setInputFormat(SequenceFileInputFormat.class);
+ inlinkJob.setMapperClass(InlinkDb.class);
+ inlinkJob.setMapOutputKeyClass(Text.class);
+ inlinkJob.setMapOutputValueClass(LinkDatum.class);
+ inlinkJob.setOutputKeyClass(Text.class);
+ inlinkJob.setOutputValueClass(LinkDatum.class);
+ FileOutputFormat.setOutputPath(inlinkJob, tempInlinkDb);
+ inlinkJob.setOutputFormat(MapFileOutputFormat.class);
+
+ try {
+
+ // run the inlink and replace any old with new
+ LOG.info("InlinkDb: running");
+ JobClient.runJob(inlinkJob);
+ LOG.info("InlinkDb: installing " + inlinkDb);
+ FSUtils.replace(fs, inlinkDb, tempInlinkDb, true);
+ LOG.info("InlinkDb: finished");
+ }
+ catch (IOException e) {
+
+ // remove lock file and and temporary directory if an error occurs
+ LockUtil.removeLockFile(fs, lock);
+ if (fs.exists(tempInlinkDb)) {
+ fs.delete(tempInlinkDb, true);
+ }
+ LOG.error(StringUtils.stringifyException(e));
+ throw e;
+ }
+
+ // node and temp node database paths
+ Path nodeDb = new Path(webGraphDb, NODE_DIR);
+ Path tempNodeDb = new Path(nodeDb + "-"
+ + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
+
+ JobConf nodeJob = new NutchJob(conf);
+ nodeJob.setJobName("NodeDb " + nodeDb);
+ LOG.info("NodeDb: adding input: " + outlinkDb);
+ LOG.info("NodeDb: adding input: " + inlinkDb);
+ FileInputFormat.addInputPath(nodeJob, outlinkDb);
+ FileInputFormat.addInputPath(nodeJob, inlinkDb);
+ nodeJob.setInputFormat(SequenceFileInputFormat.class);
+ nodeJob.setReducerClass(NodeDb.class);
+ nodeJob.setMapOutputKeyClass(Text.class);
+ nodeJob.setMapOutputValueClass(LinkDatum.class);
+ nodeJob.setOutputKeyClass(Text.class);
+ nodeJob.setOutputValueClass(Node.class);
+ FileOutputFormat.setOutputPath(nodeJob, tempNodeDb);
+ nodeJob.setOutputFormat(MapFileOutputFormat.class);
+
+ try {
+
+ // run the node job and replace old nodedb with new
+ LOG.info("NodeDb: running");
+ JobClient.runJob(nodeJob);
+ LOG.info("NodeDb: installing " + nodeDb);
+ FSUtils.replace(fs, nodeDb, tempNodeDb, true);
+ LOG.info("NodeDb: finished");
+ }
+ catch (IOException e) {
+
+ // remove lock file and and temporary directory if an error occurs
+ LockUtil.removeLockFile(fs, lock);
+ if (fs.exists(tempNodeDb)) {
+ fs.delete(tempNodeDb, true);
+ }
+ LOG.error(StringUtils.stringifyException(e));
+ throw e;
+ }
+
+ // remove the lock file for the webgraph
+ LockUtil.removeLockFile(fs, lock);
+ }
+
+ public static void main(String[] args)
+ throws Exception {
+ int res = ToolRunner.run(NutchConfiguration.create(), new WebGraph(), args);
+ System.exit(res);
+ }
+
+ /**
+ * Parses command link arguments and runs the WebGraph jobs.
+ */
+ public int run(String[] args)
+ throws Exception {
+
+ Options options = new Options();
+ Option helpOpts = OptionBuilder.withArgName("help").withDescription(
+ "show this help message").create("help");
+ Option webGraphDbOpts = OptionBuilder.withArgName("webgraphdb").hasArg().withDescription(
+ "the web graph database to use").create("webgraphdb");
+ Option segOpts = OptionBuilder.withArgName("segment").hasArgs().withDescription(
+ "the segment(s) to use").create("segment");
+ options.addOption(helpOpts);
+ options.addOption(webGraphDbOpts);
+ options.addOption(segOpts);
+
+ CommandLineParser parser = new GnuParser();
+ try {
+
+ CommandLine line = parser.parse(options, args);
+ if (line.hasOption("help") || !line.hasOption("webgraphdb")
+ || !line.hasOption("segment")) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("WebGraph", options);
+ return -1;
+ }
+
+ String webGraphDb = line.getOptionValue("webgraphdb");
+ String[] segments = line.getOptionValues("segment");
+ Path[] segPaths = new Path[segments.length];
+ for (int i = 0; i < segments.length; i++) {
+ segPaths[i] = new Path(segments[i]);
+ }
+
+ createWebGraph(new Path(webGraphDb), segPaths);
+ return 0;
+ }
+ catch (Exception e) {
+ LOG.fatal("WebGraph: " + StringUtils.stringifyException(e));
+ return -2;
+ }
+ }
+
+}
Added: lucene/nutch/trunk/src/java/org/apache/nutch/tools/compat/ReprUrlFixer.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/tools/compat/ReprUrlFixer.java?rev=723441&view=auto
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/tools/compat/ReprUrlFixer.java (added)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/tools/compat/ReprUrlFixer.java Thu Dec 4 13:16:42 2008
@@ -0,0 +1,296 @@
+package org.apache.nutch.tools.compat;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapFileOutputFormat;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.CrawlDb;
+import org.apache.nutch.crawl.MapWritable;
+import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.scoring.webgraph.Node;
+import org.apache.nutch.util.FSUtils;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.NutchJob;
+import org.apache.nutch.util.URLUtil;
+
+/**
+ * <p>
+ * Significant changes were made to representative url logic used for redirects.
+ * This tool will fix representative urls stored in current segments and crawl
+ * databases. Any new fetches will use the new representative url logic.
+ * </p>
+ *
+ * <p>
+ * All crawl datums are assumed to be temp url redirects. While this may cause
+ * some urls to be incorrectly removed, this tool is a temporary measure to be
+ * used until fetches can be rerun. This reduce logic is the same for segments
+ * fetch and parse directory as well as for existing crawl databases.
+ * </p>
+ */
+public class ReprUrlFixer
+ extends Configured
+ implements Tool, Reducer<Text, CrawlDatum, Text, CrawlDatum> {
+
+ public static final Log LOG = LogFactory.getLog(ReprUrlFixer.class);
+ private JobConf conf;
+
+ public void configure(JobConf conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * Runs the new ReprUrl logic on all crawldatums.
+ */
+ public void reduce(Text key, Iterator<CrawlDatum> values,
+ OutputCollector<Text, CrawlDatum> output, Reporter reporter)
+ throws IOException {
+
+ String url = key.toString();
+ Node node = null;
+ List<CrawlDatum> datums = new ArrayList<CrawlDatum>();
+
+ // get all crawl datums for a given url key, fetch for instance can have
+ // more than one under a given key if there are multiple redirects to a
+ // given url
+ while (values.hasNext()) {
+ CrawlDatum datum = values.next();
+ datums.add((CrawlDatum)WritableUtils.clone(datum, conf));
+ }
+
+ // apply redirect repr url logic for each datum
+ for (CrawlDatum datum : datums) {
+
+ MapWritable metadata = datum.getMetaData();
+ Text reprUrl = (Text)metadata.get(Nutch.WRITABLE_REPR_URL_KEY);
+ byte status = datum.getStatus();
+ boolean isCrawlDb = (CrawlDatum.hasDbStatus(datum));
+ boolean segFetched = (status == CrawlDatum.STATUS_FETCH_SUCCESS);
+
+ // only if the crawl datum is from the crawldb or is a successfully
+ // fetched page from the segments
+ if ((isCrawlDb || segFetched) && reprUrl != null) {
+
+ String src = reprUrl.toString();
+ String dest = url;
+ URL srcUrl = null;
+ URL dstUrl = null;
+
+ // both need to be well formed urls
+ try {
+ srcUrl = new URL(src);
+ dstUrl = new URL(url);
+ }
+ catch (MalformedURLException e) {
+ }
+
+ // if the src and repr urls are the same after the new logic then
+ // remove the repr url from the metadata as it is no longer needed
+ if (srcUrl != null && dstUrl != null) {
+ String reprOut = URLUtil.chooseRepr(src, dest, true);
+ if (reprOut.equals(dest)) {
+ LOG.info("Removing " + reprOut + " from " + dest);
+ metadata.remove(Nutch.WRITABLE_REPR_URL_KEY);
+ }
+ }
+ }
+
+ // collect each datum
+ output.collect(key, datum);
+ }
+
+ }
+
+ public void close() {
+ }
+
+ /**
+ * Run the fixer on any crawl database and segments specified.
+ */
+ public void update(Path crawlDb, Path[] segments)
+ throws IOException {
+
+ Configuration conf = getConf();
+ FileSystem fs = FileSystem.get(conf);
+
+ // run the crawl database through the repr fixer
+ if (crawlDb != null) {
+
+ LOG.info("Running ReprUtilFixer " + crawlDb);
+ Path crawlDbCurrent = new Path(crawlDb, CrawlDb.CURRENT_NAME);
+ Path newCrawlDb = new Path(crawlDb,
+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
+
+ JobConf updater = new NutchJob(conf);
+ updater.setJobName("ReprUtilFixer: " + crawlDb.toString());
+ FileInputFormat.addInputPath(updater, crawlDbCurrent);
+ FileOutputFormat.setOutputPath(updater, newCrawlDb);
+ updater.setInputFormat(SequenceFileInputFormat.class);
+ updater.setReducerClass(ReprUrlFixer.class);
+ updater.setOutputKeyClass(Text.class);
+ updater.setOutputValueClass(CrawlDatum.class);
+ updater.setOutputFormat(MapFileOutputFormat.class);
+
+ try {
+ JobClient.runJob(updater);
+ LOG.info("Installing new crawldb " + crawlDb);
+ CrawlDb.install(updater, crawlDb);
+ }
+ catch (IOException e) {
+ LOG.error(StringUtils.stringifyException(e));
+ throw e;
+ }
+ }
+
+ // run the segments through the repr fixer, logic will be run on both the
+ // crawl_parse and the crawl_fetch directories for every segment specified
+ if (segments != null) {
+
+ for (int i = 0; i < segments.length; i++) {
+
+ Path segment = segments[i];
+ LOG.info("Running ReprUtilFixer " + segment + " fetch");
+ Path segFetch = new Path(segment, CrawlDatum.FETCH_DIR_NAME);
+ Path newSegFetch = new Path(segment, CrawlDatum.FETCH_DIR_NAME + "-"
+ + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
+
+ JobConf fetch = new NutchJob(conf);
+ fetch.setJobName("ReprUtilFixer: " + segment.toString());
+ FileInputFormat.addInputPath(fetch, segFetch);
+ FileOutputFormat.setOutputPath(fetch, newSegFetch);
+ fetch.setInputFormat(SequenceFileInputFormat.class);
+ fetch.setReducerClass(ReprUrlFixer.class);
+ fetch.setOutputKeyClass(Text.class);
+ fetch.setOutputValueClass(CrawlDatum.class);
+ fetch.setOutputFormat(MapFileOutputFormat.class);
+
+ try {
+ JobClient.runJob(fetch);
+ LOG.info("Installing new segment fetch directory " + newSegFetch);
+ FSUtils.replace(fs, segFetch, newSegFetch, true);
+ LOG.info("ReprUrlFixer: finished installing segment fetch directory");
+ }
+ catch (IOException e) {
+ LOG.error(StringUtils.stringifyException(e));
+ throw e;
+ }
+
+ LOG.info("Running ReprUtilFixer " + segment + " parse");
+ Path segParse = new Path(segment, CrawlDatum.PARSE_DIR_NAME);
+ Path newSegParse = new Path(segment, CrawlDatum.PARSE_DIR_NAME + "-"
+ + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
+
+ JobConf parse = new NutchJob(conf);
+ parse.setJobName("ReprUtilFixer: " + segment.toString());
+ FileInputFormat.addInputPath(parse, segParse);
+ FileOutputFormat.setOutputPath(parse, newSegParse);
+ parse.setInputFormat(SequenceFileInputFormat.class);
+ parse.setReducerClass(ReprUrlFixer.class);
+ parse.setOutputKeyClass(Text.class);
+ parse.setOutputValueClass(CrawlDatum.class);
+ parse.setOutputFormat(MapFileOutputFormat.class);
+
+ try {
+ JobClient.runJob(parse);
+ LOG.info("Installing new segment parse directry " + newSegParse);
+ FSUtils.replace(fs, segParse, newSegParse, true);
+ LOG.info("ReprUrlFixer: finished installing segment parse directory");
+ }
+ catch (IOException e) {
+ LOG.error(StringUtils.stringifyException(e));
+ throw e;
+ }
+ }
+ }
+ }
+
+ /**
+ * Runs The ReprUrlFixer.
+ */
+ public static void main(String[] args)
+ throws Exception {
+ int res = ToolRunner.run(NutchConfiguration.create(), new ReprUrlFixer(),
+ args);
+ System.exit(res);
+ }
+
+ /**
+ * Parse command line options and execute the main update logic.
+ */
+ public int run(String[] args)
+ throws Exception {
+
+ Options options = new Options();
+ Option helpOpts = OptionBuilder.withArgName("help").withDescription(
+ "show this help message").create("help");
+ Option crawlDbOpts = OptionBuilder.withArgName("crawldb").hasArg().withDescription(
+ "the crawldb to use").create("crawldb");
+ Option segOpts = OptionBuilder.withArgName("segment").hasArgs().withDescription(
+ "the segment(s) to use").create("segment");
+ options.addOption(helpOpts);
+ options.addOption(crawlDbOpts);
+ options.addOption(segOpts);
+
+ CommandLineParser parser = new GnuParser();
+ try {
+
+ // parse out common line arguments and make sure either a crawldb or a
+ // segment are specified
+ CommandLine line = parser.parse(options, args);
+ if (line.hasOption("help")
+ || (!line.hasOption("crawldb") && !line.hasOption("segment"))) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("ReprUtilFixer", options);
+ return -1;
+ }
+
+ // create paths for all of the segments specified, multiple segments may
+ // be run at once
+ String crawlDb = line.getOptionValue("crawldb");
+ String[] segments = line.getOptionValues("segment");
+ Path[] segPaths = new Path[segments != null ? segments.length : 0];
+ if (segments != null) {
+ for (int i = 0; i < segments.length; i++) {
+ segPaths[i] = new Path(segments[i]);
+ }
+ }
+ update(new Path(crawlDb), segPaths);
+ return 0;
+ }
+ catch (Exception e) {
+ LOG.fatal("ReprUtilFixer: " + StringUtils.stringifyException(e));
+ return -1;
+ }
+ }
+}
\ No newline at end of file
Added: lucene/nutch/trunk/src/java/org/apache/nutch/util/FSUtils.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/util/FSUtils.java?rev=723441&view=auto
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/util/FSUtils.java (added)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/util/FSUtils.java Thu Dec 4 13:16:42 2008
@@ -0,0 +1,83 @@
+package org.apache.nutch.util;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.SequenceFile;
+
+/**
+ * Utility methods for common filesystem operations.
+ */
+public class FSUtils {
+
+ /**
+ * Replaces the current path with the new path and if set removes the old
+ * path. If removeOld is set to false then the old path will be set to the
+ * name current.old.
+ *
+ * @param fs The FileSystem.
+ * @param current The end path, the one being replaced.
+ * @param replacement The path to replace with.
+ * @param removeOld True if we are removing the current path.
+ *
+ * @throws IOException If an error occurs during replacement.
+ */
+ public static void replace(FileSystem fs, Path current, Path replacement,
+ boolean removeOld)
+ throws IOException {
+
+ // rename any current path to old
+ Path old = new Path(current + ".old");
+ if (fs.exists(current)) {
+ fs.rename(current, old);
+ }
+
+ // rename the new path to current and remove the old path if needed
+ fs.rename(replacement, current);
+ if (fs.exists(old) && removeOld) {
+ fs.delete(old, true);
+ }
+ }
+
+ /**
+ * Closes a group of SequenceFile readers.
+ *
+ * @param readers The SequenceFile readers to close.
+ * @throws IOException If an error occurs while closing a reader.
+ */
+ public static void closeReaders(SequenceFile.Reader[] readers)
+ throws IOException {
+
+ // loop through the readers, closing one by one
+ if (readers != null) {
+ for (int i = 0; i < readers.length; i++) {
+ SequenceFile.Reader reader = readers[i];
+ if (reader != null) {
+ reader.close();
+ }
+ }
+ }
+ }
+
+ /**
+ * Closes a group of MapFile readers.
+ *
+ * @param readers The MapFile readers to close.
+ * @throws IOException If an error occurs while closing a reader.
+ */
+ public static void closeReaders(MapFile.Reader[] readers)
+ throws IOException {
+
+ // loop through the readers closing one by one
+ if (readers != null) {
+ for (int i = 0; i < readers.length; i++) {
+ MapFile.Reader reader = readers[i];
+ if (reader != null) {
+ reader.close();
+ }
+ }
+ }
+ }
+}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/util/URLUtil.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/util/URLUtil.java?rev=723441&r1=723440&r2=723441&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/util/URLUtil.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/util/URLUtil.java Thu Dec 4 13:16:42 2008
@@ -141,57 +141,160 @@
return getHostSegments(new URL(url));
}
- /** Given two urls (source and destination of the redirect),
- * returns the representative one.
- *
- * <p>Implements the algorithm described here:
+ /**
+ * <p>Given two urls, a src and a destination of a redirect, it returns the
+ * representative url.<p>
+ *
+ * <p>This method implements an extended version of the algorithm used by the
+ * Yahoo! Slurp crawler described here:<br>
+ * <a href=
+ * "http://help.yahoo.com/l/nz/yahooxtra/search/webcrawler/slurp-11.html"> How
+ * does the Yahoo! webcrawler handle redirects?</a> <br>
* <br>
- * <a href="http://help.yahoo.com/l/nz/yahooxtra/search/webcrawler/slurp-11.html">
- * How does the Yahoo! webcrawler handle redirects?</a>
- * <br><br>
- * The algorithm is as follows:
* <ol>
- * <li>Choose target url if either url is malformed.</li>
- * <li>When a page in one domain redirects to a page in another domain,
- * choose the "target" URL.</li>
- * <li>When a top-level page in a domain presents a permanent redirect
- * to a page deep within the same domain, choose the "source" URL.</li>
- * <li>When a page deep within a domain presents a permanent redirect
- * to a page deep within the same domain, choose the "target" URL.</li>
- * <li>When a page in a domain presents a temporary redirect to
- * another page in the same domain, choose the "source" URL.<li>
- * <ol>
- * </p>
- *
- * @param src Source url of redirect
- * @param dst Destination url of redirect
- * @param temp Flag to indicate if redirect is temporary
- * @return Representative url (either src or dst)
+ * <li>Choose target url if either url is malformed.</li>
+ * <li>If different domains the keep the destination whether or not the
+ * redirect is temp or perm</li>
+ * <ul><li>a.com -> b.com*</li></ul>
+ * <li>If the redirect is permanent and the source is root, keep the source.</li>
+ * <ul><li>*a.com -> a.com?y=1 || *a.com -> a.com/xyz/index.html</li></ul>
+ * <li>If the redirect is permanent and the source is not root and the
+ * destination is root, keep the destination</li>
+ * <ul><li>a.com/xyz/index.html -> a.com*</li></ul>
+ * <li>If the redirect is permanent and neither the source nor the destination
+ * is root, then keep the destination</li>
+ * <ul><li>a.com/xyz/index.html -> a.com/abc/page.html*</li></ul>
+ * <li>If the redirect is temporary and source is root and destination is not
+ * root, then keep the source</li>
+ * <ul><li>*a.com -> a.com/xyz/index.html</li></ul>
+ * <li>If the redirect is temporary and source is not root and destination is
+ * root, then keep the destination</li>
+ * <ul><li>a.com/xyz/index.html -> a.com*</li></ul>
+ * <li>If the redirect is temporary and neither the source or the destination
+ * is root, then keep the shortest url. First check for the shortest host,
+ * and if both are equal then check by path. Path is first by length then by
+ * the number of / path separators.</li>
+ * <ul>
+ * <li>a.com/xyz/index.html -> a.com/abc/page.html*</li>
+ * <li>*www.a.com/xyz/index.html -> www.news.a.com/xyz/index.html</li>
+ * </ul>
+ * <li>If the redirect is temporary and both the source and the destination
+ * are root, then keep the shortest sub-domain</li>
+ * <ul><li>*www.a.com -> www.news.a.com</li></ul>
+ * <br>
+ * While not in this logic there is a further piece of representative url
+ * logic that occurs during indexing and after scoring. During creation of
+ * the basic fields before indexing, if a url has a representative url stored
+ * we check both the url and its representative url (which should never be
+ * the same) against their linkrank scores and the highest scoring one is
+ * kept as the url and the lower scoring one is held as the orig url inside
+ * of the index.
+ *
+ * @param src The source url.
+ * @param dst The destination url.
+ * @param temp Is the redirect a temporary redirect.
+ *
+ * @return String The representative url.
*/
public static String chooseRepr(String src, String dst, boolean temp) {
+
+ // validate both are well formed urls
URL srcUrl;
URL dstUrl;
try {
srcUrl = new URL(src);
dstUrl = new URL(dst);
- } catch (MalformedURLException e) {
+ }
+ catch (MalformedURLException e) {
return dst;
}
+ // get the source and destination domain, host, and page
String srcDomain = URLUtil.getDomainName(srcUrl);
String dstDomain = URLUtil.getDomainName(dstUrl);
+ String srcHost = srcUrl.getHost();
+ String dstHost = dstUrl.getHost();
+ String srcFile = srcUrl.getFile();
+ String dstFile = dstUrl.getFile();
+
+ // are the source and destination the root path url.com/ or url.com
+ boolean srcRoot = (srcFile.equals("/") || srcFile.length() == 0);
+ boolean destRoot = (dstFile.equals("/") || dstFile.length() == 0);
+
+ // 1) different domain them keep dest, temp or perm
+ // a.com -> b.com*
+ //
+ // 2) permanent and root, keep src
+ // *a.com -> a.com?y=1 || *a.com -> a.com/xyz/index.html
+ //
+ // 3) permanent and not root and dest root, keep dest
+ // a.com/xyz/index.html -> a.com*
+ //
+ // 4) permanent and neither root keep dest
+ // a.com/xyz/index.html -> a.com/abc/page.html*
+ //
+ // 5) temp and root and dest not root keep src
+ // *a.com -> a.com/xyz/index.html
+ //
+ // 7) temp and not root and dest root keep dest
+ // a.com/xyz/index.html -> a.com*
+ //
+ // 8) temp and neither root, keep shortest, if hosts equal by path else by
+ // hosts. paths are first by length then by number of / separators
+ // a.com/xyz/index.html -> a.com/abc/page.html*
+ // *www.a.com/xyz/index.html -> www.news.a.com/xyz/index.html
+ //
+ // 9) temp and both root keep shortest sub domain
+ // *www.a.com -> www.news.a.com
+ // if we are dealing with a redirect from one domain to another keep the
+ // destination
if (!srcDomain.equals(dstDomain)) {
return dst;
}
- String srcFile = srcUrl.getFile();
-
- if (!temp && srcFile.equals("/")) {
- return src;
+ // if it is a permanent redirect
+ if (!temp) {
+
+ // if source is root return source, otherwise destination
+ if (srcRoot) {
+ return src;
+ }
+ else {
+ return dst;
+ }
+ }
+ else { // temporary redirect
+
+ // source root and destination not root
+ if (srcRoot && !destRoot) {
+ return src;
+ }
+ else if (!srcRoot && destRoot) { // destination root and source not
+ return dst;
+ }
+ else if (!srcRoot && !destRoot && (srcHost.equals(dstHost))) {
+
+ // source and destination hosts are the same, check paths, host length
+ int numSrcPaths = srcFile.split("/").length;
+ int numDstPaths = dstFile.split("/").length;
+ if (numSrcPaths != numDstPaths) {
+ return (numDstPaths < numSrcPaths ? dst : src);
+ }
+ else {
+ int srcPathLength = srcFile.length();
+ int dstPathLength = dstFile.length();
+ return (dstPathLength < srcPathLength ? dst : src);
+ }
+ }
+ else {
+
+ // different host names and both root take the shortest
+ int numSrcSubs = srcHost.split("\\.").length;
+ int numDstSubs = dstHost.split("\\.").length;
+ return (numDstSubs < numSrcSubs ? dst : src);
+ }
}
-
- return temp ? src : dst;
}
/**
Modified: lucene/nutch/trunk/src/plugin/build.xml
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/build.xml?rev=723441&r1=723440&r2=723441&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/build.xml (original)
+++ lucene/nutch/trunk/src/plugin/build.xml Thu Dec 4 13:16:42 2008
@@ -66,6 +66,7 @@
<ant dir="query-site" target="deploy"/>
<ant dir="query-url" target="deploy"/>
<ant dir="scoring-opic" target="deploy"/>
+ <ant dir="scoring-link" target="deploy"/>
<ant dir="summary-basic" target="deploy"/>
<ant dir="subcollection" target="deploy"/>
<ant dir="summary-lucene" target="deploy"/>
@@ -160,6 +161,7 @@
<ant dir="query-site" target="clean"/>
<ant dir="query-url" target="clean"/>
<ant dir="scoring-opic" target="clean"/>
+ <ant dir="scoring-link" target="clean"/>
<ant dir="subcollection" target="clean"/>
<ant dir="summary-basic" target="clean"/>
<ant dir="summary-lucene" target="clean"/>
Added: lucene/nutch/trunk/src/plugin/scoring-link/build.xml
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/scoring-link/build.xml?rev=723441&view=auto
==============================================================================
--- lucene/nutch/trunk/src/plugin/scoring-link/build.xml (added)
+++ lucene/nutch/trunk/src/plugin/scoring-link/build.xml Thu Dec 4 13:16:42 2008
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project name="scoring-link" default="jar-core">
+
+ <import file="../build-plugin.xml"/>
+
+ <!-- Deploy Unit test dependencies -->
+ <target name="deps-test">
+ <ant target="deploy" inheritall="false" dir="../nutch-extensionpoints"/>
+ </target>
+
+</project>
Added: lucene/nutch/trunk/src/plugin/scoring-link/plugin.xml
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/scoring-link/plugin.xml?rev=723441&view=auto
==============================================================================
--- lucene/nutch/trunk/src/plugin/scoring-link/plugin.xml (added)
+++ lucene/nutch/trunk/src/plugin/scoring-link/plugin.xml Thu Dec 4 13:16:42 2008
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<plugin
+ id="scoring-link"
+ name="Link Analysis Scoring Plug-in"
+ version="1.0.0"
+ provider-name="nutch.org">
+
+
+ <runtime>
+ <library name="scoring-link.jar">
+ <export name="*"/>
+ </library>
+ </runtime>
+
+ <extension id="org.apache.nutch.scoring.link"
+ name="LinkAnalysisScoring"
+ point="org.apache.nutch.scoring.ScoringFilter">
+
+ <implementation id="org.apache.nutch.scoring.link.LinkAnalysisScoringFilter"
+ class="org.apache.nutch.scoring.link.LinkAnalysisScoringFilter" />
+ </extension>
+
+</plugin>
Added: lucene/nutch/trunk/src/plugin/scoring-link/src/java/org/apache/nutch/scoring/link/LinkAnalysisScoringFilter.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/scoring-link/src/java/org/apache/nutch/scoring/link/LinkAnalysisScoringFilter.java?rev=723441&view=auto
==============================================================================
--- lucene/nutch/trunk/src/plugin/scoring-link/src/java/org/apache/nutch/scoring/link/LinkAnalysisScoringFilter.java (added)
+++ lucene/nutch/trunk/src/plugin/scoring-link/src/java/org/apache/nutch/scoring/link/LinkAnalysisScoringFilter.java Thu Dec 4 13:16:42 2008
@@ -0,0 +1,85 @@
+package org.apache.nutch.scoring.link;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.lucene.document.Document;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.Inlinks;
+import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.parse.ParseData;
+import org.apache.nutch.protocol.Content;
+import org.apache.nutch.scoring.ScoringFilter;
+import org.apache.nutch.scoring.ScoringFilterException;
+
+public class LinkAnalysisScoringFilter
+ implements ScoringFilter {
+
+ private Configuration conf;
+ private float scoreInjected = 0.001f;
+ private float normalizedScore = 1.00f;
+
+ public LinkAnalysisScoringFilter() {
+
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ normalizedScore = conf.getFloat("link.analyze.normalize.score", 1.00f);
+ scoreInjected = conf.getFloat("link.analyze.injected.score", 1.00f);
+ }
+
+ public CrawlDatum distributeScoreToOutlinks(Text fromUrl,
+ ParseData parseData, Collection<Entry<Text, CrawlDatum>> targets,
+ CrawlDatum adjust, int allCount)
+ throws ScoringFilterException {
+ return adjust;
+ }
+
+ public float generatorSortValue(Text url, CrawlDatum datum, float initSort)
+ throws ScoringFilterException {
+ return datum.getScore() * initSort;
+ }
+
+ public float indexerScore(Text url, Document doc, CrawlDatum dbDatum,
+ CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore)
+ throws ScoringFilterException {
+ return (normalizedScore * dbDatum.getScore());
+ }
+
+ public void initialScore(Text url, CrawlDatum datum)
+ throws ScoringFilterException {
+ datum.setScore(0.0f);
+ }
+
+ public void injectedScore(Text url, CrawlDatum datum)
+ throws ScoringFilterException {
+ datum.setScore(scoreInjected);
+ }
+
+ public void passScoreAfterParsing(Text url, Content content, Parse parse)
+ throws ScoringFilterException {
+ parse.getData().getContentMeta().set(Nutch.SCORE_KEY,
+ content.getMetadata().get(Nutch.SCORE_KEY));
+ }
+
+ public void passScoreBeforeParsing(Text url, CrawlDatum datum, Content content)
+ throws ScoringFilterException {
+ content.getMetadata().set(Nutch.SCORE_KEY, "" + datum.getScore());
+ }
+
+ public void updateDbScore(Text url, CrawlDatum old, CrawlDatum datum,
+ List<CrawlDatum> inlinked)
+ throws ScoringFilterException {
+ // nothing to do
+ }
+
+}
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/util/TestURLUtil.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/test/org/apache/nutch/util/TestURLUtil.java?rev=723441&r1=723440&r2=723441&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/util/TestURLUtil.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/util/TestURLUtil.java Thu Dec 4 13:16:42 2008
@@ -22,14 +22,17 @@
import junit.framework.TestCase;
/** Test class for URLUtil */
-public class TestURLUtil extends TestCase {
+public class TestURLUtil
+ extends TestCase {
@Override
- protected void setUp() throws Exception {
+ protected void setUp()
+ throws Exception {
super.setUp();
}
- public void testGetDomainName() throws Exception{
+ public void testGetDomainName()
+ throws Exception {
URL url = null;
@@ -51,7 +54,7 @@
url = new URL("http://www.example.co.uk.com");
assertEquals("uk.com", URLUtil.getDomainName(url));
- //"nn" is not a tld
+ // "nn" is not a tld
url = new URL("http://example.com.nn");
assertEquals("nn", URLUtil.getDomainName(url));
@@ -60,25 +63,26 @@
url = new URL("http://www.edu.tr.xyz");
assertEquals("xyz", URLUtil.getDomainName(url));
-
+
url = new URL("http://www.example.c.se");
assertEquals("example.c.se", URLUtil.getDomainName(url));
- //plc.co.im is listed as a domain suffix
+ // plc.co.im is listed as a domain suffix
url = new URL("http://www.example.plc.co.im");
assertEquals("example.plc.co.im", URLUtil.getDomainName(url));
-
- //2000.hu is listed as a domain suffix
+
+ // 2000.hu is listed as a domain suffix
url = new URL("http://www.example.2000.hu");
assertEquals("example.2000.hu", URLUtil.getDomainName(url));
-
- //test non-ascii
+
+ // test non-ascii
url = new URL("http://www.example.åæ¥.tw");
assertEquals("example.åæ¥.tw", URLUtil.getDomainName(url));
-
+
}
- public void testGetDomainSuffix() throws Exception{
+ public void testGetDomainSuffix()
+ throws Exception {
URL url = null;
url = new URL("http://lucene.apache.org/nutch");
@@ -96,7 +100,7 @@
url = new URL("http://www.example.co.uk.com");
assertEquals("com", URLUtil.getDomainSuffix(url).getDomain());
- //"nn" is not a tld
+ // "nn" is not a tld
url = new URL("http://example.com.nn");
assertNull(URLUtil.getDomainSuffix(url));
@@ -105,59 +109,108 @@
url = new URL("http://www.edu.tr.xyz");
assertNull(URLUtil.getDomainSuffix(url));
-
+
url = new URL("http://subdomain.example.edu.tr");
assertEquals("edu.tr", URLUtil.getDomainSuffix(url).getDomain());
-
+
url = new URL("http://subdomain.example.presse.fr");
assertEquals("presse.fr", URLUtil.getDomainSuffix(url).getDomain());
-
+
url = new URL("http://subdomain.example.presse.tr");
assertEquals("tr", URLUtil.getDomainSuffix(url).getDomain());
-
- //plc.co.im is listed as a domain suffix
+
+ // plc.co.im is listed as a domain suffix
url = new URL("http://www.example.plc.co.im");
assertEquals("plc.co.im", URLUtil.getDomainSuffix(url).getDomain());
-
- //2000.hu is listed as a domain suffix
+
+ // 2000.hu is listed as a domain suffix
url = new URL("http://www.example.2000.hu");
assertEquals("2000.hu", URLUtil.getDomainSuffix(url).getDomain());
-
- //test non-ascii
+
+ // test non-ascii
url = new URL("http://www.example.åæ¥.tw");
assertEquals("åæ¥.tw", URLUtil.getDomainSuffix(url).getDomain());
-
+
}
-
- public void testGetHostSegments() throws Exception{
+
+ public void testGetHostSegments()
+ throws Exception {
URL url;
String[] segments;
-
+
url = new URL("http://subdomain.example.edu.tr");
segments = URLUtil.getHostSegments(url);
assertEquals("subdomain", segments[0]);
assertEquals("example", segments[1]);
assertEquals("edu", segments[2]);
assertEquals("tr", segments[3]);
-
+
url = new URL("http://");
segments = URLUtil.getHostSegments(url);
assertEquals(1, segments.length);
assertEquals("", segments[0]);
-
+
url = new URL("http://140.211.11.130/foundation/contributing.html");
segments = URLUtil.getHostSegments(url);
assertEquals(1, segments.length);
assertEquals("140.211.11.130", segments[0]);
-
- //test non-ascii
+
+ // test non-ascii
url = new URL("http://www.example.åæ¥.tw");
segments = URLUtil.getHostSegments(url);
assertEquals("www", segments[0]);
assertEquals("example", segments[1]);
assertEquals("åæ¥", segments[2]);
assertEquals("tw", segments[3]);
+
+ }
+
+ public void testChooseRepr()
+ throws Exception {
+ String aDotCom = "http://www.a.com";
+ String bDotCom = "http://www.b.com";
+ String aSubDotCom = "http://www.news.a.com";
+ String aQStr = "http://www.a.com?y=1";
+ String aPath = "http://www.a.com/xyz/index.html";
+ String aPath2 = "http://www.a.com/abc/page.html";
+ String aPath3 = "http://www.news.a.com/abc/page.html";
+
+ // 1) different domain them keep dest, temp or perm
+ // a.com -> b.com*
+ assertEquals(bDotCom, URLUtil.chooseRepr(aDotCom, bDotCom, true));
+ assertEquals(bDotCom, URLUtil.chooseRepr(aDotCom, bDotCom, false));
+
+ // 2) permanent and root, keep src
+ // *a.com -> a.com?y=1 || *a.com -> a.com/xyz/index.html
+ assertEquals(aDotCom, URLUtil.chooseRepr(aDotCom, aQStr, false));
+ assertEquals(aDotCom, URLUtil.chooseRepr(aDotCom, aPath, false));
+
+ //3) permanent and not root and dest root, keep dest
+ //a.com/xyz/index.html -> a.com*
+ assertEquals(aDotCom, URLUtil.chooseRepr(aPath, aDotCom, false));
+
+ //4) permanent and neither root keep dest
+ // a.com/xyz/index.html -> a.com/abc/page.html*
+ assertEquals(aPath2, URLUtil.chooseRepr(aPath, aPath2, false));
+
+ //5) temp and root and dest not root keep src
+ //*a.com -> a.com/xyz/index.html
+ assertEquals(aDotCom, URLUtil.chooseRepr(aDotCom, aPath, true));
+
+ //6) temp and not root and dest root keep dest
+ // a.com/xyz/index.html -> a.com*
+ assertEquals(aDotCom, URLUtil.chooseRepr(aPath, aDotCom, true));
+
+ //7) temp and neither root, keep shortest, if hosts equal by path else by hosts
+ // a.com/xyz/index.html -> a.com/abc/page.html*
+ // *www.a.com/xyz/index.html -> www.news.a.com/xyz/index.html
+ assertEquals(aPath2, URLUtil.chooseRepr(aPath, aPath2, true));
+ assertEquals(aPath, URLUtil.chooseRepr(aPath, aPath3, true));
+
+ //8) temp and both root keep shortest sub domain
+ // *www.a.com -> www.news.a.com
+ assertEquals(aDotCom, URLUtil.chooseRepr(aDotCom, aSubDotCom, true));
}
}
Re: svn commit: r723441 [2/2] - in /lucene/nutch/trunk: ./ src/java/org/apache/nutch/scoring/ src/java/org/apache/nutch/scoring/webgraph/ src/java/org/apache/nutch/tools/compat/ src/java/org/apache/nutch/util/ src/plugin/ src/plugin/scoring-link/ src/plugi
Posted by Fu Chen <ch...@servst.com>.
why me email is rejected ? thank you!
Best Regards
Fu Chen （陈福）
---------------
Inst.Service Science&Technology
Room 1-211, Future Internet Technology Research Center(FIT)
Tsinghua University, 100084, Beijing, China
Tel: 86-10-62603217-823，86-13520253784(mobile)
E_Mail:chenfu@serv.edu.cn;chenfurun@gmail.com
http://nmgroup.tsinghua.edu.cn/cn/people_asso.htm
----- Original Message -----
From: <ku...@apache.org>
To: <nu...@lucene.apache.org>
Sent: Friday, December 05, 2008 5:16 AM
Subject: svn commit: r723441 [2/2] - in /lucene/nutch/trunk: ./ src/java/org/apache/nutch/scoring/ src/java/org/apache/nutch/scoring/webgraph/ src/java/org/apache/nutch/tools/compat/ src/java/org/apache/nutch/util/ src/plugin/ src/plugin/scoring-link/ src/plugi...
> Added: lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java
> URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java?rev=723441&view=auto
> ==============================================================================
> --- lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java (added)
> +++ lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java Thu Dec 4 13:16:42 2008
> @@ -0,0 +1,226 @@
> +package org.apache.nutch.scoring.webgraph;
> +
> +import java.io.IOException;
> +import java.util.Iterator;
> +import java.util.Random;
> +
> +import org.apache.commons.cli.CommandLine;
> +import org.apache.commons.cli.CommandLineParser;
> +import org.apache.commons.cli.GnuParser;
> +import org.apache.commons.cli.HelpFormatter;
> +import org.apache.commons.cli.Option;
> +import org.apache.commons.cli.OptionBuilder;
> +import org.apache.commons.cli.Options;
> +import org.apache.commons.logging.Log;
> +import org.apache.commons.logging.LogFactory;
> +import org.apache.hadoop.conf.Configuration;
> +import org.apache.hadoop.conf.Configured;
> +import org.apache.hadoop.fs.FileSystem;
> +import org.apache.hadoop.fs.Path;
> +import org.apache.hadoop.io.ObjectWritable;
> +import org.apache.hadoop.io.Text;
> +import org.apache.hadoop.io.Writable;
> +import org.apache.hadoop.mapred.FileInputFormat;
> +import org.apache.hadoop.mapred.FileOutputFormat;
> +import org.apache.hadoop.mapred.JobClient;
> +import org.apache.hadoop.mapred.JobConf;
> +import org.apache.hadoop.mapred.MapFileOutputFormat;
> +import org.apache.hadoop.mapred.Mapper;
> +import org.apache.hadoop.mapred.OutputCollector;
> +import org.apache.hadoop.mapred.Reducer;
> +import org.apache.hadoop.mapred.Reporter;
> +import org.apache.hadoop.mapred.SequenceFileInputFormat;
> +import org.apache.hadoop.util.StringUtils;
> +import org.apache.hadoop.util.Tool;
> +import org.apache.hadoop.util.ToolRunner;
> +import org.apache.nutch.crawl.CrawlDatum;
> +import org.apache.nutch.crawl.CrawlDb;
> +import org.apache.nutch.util.NutchConfiguration;
> +import org.apache.nutch.util.NutchJob;
> +
> +/**
> + * Updates the score from the WebGraph node database into the crawl database.
> + * Any score that is not in the node database is set to the clear score in the
> + * crawl database.
> + */
> +public class ScoreUpdater
> + extends Configured
> + implements Tool, Mapper<Text, Writable, Text, ObjectWritable>,
> + Reducer<Text, ObjectWritable, Text, CrawlDatum> {
> +
> + public static final Log LOG = LogFactory.getLog(ScoreUpdater.class);
> +
> + private JobConf conf;
> + private float clearScore = 0.0f;
> +
> + public void configure(JobConf conf) {
> + this.conf = conf;
> + clearScore = conf.getFloat("link.score.updater.clear.score", 0.0f);
> + }
> +
> + /**
> + * Changes input into ObjectWritables.
> + */
> + public void map(Text key, Writable value,
> + OutputCollector<Text, ObjectWritable> output, Reporter reporter)
> + throws IOException {
> +
> + ObjectWritable objWrite = new ObjectWritable();
> + objWrite.set(value);
> + output.collect(key, objWrite);
> + }
> +
> + /**
> + * Creates new CrawlDatum objects with the updated score from the NodeDb or
> + * with a cleared score.
> + */
> + public void reduce(Text key, Iterator<ObjectWritable> values,
> + OutputCollector<Text, CrawlDatum> output, Reporter reporter)
> + throws IOException {
> +
> + String url = key.toString();
> + Node node = null;
> + CrawlDatum datum = null;
> +
> + // set the node and the crawl datum, should be one of each unless no node
> + // for url in the crawldb
> + while (values.hasNext()) {
> + ObjectWritable next = values.next();
> + Object value = next.get();
> + if (value instanceof Node) {
> + node = (Node)value;
> + }
> + else if (value instanceof CrawlDatum) {
> + datum = (CrawlDatum)value;
> + }
> + }
> +
> + // datum should never be null, could happen if somehow the url was
> + // normalized or changed after being pulled from the crawldb
> + if (datum != null) {
> +
> + if (node != null) {
> +
> + // set the inlink score in the nodedb
> + float inlinkScore = node.getInlinkScore();
> + datum.setScore(inlinkScore);
> + LOG.debug(url + ": setting to score " + inlinkScore);
> + }
> + else {
> +
> + // clear out the score in the crawldb
> + datum.setScore(clearScore);
> + LOG.debug(url + ": setting to clear score of " + clearScore);
> + }
> +
> + output.collect(key, datum);
> + }
> + else {
> + LOG.debug(url + ": no datum");
> + }
> + }
> +
> + public void close() {
> + }
> +
> + /**
> + * Updates the inlink score in the web graph node databsae into the crawl
> + * database.
> + *
> + * @param crawlDb The crawl database to update
> + * @param webGraphDb The webgraph database to use.
> + *
> + * @throws IOException If an error occurs while updating the scores.
> + */
> + public void update(Path crawlDb, Path webGraphDb)
> + throws IOException {
> +
> + Configuration conf = getConf();
> + FileSystem fs = FileSystem.get(conf);
> +
> + // create a temporary crawldb with the new scores
> + LOG.info("Running crawldb update " + crawlDb);
> + Path nodeDb = new Path(webGraphDb, WebGraph.NODE_DIR);
> + Path crawlDbCurrent = new Path(crawlDb, CrawlDb.CURRENT_NAME);
> + Path newCrawlDb = new Path(crawlDb,
> + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
> +
> + // run the updater job outputting to the temp crawl database
> + JobConf updater = new NutchJob(conf);
> + updater.setJobName("Update CrawlDb from WebGraph");
> + FileInputFormat.addInputPath(updater, crawlDbCurrent);
> + FileInputFormat.addInputPath(updater, nodeDb);
> + FileOutputFormat.setOutputPath(updater, newCrawlDb);
> + updater.setInputFormat(SequenceFileInputFormat.class);
> + updater.setMapperClass(ScoreUpdater.class);
> + updater.setReducerClass(ScoreUpdater.class);
> + updater.setMapOutputKeyClass(Text.class);
> + updater.setMapOutputValueClass(ObjectWritable.class);
> + updater.setOutputKeyClass(Text.class);
> + updater.setOutputValueClass(CrawlDatum.class);
> + updater.setOutputFormat(MapFileOutputFormat.class);
> +
> + try {
> + JobClient.runJob(updater);
> + }
> + catch (IOException e) {
> + LOG.error(StringUtils.stringifyException(e));
> +
> + // remove the temp crawldb on error
> + if (fs.exists(newCrawlDb)) {
> + fs.delete(newCrawlDb, true);
> + }
> + throw e;
> + }
> +
> + // install the temp crawl database
> + LOG.info("Installing new crawldb " + crawlDb);
> + CrawlDb.install(updater, crawlDb);
> + }
> +
> + public static void main(String[] args)
> + throws Exception {
> + int res = ToolRunner.run(NutchConfiguration.create(), new ScoreUpdater(),
> + args);
> + System.exit(res);
> + }
> +
> + /**
> + * Runs the ScoreUpdater tool.
> + */
> + public int run(String[] args)
> + throws Exception {
> +
> + Options options = new Options();
> + Option helpOpts = OptionBuilder.withArgName("help").withDescription(
> + "show this help message").create("help");
> + Option crawlDbOpts = OptionBuilder.withArgName("crawldb").hasArg().withDescription(
> + "the crawldb to use").create("crawldb");
> + Option webGraphOpts = OptionBuilder.withArgName("webgraphdb").hasArg().withDescription(
> + "the webgraphdb to use").create("webgraphdb");
> + options.addOption(helpOpts);
> + options.addOption(crawlDbOpts);
> + options.addOption(webGraphOpts);
> +
> + CommandLineParser parser = new GnuParser();
> + try {
> +
> + CommandLine line = parser.parse(options, args);
> + if (line.hasOption("help") || !line.hasOption("webgraphdb")
> + || !line.hasOption("crawldb")) {
> + HelpFormatter formatter = new HelpFormatter();
> + formatter.printHelp("ScoreUpdater", options);
> + return -1;
> + }
> +
> + String crawlDb = line.getOptionValue("crawldb");
> + String webGraphDb = line.getOptionValue("webgraphdb");
> + update(new Path(crawlDb), new Path(webGraphDb));
> + return 0;
> + }
> + catch (Exception e) {
> + LOG.fatal("ScoreUpdater: " + StringUtils.stringifyException(e));
> + return -1;
> + }
> + }
> +}
>
> Added: lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
> URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java?rev=723441&view=auto
> ==============================================================================
> --- lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java (added)
> +++ lucene/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java Thu Dec 4 13:16:42 2008
> @@ -0,0 +1,629 @@
> +package org.apache.nutch.scoring.webgraph;
> +
> +import java.io.IOException;
> +import java.util.ArrayList;
> +import java.util.HashSet;
> +import java.util.Iterator;
> +import java.util.LinkedHashMap;
> +import java.util.List;
> +import java.util.Map;
> +import java.util.Random;
> +import java.util.Set;
> +
> +import org.apache.commons.cli.CommandLine;
> +import org.apache.commons.cli.CommandLineParser;
> +import org.apache.commons.cli.GnuParser;
> +import org.apache.commons.cli.HelpFormatter;
> +import org.apache.commons.cli.Option;
> +import org.apache.commons.cli.OptionBuilder;
> +import org.apache.commons.cli.Options;
> +import org.apache.commons.logging.Log;
> +import org.apache.commons.logging.LogFactory;
> +import org.apache.hadoop.conf.Configuration;
> +import org.apache.hadoop.conf.Configured;
> +import org.apache.hadoop.fs.FileSystem;
> +import org.apache.hadoop.fs.Path;
> +import org.apache.hadoop.io.Text;
> +import org.apache.hadoop.io.Writable;
> +import org.apache.hadoop.io.WritableUtils;
> +import org.apache.hadoop.mapred.FileInputFormat;
> +import org.apache.hadoop.mapred.FileOutputFormat;
> +import org.apache.hadoop.mapred.JobClient;
> +import org.apache.hadoop.mapred.JobConf;
> +import org.apache.hadoop.mapred.MapFileOutputFormat;
> +import org.apache.hadoop.mapred.Mapper;
> +import org.apache.hadoop.mapred.OutputCollector;
> +import org.apache.hadoop.mapred.Reducer;
> +import org.apache.hadoop.mapred.Reporter;
> +import org.apache.hadoop.mapred.SequenceFileInputFormat;
> +import org.apache.hadoop.util.StringUtils;
> +import org.apache.hadoop.util.Tool;
> +import org.apache.hadoop.util.ToolRunner;
> +import org.apache.nutch.metadata.Nutch;
> +import org.apache.nutch.net.URLNormalizers;
> +import org.apache.nutch.parse.Outlink;
> +import org.apache.nutch.parse.ParseData;
> +import org.apache.nutch.util.FSUtils;
> +import org.apache.nutch.util.LockUtil;
> +import org.apache.nutch.util.NutchConfiguration;
> +import org.apache.nutch.util.NutchJob;
> +import org.apache.nutch.util.URLUtil;
> +
> +/**
> + * Creates three databases, one for inlinks, one for outlinks, and a node
> + * database that holds the number of in and outlinks to a url and the current
> + * score for the url.
> + *
> + * The score is set by an analysis program such as LinkRank. The WebGraph is an
> + * update-able database. Outlinks are stored by their fetch time or by the
> + * current system time if no fetch time is available. Only the most recent
> + * version of outlinks for a given url is stored. As more crawls are executed
> + * and the WebGraph updated, newer Outlinks will replace older Outlinks. This
> + * allows the WebGraph to adapt to changes in the link structure of the web.
> + *
> + * The Inlink database is created from the Outlink database and is regenerated
> + * when the WebGraph is updated. The Node database is created from both the
> + * Inlink and Outlink databases. Because the Node database is overwritten when
> + * the WebGraph is updated and because the Node database holds current scores
> + * for urls it is recommended that a crawl-cyle (one or more full crawls) fully
> + * complete before the WebGraph is updated and some type of analysis, such as
> + * LinkRank, is run to update scores in the Node database in a stable fashion.
> + */
> +public class WebGraph
> + extends Configured
> + implements Tool {
> +
> + public static final Log LOG = LogFactory.getLog(WebGraph.class);
> + public static final String LOCK_NAME = ".locked";
> + public static final String INLINK_DIR = "inlinks";
> + public static final String OUTLINK_DIR = "outlinks";
> + public static final String NODE_DIR = "nodes";
> +
> + /**
> + * The OutlinkDb creates a database of all outlinks. Outlinks to internal urls
> + * by domain and host can be ignored. The number of Outlinks out to a given
> + * page or domain can also be limited.
> + */
> + public static class OutlinkDb
> + extends Configured
> + implements Mapper<Text, Writable, Text, LinkDatum>,
> + Reducer<Text, LinkDatum, Text, LinkDatum> {
> +
> + // ignoring internal domains, internal hosts
> + private boolean ignoreDomain = true;
> + private boolean ignoreHost = true;
> +
> + // limiting urls out to a page or to a domain
> + private boolean limitPages = true;
> + private boolean limitDomains = true;
> +
> + // url normalizers and job configuration
> + private URLNormalizers urlNormalizers;
> + private JobConf conf;
> +
> + /**
> + * Normalizes and trims extra whitespace from the given url.
> + *
> + * @param url The url to normalize.
> + *
> + * @return The normalized url.
> + */
> + private String normalizeUrl(String url) {
> +
> + String normalized = null;
> + if (urlNormalizers != null) {
> + try {
> +
> + // normalize and trim the url
> + normalized = urlNormalizers.normalize(url,
> + URLNormalizers.SCOPE_DEFAULT);
> + normalized = normalized.trim();
> + }
> + catch (Exception e) {
> + LOG.warn("Skipping " + url + ":" + e);
> + normalized = null;
> + }
> + }
> + return normalized;
> + }
> +
> + /**
> + * Returns the fetch time from the parse data or the current system time if
> + * the fetch time doesn't exist.
> + *
> + * @param data The parse data.
> + *
> + * @return The fetch time as a long.
> + */
> + private long getFetchTime(ParseData data) {
> +
> + // default to current system time
> + long fetchTime = System.currentTimeMillis();
> + String fetchTimeStr = data.getContentMeta().get(Nutch.FETCH_TIME_KEY);
> + try {
> +
> + // get the fetch time from the parse data
> + fetchTime = Long.parseLong(fetchTimeStr);
> + }
> + catch (Exception e) {
> + fetchTime = System.currentTimeMillis();
> + }
> + return fetchTime;
> + }
> +
> + /**
> + * Default constructor.
> + */
> + public OutlinkDb() {
> + }
> +
> + /**
> + * Configurable constructor.
> + */
> + public OutlinkDb(Configuration conf) {
> + setConf(conf);
> + }
> +
> + /**
> + * Configures the OutlinkDb job. Sets up internal links and link limiting.
> + */
> + public void configure(JobConf conf) {
> + this.conf = conf;
> + ignoreHost = conf.getBoolean("link.ignore.internal.host", true);
> + ignoreDomain = conf.getBoolean("link.ignore.internal.domain", true);
> + limitPages = conf.getBoolean("link.ignore.limit.page", true);
> + limitDomains = conf.getBoolean("link.ignore.limit.domain", true);
> + urlNormalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_DEFAULT);
> + }
> +
> + /**
> + * Passes through existing LinkDatum objects from an existing OutlinkDb and
> + * maps out new LinkDatum objects from new crawls ParseData.
> + */
> + public void map(Text key, Writable value,
> + OutputCollector<Text, LinkDatum> output, Reporter reporter)
> + throws IOException {
> +
> + // normalize url, stop processing if null
> + String url = normalizeUrl(key.toString());
> + if (url == null) {
> + return;
> + }
> +
> + if (value instanceof ParseData) {
> +
> + // get the parse data and the outlinks from the parse data, along with
> + // the fetch time for those links
> + ParseData data = (ParseData)value;
> + long fetchTime = getFetchTime(data);
> + Outlink[] outlinkAr = data.getOutlinks();
> + Map<String, String> outlinkMap = new LinkedHashMap<String, String>();
> +
> + // normalize urls and put into map
> + if (outlinkAr != null && outlinkAr.length > 0) {
> + for (int i = 0; i < outlinkAr.length; i++) {
> + Outlink outlink = outlinkAr[i];
> + String toUrl = normalizeUrl(outlink.getToUrl());
> +
> + // only put into map if the url doesn't already exist in the map or
> + // if it does and the anchor for that link is null, will replace if
> + // url is existing
> + boolean existingUrl = outlinkMap.containsKey(toUrl);
> + if (toUrl != null
> + && (!existingUrl || (existingUrl && outlinkMap.get(toUrl) == null))) {
> + outlinkMap.put(toUrl, outlink.getAnchor());
> + }
> + }
> + }
> +
> + // collect the outlinks under the fetch time
> + for (String outlinkUrl : outlinkMap.keySet()) {
> + String anchor = outlinkMap.get(outlinkUrl);
> + LinkDatum datum = new LinkDatum(outlinkUrl, anchor, fetchTime);
> + output.collect(key, datum);
> + }
> + }
> + else if (value instanceof LinkDatum) {
> +
> + // collect existing outlinks from existing OutlinkDb
> + output.collect(key, (LinkDatum)value);
> + }
> + }
> +
> + public void reduce(Text key, Iterator<LinkDatum> values,
> + OutputCollector<Text, LinkDatum> output, Reporter reporter)
> + throws IOException {
> +
> + // aggregate all outlinks, get the most recent timestamp for a fetch
> + // which should be the timestamp for all of the most recent outlinks
> + long mostRecent = 0L;
> + List<LinkDatum> outlinkList = new ArrayList<LinkDatum>();
> + while (values.hasNext()) {
> +
> + // loop through, change out most recent timestamp if needed
> + LinkDatum next = values.next();
> + long timestamp = next.getTimestamp();
> + if (mostRecent == 0L || mostRecent < timestamp) {
> + mostRecent = timestamp;
> + }
> + outlinkList.add((LinkDatum)WritableUtils.clone(next, conf));
> + }
> +
> + // get the url, domain, and host for the url
> + String url = key.toString();
> + String domain = URLUtil.getDomainName(url);
> + String host = URLUtil.getHost(url);
> +
> + // setup checking sets for domains and pages
> + Set<String> domains = new HashSet<String>();
> + Set<String> pages = new HashSet<String>();
> +
> + // loop through the link datums
> + for (LinkDatum datum : outlinkList) {
> +
> + // get the url, host, domain, and page for each outlink
> + String toUrl = datum.getUrl();
> + String toDomain = URLUtil.getDomainName(toUrl);
> + String toHost = URLUtil.getHost(toUrl);
> + String toPage = URLUtil.getPage(toUrl);
> + datum.setLinkType(LinkDatum.OUTLINK);
> +
> + // outlinks must be the most recent and conform to internal url and
> + // limiting rules, if it does collect it
> + if (datum.getTimestamp() == mostRecent
> + && (!limitPages || (limitPages && !pages.contains(toPage)))
> + && (!limitDomains || (limitDomains && !domains.contains(toDomain)))
> + && (!ignoreHost || (ignoreHost && !toHost.equalsIgnoreCase(host)))
> + && (!ignoreDomain || (ignoreDomain && !toDomain.equalsIgnoreCase(domain)))) {
> + output.collect(key, datum);
> + pages.add(toPage);
> + domains.add(toDomain);
> + }
> + }
> + }
> +
> + public void close() {
> + }
> + }
> +
> + /**
> + * The InlinkDb creates a database of Inlinks. Inlinks are inverted from the
> + * OutlinkDb LinkDatum objects and are regenerated each time the WebGraph is
> + * updated.
> + */
> + private static class InlinkDb
> + extends Configured
> + implements Mapper<Text, LinkDatum, Text, LinkDatum> {
> +
> + private JobConf conf;
> + private long timestamp;
> +
> + /**
> + * Default constructor.
> + */
> + public InlinkDb() {
> + }
> +
> + /**
> + * Configurable constructor.
> + */
> + public InlinkDb(Configuration conf) {
> + setConf(conf);
> + }
> +
> + /**
> + * Configures job. Sets timestamp for all Inlink LinkDatum objects to the
> + * current system time.
> + */
> + public void configure(JobConf conf) {
> + this.conf = conf;
> + timestamp = System.currentTimeMillis();
> + }
> +
> + public void close() {
> + }
> +
> + /**
> + * Inverts the Outlink LinkDatum objects into new LinkDatum objects with a
> + * new system timestamp, type and to and from url switched.
> + */
> + public void map(Text key, LinkDatum datum,
> + OutputCollector<Text, LinkDatum> output, Reporter reporter)
> + throws IOException {
> +
> + // get the to and from url and the anchor
> + String fromUrl = key.toString();
> + String toUrl = datum.getUrl();
> + String anchor = datum.getAnchor();
> +
> + // flip the from and to url and set the new link type
> + LinkDatum inlink = new LinkDatum(fromUrl, anchor, timestamp);
> + inlink.setLinkType(LinkDatum.INLINK);
> + output.collect(new Text(toUrl), inlink);
> + }
> + }
> +
> + /**
> + * Creates the Node database which consists of the number of in and outlinks
> + * for each url and a score slot for analysis programs such as LinkRank.
> + */
> + private static class NodeDb
> + extends Configured
> + implements Reducer<Text, LinkDatum, Text, Node> {
> +
> + private JobConf conf;
> +
> + /**
> + * Default constructor.
> + */
> + public NodeDb() {
> + }
> +
> + /**
> + * Configurable constructor.
> + */
> + public NodeDb(Configuration conf) {
> + setConf(conf);
> + }
> +
> + /**
> + * Configures job.
> + */
> + public void configure(JobConf conf) {
> + this.conf = conf;
> + }
> +
> + public void close() {
> + }
> +
> + /**
> + * Counts the number of inlinks and outlinks for each url and sets a default
> + * score of 0.0 for each url (node) in the webgraph.
> + */
> + public void reduce(Text key, Iterator<LinkDatum> values,
> + OutputCollector<Text, Node> output, Reporter reporter)
> + throws IOException {
> +
> + Node node = new Node();
> + int numInlinks = 0;
> + int numOutlinks = 0;
> +
> + // loop through counting number of in and out links
> + while (values.hasNext()) {
> + LinkDatum next = values.next();
> + if (next.getLinkType() == LinkDatum.INLINK) {
> + numInlinks++;
> + }
> + else if (next.getLinkType() == LinkDatum.OUTLINK) {
> + numOutlinks++;
> + }
> + }
> +
> + // set the in and outlinks and a default score of 0
> + node.setNumInlinks(numInlinks);
> + node.setNumOutlinks(numOutlinks);
> + node.setInlinkScore(0.0f);
> + output.collect(key, node);
> + }
> + }
> +
> + /**
> + * Creates the three different WebGraph databases, Outlinks, Inlinks, and
> + * Node. If a current WebGraph exists then it is updated, if it doesn't exist
> + * then a new WebGraph database is created.
> + *
> + * @param webGraphDb The WebGraph to create or update.
> + * @param segments The array of segments used to update the WebGraph. Newer
> + * segments and fetch times will overwrite older segments.
> + *
> + * @throws IOException If an error occurs while processing the WebGraph.
> + */
> + public void createWebGraph(Path webGraphDb, Path[] segments)
> + throws IOException {
> +
> + if (LOG.isInfoEnabled()) {
> + LOG.info("WebGraphDb: starting");
> + LOG.info("WebGraphDb: webgraphdb: " + webGraphDb);
> + }
> +
> + Configuration conf = getConf();
> + FileSystem fs = FileSystem.get(conf);
> +
> + // lock an existing webgraphdb to prevent multiple simultaneous updates
> + Path lock = new Path(webGraphDb, LOCK_NAME);
> + boolean webGraphDbExists = fs.exists(webGraphDb);
> + if (webGraphDbExists) {
> + LockUtil.createLockFile(fs, lock, false);
> + }
> + else {
> +
> + // if the webgraph doesn't exist, create it
> + fs.mkdirs(webGraphDb);
> + }
> +
> + // outlink and temp outlink database paths
> + Path outlinkDb = new Path(webGraphDb, OUTLINK_DIR);
> + Path tempOutlinkDb = new Path(outlinkDb + "-"
> + + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
> + JobConf outlinkJob = new NutchJob(conf);
> + outlinkJob.setJobName("Outlinkdb: " + outlinkDb);
> +
> + // get the parse data for all segments
> + if (segments != null) {
> + for (int i = 0; i < segments.length; i++) {
> + Path parseData = new Path(segments[i], ParseData.DIR_NAME);
> + if (fs.exists(parseData)) {
> + LOG.info("OutlinkDb: adding input: " + parseData);
> + FileInputFormat.addInputPath(outlinkJob, parseData);
> + }
> + }
> + }
> +
> + // add the existing webgraph
> + if (webGraphDbExists) {
> + LOG.info("OutlinkDb: adding input: " + outlinkDb);
> + FileInputFormat.addInputPath(outlinkJob, outlinkDb);
> + }
> +
> + outlinkJob.setInputFormat(SequenceFileInputFormat.class);
> + outlinkJob.setMapperClass(OutlinkDb.class);
> + outlinkJob.setReducerClass(OutlinkDb.class);
> + outlinkJob.setMapOutputKeyClass(Text.class);
> + outlinkJob.setMapOutputValueClass(LinkDatum.class);
> + outlinkJob.setOutputKeyClass(Text.class);
> + outlinkJob.setOutputValueClass(LinkDatum.class);
> + FileOutputFormat.setOutputPath(outlinkJob, tempOutlinkDb);
> + outlinkJob.setOutputFormat(MapFileOutputFormat.class);
> +
> + // run the outlinkdb job and replace any old outlinkdb with the new one
> + try {
> + LOG.info("OutlinkDb: running");
> + JobClient.runJob(outlinkJob);
> + LOG.info("OutlinkDb: installing " + outlinkDb);
> + FSUtils.replace(fs, outlinkDb, tempOutlinkDb, true);
> + LOG.info("OutlinkDb: finished");
> + }
> + catch (IOException e) {
> +
> + // remove lock file and and temporary directory if an error occurs
> + LockUtil.removeLockFile(fs, lock);
> + if (fs.exists(tempOutlinkDb)) {
> + fs.delete(tempOutlinkDb, true);
> + }
> + LOG.error(StringUtils.stringifyException(e));
> + throw e;
> + }
> +
> + // inlink and temp link database paths
> + Path inlinkDb = new Path(webGraphDb, INLINK_DIR);
> + Path tempInlinkDb = new Path(inlinkDb + "-"
> + + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
> +
> + JobConf inlinkJob = new NutchJob(conf);
> + inlinkJob.setJobName("Inlinkdb " + inlinkDb);
> + LOG.info("InlinkDb: adding input: " + outlinkDb);
> + FileInputFormat.addInputPath(inlinkJob, outlinkDb);
> + inlinkJob.setInputFormat(SequenceFileInputFormat.class);
> + inlinkJob.setMapperClass(InlinkDb.class);
> + inlinkJob.setMapOutputKeyClass(Text.class);
> + inlinkJob.setMapOutputValueClass(LinkDatum.class);
> + inlinkJob.setOutputKeyClass(Text.class);
> + inlinkJob.setOutputValueClass(LinkDatum.class);
> + FileOutputFormat.setOutputPath(inlinkJob, tempInlinkDb);
> + inlinkJob.setOutputFormat(MapFileOutputFormat.class);
> +
> + try {
> +
> + // run the inlink and replace any old with new
> + LOG.info("InlinkDb: running");
> + JobClient.runJob(inlinkJob);
> + LOG.info("InlinkDb: installing " + inlinkDb);
> + FSUtils.replace(fs, inlinkDb, tempInlinkDb, true);
> + LOG.info("InlinkDb: finished");
> + }
> + catch (IOException e) {
> +
> + // remove lock file and and temporary directory if an error occurs
> + LockUtil.removeLockFile(fs, lock);
> + if (fs.exists(tempInlinkDb)) {
> + fs.delete(tempInlinkDb, true);
> + }
> + LOG.error(StringUtils.stringifyException(e));
> + throw e;
> + }
> +
> + // node and temp node database paths
> + Path nodeDb = new Path(webGraphDb, NODE_DIR);
> + Path tempNodeDb = new Path(nodeDb + "-"
> + + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
> +
> + JobConf nodeJob = new NutchJob(conf);
> + nodeJob.setJobName("NodeDb " + nodeDb);
> + LOG.info("NodeDb: adding input: " + outlinkDb);
> + LOG.info("NodeDb: adding input: " + inlinkDb);
> + FileInputFormat.addInputPath(nodeJob, outlinkDb);
> + FileInputFormat.addInputPath(nodeJob, inlinkDb);
> + nodeJob.setInputFormat(SequenceFileInputFormat.class);
> + nodeJob.setReducerClass(NodeDb.class);
> + nodeJob.setMapOutputKeyClass(Text.class);
> + nodeJob.setMapOutputValueClass(LinkDatum.class);
> + nodeJob.setOutputKeyClass(Text.class);
> + nodeJob.setOutputValueClass(Node.class);
> + FileOutputFormat.setOutputPath(nodeJob, tempNodeDb);
> + nodeJob.setOutputFormat(MapFileOutputFormat.class);
> +
> + try {
> +
> + // run the node job and replace old nodedb with new
> + LOG.info("NodeDb: running");
> + JobClient.runJob(nodeJob);
> + LOG.info("NodeDb: installing " + nodeDb);
> + FSUtils.replace(fs, nodeDb, tempNodeDb, true);
> + LOG.info("NodeDb: finished");
> + }
> + catch (IOException e) {
> +
> + // remove lock file and and temporary directory if an error occurs
> + LockUtil.removeLockFile(fs, lock);
> + if (fs.exists(tempNodeDb)) {
> + fs.delete(tempNodeDb, true);
> + }
> + LOG.error(StringUtils.stringifyException(e));
> + throw e;
> + }
> +
> + // remove the lock file for the webgraph
> + LockUtil.removeLockFile(fs, lock);
> + }
> +
> + public static void main(String[] args)
> + throws Exception {
> + int res = ToolRunner.run(NutchConfiguration.create(), new WebGraph(), args);
> + System.exit(res);
> + }
> +
> + /**
> + * Parses command link arguments and runs the WebGraph jobs.
> + */
> + public int run(String[] args)
> + throws Exception {
> +
> + Options options = new Options();
> + Option helpOpts = OptionBuilder.withArgName("help").withDescription(
> + "show this help message").create("help");
> + Option webGraphDbOpts = OptionBuilder.withArgName("webgraphdb").hasArg().withDescription(
> + "the web graph database to use").create("webgraphdb");
> + Option segOpts = OptionBuilder.withArgName("segment").hasArgs().withDescription(
> + "the segment(s) to use").create("segment");
> + options.addOption(helpOpts);
> + options.addOption(webGraphDbOpts);
> + options.addOption(segOpts);
> +
> + CommandLineParser parser = new GnuParser();
> + try {
> +
> + CommandLine line = parser.parse(options, args);
> + if (line.hasOption("help") || !line.hasOption("webgraphdb")
> + || !line.hasOption("segment")) {
> + HelpFormatter formatter = new HelpFormatter();
> + formatter.printHelp("WebGraph", options);
> + return -1;
> + }
> +
> + String webGraphDb = line.getOptionValue("webgraphdb");
> + String[] segments = line.getOptionValues("segment");
> + Path[] segPaths = new Path[segments.length];
> + for (int i = 0; i < segments.length; i++) {
> + segPaths[i] = new Path(segments[i]);
> + }
> +
> + createWebGraph(new Path(webGraphDb), segPaths);
> + return 0;
> + }
> + catch (Exception e) {
> + LOG.fatal("WebGraph: " + StringUtils.stringifyException(e));
> + return -2;
> + }
> + }
> +
> +}
>
> Added: lucene/nutch/trunk/src/java/org/apache/nutch/tools/compat/ReprUrlFixer.java
> URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/tools/compat/ReprUrlFixer.java?rev=723441&view=auto
> ==============================================================================
> --- lucene/nutch/trunk/src/java/org/apache/nutch/tools/compat/ReprUrlFixer.java (added)
> +++ lucene/nutch/trunk/src/java/org/apache/nutch/tools/compat/ReprUrlFixer.java Thu Dec 4 13:16:42 2008
> @@ -0,0 +1,296 @@
> +package org.apache.nutch.tools.compat;
> +
> +import java.io.IOException;
> +import java.net.MalformedURLException;
> +import java.net.URL;
> +import java.util.ArrayList;
> +import java.util.Iterator;
> +import java.util.List;
> +import java.util.Random;
> +
> +import org.apache.commons.cli.CommandLine;
> +import org.apache.commons.cli.CommandLineParser;
> +import org.apache.commons.cli.GnuParser;
> +import org.apache.commons.cli.HelpFormatter;
> +import org.apache.commons.cli.Option;
> +import org.apache.commons.cli.OptionBuilder;
> +import org.apache.commons.cli.Options;
> +import org.apache.commons.logging.Log;
> +import org.apache.commons.logging.LogFactory;
> +import org.apache.hadoop.conf.Configuration;
> +import org.apache.hadoop.conf.Configured;
> +import org.apache.hadoop.fs.FileSystem;
> +import org.apache.hadoop.fs.Path;
> +import org.apache.hadoop.io.Text;
> +import org.apache.hadoop.io.WritableUtils;
> +import org.apache.hadoop.mapred.FileInputFormat;
> +import org.apache.hadoop.mapred.FileOutputFormat;
> +import org.apache.hadoop.mapred.JobClient;
> +import org.apache.hadoop.mapred.JobConf;
> +import org.apache.hadoop.mapred.MapFileOutputFormat;
> +import org.apache.hadoop.mapred.OutputCollector;
> +import org.apache.hadoop.mapred.Reducer;
> +import org.apache.hadoop.mapred.Reporter;
> +import org.apache.hadoop.mapred.SequenceFileInputFormat;
> +import org.apache.hadoop.util.StringUtils;
> +import org.apache.hadoop.util.Tool;
> +import org.apache.hadoop.util.ToolRunner;
> +import org.apache.nutch.crawl.CrawlDatum;
> +import org.apache.nutch.crawl.CrawlDb;
> +import org.apache.nutch.crawl.MapWritable;
> +import org.apache.nutch.metadata.Nutch;
> +import org.apache.nutch.scoring.webgraph.Node;
> +import org.apache.nutch.util.FSUtils;
> +import org.apache.nutch.util.NutchConfiguration;
> +import org.apache.nutch.util.NutchJob;
> +import org.apache.nutch.util.URLUtil;
> +
> +/**
> + * <p>
> + * Significant changes were made to representative url logic used for redirects.
> + * This tool will fix representative urls stored in current segments and crawl
> + * databases. Any new fetches will use the new representative url logic.
> + * </p>
> + *
> + * <p>
> + * All crawl datums are assumed to be temp url redirects. While this may cause
> + * some urls to be incorrectly removed, this tool is a temporary measure to be
> + * used until fetches can be rerun. This reduce logic is the same for segments
> + * fetch and parse directory as well as for existing crawl databases.
> + * </p>
> + */
> +public class ReprUrlFixer
> + extends Configured
> + implements Tool, Reducer<Text, CrawlDatum, Text, CrawlDatum> {
> +
> + public static final Log LOG = LogFactory.getLog(ReprUrlFixer.class);
> + private JobConf conf;
> +
> + public void configure(JobConf conf) {
> + this.conf = conf;
> + }
> +
> + /**
> + * Runs the new ReprUrl logic on all crawldatums.
> + */
> + public void reduce(Text key, Iterator<CrawlDatum> values,
> + OutputCollector<Text, CrawlDatum> output, Reporter reporter)
> + throws IOException {
> +
> + String url = key.toString();
> + Node node = null;
> + List<CrawlDatum> datums = new ArrayList<CrawlDatum>();
> +
> + // get all crawl datums for a given url key, fetch for instance can have
> + // more than one under a given key if there are multiple redirects to a
> + // given url
> + while (values.hasNext()) {
> + CrawlDatum datum = values.next();
> + datums.add((CrawlDatum)WritableUtils.clone(datum, conf));
> + }
> +
> + // apply redirect repr url logic for each datum
> + for (CrawlDatum datum : datums) {
> +
> + MapWritable metadata = datum.getMetaData();
> + Text reprUrl = (Text)metadata.get(Nutch.WRITABLE_REPR_URL_KEY);
> + byte status = datum.getStatus();
> + boolean isCrawlDb = (CrawlDatum.hasDbStatus(datum));
> + boolean segFetched = (status == CrawlDatum.STATUS_FETCH_SUCCESS);
> +
> + // only if the crawl datum is from the crawldb or is a successfully
> + // fetched page from the segments
> + if ((isCrawlDb || segFetched) && reprUrl != null) {
> +
> + String src = reprUrl.toString();
> + String dest = url;
> + URL srcUrl = null;
> + URL dstUrl = null;
> +
> + // both need to be well formed urls
> + try {
> + srcUrl = new URL(src);
> + dstUrl = new URL(url);
> + }
> + catch (MalformedURLException e) {
> + }
> +
> + // if the src and repr urls are the same after the new logic then
> + // remove the repr url from the metadata as it is no longer needed
> + if (srcUrl != null && dstUrl != null) {
> + String reprOut = URLUtil.chooseRepr(src, dest, true);
> + if (reprOut.equals(dest)) {
> + LOG.info("Removing " + reprOut + " from " + dest);
> + metadata.remove(Nutch.WRITABLE_REPR_URL_KEY);
> + }
> + }
> + }
> +
> + // collect each datum
> + output.collect(key, datum);
> + }
> +
> + }
> +
> + public void close() {
> + }
> +
> + /**
> + * Run the fixer on any crawl database and segments specified.
> + */
> + public void update(Path crawlDb, Path[] segments)
> + throws IOException {
> +
> + Configuration conf = getConf();
> + FileSystem fs = FileSystem.get(conf);
> +
> + // run the crawl database through the repr fixer
> + if (crawlDb != null) {
> +
> + LOG.info("Running ReprUtilFixer " + crawlDb);
> + Path crawlDbCurrent = new Path(crawlDb, CrawlDb.CURRENT_NAME);
> + Path newCrawlDb = new Path(crawlDb,
> + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
> +
> + JobConf updater = new NutchJob(conf);
> + updater.setJobName("ReprUtilFixer: " + crawlDb.toString());
> + FileInputFormat.addInputPath(updater, crawlDbCurrent);
> + FileOutputFormat.setOutputPath(updater, newCrawlDb);
> + updater.setInputFormat(SequenceFileInputFormat.class);
> + updater.setReducerClass(ReprUrlFixer.class);
> + updater.setOutputKeyClass(Text.class);
> + updater.setOutputValueClass(CrawlDatum.class);
> + updater.setOutputFormat(MapFileOutputFormat.class);
> +
> + try {
> + JobClient.runJob(updater);
> + LOG.info("Installing new crawldb " + crawlDb);
> + CrawlDb.install(updater, crawlDb);
> + }
> + catch (IOException e) {
> + LOG.error(StringUtils.stringifyException(e));
> + throw e;
> + }
> + }
> +
> + // run the segments through the repr fixer, logic will be run on both the
> + // crawl_parse and the crawl_fetch directories for every segment specified
> + if (segments != null) {
> +
> + for (int i = 0; i < segments.length; i++) {
> +
> + Path segment = segments[i];
> + LOG.info("Running ReprUtilFixer " + segment + " fetch");
> + Path segFetch = new Path(segment, CrawlDatum.FETCH_DIR_NAME);
> + Path newSegFetch = new Path(segment, CrawlDatum.FETCH_DIR_NAME + "-"
> + + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
> +
> + JobConf fetch = new NutchJob(conf);
> + fetch.setJobName("ReprUtilFixer: " + segment.toString());
> + FileInputFormat.addInputPath(fetch, segFetch);
> + FileOutputFormat.setOutputPath(fetch, newSegFetch);
> + fetch.setInputFormat(SequenceFileInputFormat.class);
> + fetch.setReducerClass(ReprUrlFixer.class);
> + fetch.setOutputKeyClass(Text.class);
> + fetch.setOutputValueClass(CrawlDatum.class);
> + fetch.setOutputFormat(MapFileOutputFormat.class);
> +
> + try {
> + JobClient.runJob(fetch);
> + LOG.info("Installing new segment fetch directory " + newSegFetch);
> + FSUtils.replace(fs, segFetch, newSegFetch, true);
> + LOG.info("ReprUrlFixer: finished installing segment fetch directory");
> + }
> + catch (IOException e) {
> + LOG.error(StringUtils.stringifyException(e));
> + throw e;
> + }
> +
> + LOG.info("Running ReprUtilFixer " + segment + " parse");
> + Path segParse = new Path(segment, CrawlDatum.PARSE_DIR_NAME);
> + Path newSegParse = new Path(segment, CrawlDatum.PARSE_DIR_NAME + "-"
> + + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
> +
> + JobConf parse = new NutchJob(conf);
> + parse.setJobName("ReprUtilFixer: " + segment.toString());
> + FileInputFormat.addInputPath(parse, segParse);
> + FileOutputFormat.setOutputPath(parse, newSegParse);
> + parse.setInputFormat(SequenceFileInputFormat.class);
> + parse.setReducerClass(ReprUrlFixer.class);
> + parse.setOutputKeyClass(Text.class);
> + parse.setOutputValueClass(CrawlDatum.class);
> + parse.setOutputFormat(MapFileOutputFormat.class);
> +
> + try {
> + JobClient.runJob(parse);
> + LOG.info("Installing new segment parse directry " + newSegParse);
> + FSUtils.replace(fs, segParse, newSegParse, true);
> + LOG.info("ReprUrlFixer: finished installing segment parse directory");
> + }
> + catch (IOException e) {
> + LOG.error(StringUtils.stringifyException(e));
> + throw e;
> + }
> + }
> + }
> + }
> +
> + /**
> + * Runs The ReprUrlFixer.
> + */
> + public static void main(String[] args)
> + throws Exception {
> + int res = ToolRunner.run(NutchConfiguration.create(), new ReprUrlFixer(),
> + args);
> + System.exit(res);
> + }
> +
> + /**
> + * Parse command line options and execute the main update logic.
> + */
> + public int run(String[] args)
> + throws Exception {
> +
> + Options options = new Options();
> + Option helpOpts = OptionBuilder.withArgName("help").withDescription(
> + "show this help message").create("help");
> + Option crawlDbOpts = OptionBuilder.withArgName("crawldb").hasArg().withDescription(
> + "the crawldb to use").create("crawldb");
> + Option segOpts = OptionBuilder.withArgName("segment").hasArgs().withDescription(
> + "the segment(s) to use").create("segment");
> + options.addOption(helpOpts);
> + options.addOption(crawlDbOpts);
> + options.addOption(segOpts);
> +
> + CommandLineParser parser = new GnuParser();
> + try {
> +
> + // parse out common line arguments and make sure either a crawldb or a
> + // segment are specified
> + CommandLine line = parser.parse(options, args);
> + if (line.hasOption("help")
> + || (!line.hasOption("crawldb") && !line.hasOption("segment"))) {
> + HelpFormatter formatter = new HelpFormatter();
> + formatter.printHelp("ReprUtilFixer", options);
> + return -1;
> + }
> +
> + // create paths for all of the segments specified, multiple segments may
> + // be run at once
> + String crawlDb = line.getOptionValue("crawldb");
> + String[] segments = line.getOptionValues("segment");
> + Path[] segPaths = new Path[segments != null ? segments.length : 0];
> + if (segments != null) {
> + for (int i = 0; i < segments.length; i++) {
> + segPaths[i] = new Path(segments[i]);
> + }
> + }
> + update(new Path(crawlDb), segPaths);
> + return 0;
> + }
> + catch (Exception e) {
> + LOG.fatal("ReprUtilFixer: " + StringUtils.stringifyException(e));
> + return -1;
> + }
> + }
> +}
> \ No newline at end of file
>
> Added: lucene/nutch/trunk/src/java/org/apache/nutch/util/FSUtils.java
> URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/util/FSUtils.java?rev=723441&view=auto
> ==============================================================================
> --- lucene/nutch/trunk/src/java/org/apache/nutch/util/FSUtils.java (added)
> +++ lucene/nutch/trunk/src/java/org/apache/nutch/util/FSUtils.java Thu Dec 4 13:16:42 2008
> @@ -0,0 +1,83 @@
> +package org.apache.nutch.util;
> +
> +import java.io.IOException;
> +
> +import org.apache.hadoop.fs.FileSystem;
> +import org.apache.hadoop.fs.Path;
> +import org.apache.hadoop.io.MapFile;
> +import org.apache.hadoop.io.SequenceFile;
> +
> +/**
> + * Utility methods for common filesystem operations.
> + */
> +public class FSUtils {
> +
> + /**
> + * Replaces the current path with the new path and if set removes the old
> + * path. If removeOld is set to false then the old path will be set to the
> + * name current.old.
> + *
> + * @param fs The FileSystem.
> + * @param current The end path, the one being replaced.
> + * @param replacement The path to replace with.
> + * @param removeOld True if we are removing the current path.
> + *
> + * @throws IOException If an error occurs during replacement.
> + */
> + public static void replace(FileSystem fs, Path current, Path replacement,
> + boolean removeOld)
> + throws IOException {
> +
> + // rename any current path to old
> + Path old = new Path(current + ".old");
> + if (fs.exists(current)) {
> + fs.rename(current, old);
> + }
> +
> + // rename the new path to current and remove the old path if needed
> + fs.rename(replacement, current);
> + if (fs.exists(old) && removeOld) {
> + fs.delete(old, true);
> + }
> + }
> +
> + /**
> + * Closes a group of SequenceFile readers.
> + *
> + * @param readers The SequenceFile readers to close.
> + * @throws IOException If an error occurs while closing a reader.
> + */
> + public static void closeReaders(SequenceFile.Reader[] readers)
> + throws IOException {
> +
> + // loop through the readers, closing one by one
> + if (readers != null) {
> + for (int i = 0; i < readers.length; i++) {
> + SequenceFile.Reader reader = readers[i];
> + if (reader != null) {
> + reader.close();
> + }
> + }
> + }
> + }
> +
> + /**
> + * Closes a group of MapFile readers.
> + *
> + * @param readers The MapFile readers to close.
> + * @throws IOException If an error occurs while closing a reader.
> + */
> + public static void closeReaders(MapFile.Reader[] readers)
> + throws IOException {
> +
> + // loop through the readers closing one by one
> + if (readers != null) {
> + for (int i = 0; i < readers.length; i++) {
> + MapFile.Reader reader = readers[i];
> + if (reader != null) {
> + reader.close();
> + }
> + }
> + }
> + }
> +}
>
> Modified: lucene/nutch/trunk/src/java/org/apache/nutch/util/URLUtil.java
> URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/util/URLUtil.java?rev=723441&r1=723440&r2=723441&view=diff
> ==============================================================================
> --- lucene/nutch/trunk/src/java/org/apache/nutch/util/URLUtil.java (original)
> +++ lucene/nutch/trunk/src/java/org/apache/nutch/util/URLUtil.java Thu Dec 4 13:16:42 2008
> @@ -141,57 +141,160 @@
> return getHostSegments(new URL(url));
> }
>
> - /** Given two urls (source and destination of the redirect),
> - * returns the representative one.
> - *
> - * <p>Implements the algorithm described here:
> + /**
> + * <p>Given two urls, a src and a destination of a redirect, it returns the
> + * representative url.<p>
> + *
> + * <p>This method implements an extended version of the algorithm used by the
> + * Yahoo! Slurp crawler described here:<br>
> + * <a href=
> + * "http://help.yahoo.com/l/nz/yahooxtra/search/webcrawler/slurp-11.html"> How
> + * does the Yahoo! webcrawler handle redirects?</a> <br>
> * <br>
> - * <a href="http://help.yahoo.com/l/nz/yahooxtra/search/webcrawler/slurp-11.html">
> - * How does the Yahoo! webcrawler handle redirects?</a>
> - * <br><br>
> - * The algorithm is as follows:
> * <ol>
> - * <li>Choose target url if either url is malformed.</li>
> - * <li>When a page in one domain redirects to a page in another domain,
> - * choose the "target" URL.</li>
> - * <li>When a top-level page in a domain presents a permanent redirect
> - * to a page deep within the same domain, choose the "source" URL.</li>
> - * <li>When a page deep within a domain presents a permanent redirect
> - * to a page deep within the same domain, choose the "target" URL.</li>
> - * <li>When a page in a domain presents a temporary redirect to
> - * another page in the same domain, choose the "source" URL.<li>
> - * <ol>
> - * </p>
> - *
> - * @param src Source url of redirect
> - * @param dst Destination url of redirect
> - * @param temp Flag to indicate if redirect is temporary
> - * @return Representative url (either src or dst)
> + * <li>Choose target url if either url is malformed.</li>
> + * <li>If different domains the keep the destination whether or not the
> + * redirect is temp or perm</li>
> + * <ul><li>a.com -> b.com*</li></ul>
> + * <li>If the redirect is permanent and the source is root, keep the source.</li>
> + * <ul><li>*a.com -> a.com?y=1 || *a.com -> a.com/xyz/index.html</li></ul>
> + * <li>If the redirect is permanent and the source is not root and the
> + * destination is root, keep the destination</li>
> + * <ul><li>a.com/xyz/index.html -> a.com*</li></ul>
> + * <li>If the redirect is permanent and neither the source nor the destination
> + * is root, then keep the destination</li>
> + * <ul><li>a.com/xyz/index.html -> a.com/abc/page.html*</li></ul>
> + * <li>If the redirect is temporary and source is root and destination is not
> + * root, then keep the source</li>
> + * <ul><li>*a.com -> a.com/xyz/index.html</li></ul>
> + * <li>If the redirect is temporary and source is not root and destination is
> + * root, then keep the destination</li>
> + * <ul><li>a.com/xyz/index.html -> a.com*</li></ul>
> + * <li>If the redirect is temporary and neither the source or the destination
> + * is root, then keep the shortest url. First check for the shortest host,
> + * and if both are equal then check by path. Path is first by length then by
> + * the number of / path separators.</li>
> + * <ul>
> + * <li>a.com/xyz/index.html -> a.com/abc/page.html*</li>
> + * <li>*www.a.com/xyz/index.html -> www.news.a.com/xyz/index.html</li>
> + * </ul>
> + * <li>If the redirect is temporary and both the source and the destination
> + * are root, then keep the shortest sub-domain</li>
> + * <ul><li>*www.a.com -> www.news.a.com</li></ul>
> + * <br>
> + * While not in this logic there is a further piece of representative url
> + * logic that occurs during indexing and after scoring. During creation of
> + * the basic fields before indexing, if a url has a representative url stored
> + * we check both the url and its representative url (which should never be
> + * the same) against their linkrank scores and the highest scoring one is
> + * kept as the url and the lower scoring one is held as the orig url inside
> + * of the index.
> + *
> + * @param src The source url.
> + * @param dst The destination url.
> + * @param temp Is the redirect a temporary redirect.
> + *
> + * @return String The representative url.
> */
> public static String chooseRepr(String src, String dst, boolean temp) {
> +
> + // validate both are well formed urls
> URL srcUrl;
> URL dstUrl;
> try {
> srcUrl = new URL(src);
> dstUrl = new URL(dst);
> - } catch (MalformedURLException e) {
> + }
> + catch (MalformedURLException e) {
> return dst;
> }
>
> + // get the source and destination domain, host, and page
> String srcDomain = URLUtil.getDomainName(srcUrl);
> String dstDomain = URLUtil.getDomainName(dstUrl);
> + String srcHost = srcUrl.getHost();
> + String dstHost = dstUrl.getHost();
> + String srcFile = srcUrl.getFile();
> + String dstFile = dstUrl.getFile();
> +
> + // are the source and destination the root path url.com/ or url.com
> + boolean srcRoot = (srcFile.equals("/") || srcFile.length() == 0);
> + boolean destRoot = (dstFile.equals("/") || dstFile.length() == 0);
> +
> + // 1) different domain them keep dest, temp or perm
> + // a.com -> b.com*
> + //
> + // 2) permanent and root, keep src
> + // *a.com -> a.com?y=1 || *a.com -> a.com/xyz/index.html
> + //
> + // 3) permanent and not root and dest root, keep dest
> + // a.com/xyz/index.html -> a.com*
> + //
> + // 4) permanent and neither root keep dest
> + // a.com/xyz/index.html -> a.com/abc/page.html*
> + //
> + // 5) temp and root and dest not root keep src
> + // *a.com -> a.com/xyz/index.html
> + //
> + // 7) temp and not root and dest root keep dest
> + // a.com/xyz/index.html -> a.com*
> + //
> + // 8) temp and neither root, keep shortest, if hosts equal by path else by
> + // hosts. paths are first by length then by number of / separators
> + // a.com/xyz/index.html -> a.com/abc/page.html*
> + // *www.a.com/xyz/index.html -> www.news.a.com/xyz/index.html
> + //
> + // 9) temp and both root keep shortest sub domain
> + // *www.a.com -> www.news.a.com
>
> + // if we are dealing with a redirect from one domain to another keep the
> + // destination
> if (!srcDomain.equals(dstDomain)) {
> return dst;
> }
>
> - String srcFile = srcUrl.getFile();
> -
> - if (!temp && srcFile.equals("/")) {
> - return src;
> + // if it is a permanent redirect
> + if (!temp) {
> +
> + // if source is root return source, otherwise destination
> + if (srcRoot) {
> + return src;
> + }
> + else {
> + return dst;
> + }
> + }
> + else { // temporary redirect
> +
> + // source root and destination not root
> + if (srcRoot && !destRoot) {
> + return src;
> + }
> + else if (!srcRoot && destRoot) { // destination root and source not
> + return dst;
> + }
> + else if (!srcRoot && !destRoot && (srcHost.equals(dstHost))) {
> +
> + // source and destination hosts are the same, check paths, host length
> + int numSrcPaths = srcFile.split("/").length;
> + int numDstPaths = dstFile.split("/").length;
> + if (numSrcPaths != numDstPaths) {
> + return (numDstPaths < numSrcPaths ? dst : src);
> + }
> + else {
> + int srcPathLength = srcFile.length();
> + int dstPathLength = dstFile.length();
> + return (dstPathLength < srcPathLength ? dst : src);
> + }
> + }
> + else {
> +
> + // different host names and both root take the shortest
> + int numSrcSubs = srcHost.split("\\.").length;
> + int numDstSubs = dstHost.split("\\.").length;
> + return (numDstSubs < numSrcSubs ? dst : src);
> + }
> }
> -
> - return temp ? src : dst;
> }
>
> /**
>
> Modified: lucene/nutch/trunk/src/plugin/build.xml
> URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/build.xml?rev=723441&r1=723440&r2=723441&view=diff
> ==============================================================================
> --- lucene/nutch/trunk/src/plugin/build.xml (original)
> +++ lucene/nutch/trunk/src/plugin/build.xml Thu Dec 4 13:16:42 2008
> @@ -66,6 +66,7 @@
> <ant dir="query-site" target="deploy"/>
> <ant dir="query-url" target="deploy"/>
> <ant dir="scoring-opic" target="deploy"/>
> + <ant dir="scoring-link" target="deploy"/>
> <ant dir="summary-basic" target="deploy"/>
> <ant dir="subcollection" target="deploy"/>
> <ant dir="summary-lucene" target="deploy"/>
> @@ -160,6 +161,7 @@
> <ant dir="query-site" target="clean"/>
> <ant dir="query-url" target="clean"/>
> <ant dir="scoring-opic" target="clean"/>
> + <ant dir="scoring-link" target="clean"/>
> <ant dir="subcollection" target="clean"/>
> <ant dir="summary-basic" target="clean"/>
> <ant dir="summary-lucene" target="clean"/>
>
> Added: lucene/nutch/trunk/src/plugin/scoring-link/build.xml
> URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/scoring-link/build.xml?rev=723441&view=auto
> ==============================================================================
> --- lucene/nutch/trunk/src/plugin/scoring-link/build.xml (added)
> +++ lucene/nutch/trunk/src/plugin/scoring-link/build.xml Thu Dec 4 13:16:42 2008
> @@ -0,0 +1,27 @@
> +<?xml version="1.0"?>
> +<!--
> + Licensed to the Apache Software Foundation (ASF) under one or more
> + contributor license agreements. See the NOTICE file distributed with
> + this work for additional information regarding copyright ownership.
> + The ASF licenses this file to You under the Apache License, Version 2.0
> + (the "License"); you may not use this file except in compliance with
> + the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> +-->
> +<project name="scoring-link" default="jar-core">
> +
> + <import file="../build-plugin.xml"/>
> +
> + <!-- Deploy Unit test dependencies -->
> + <target name="deps-test">
> + <ant target="deploy" inheritall="false" dir="../nutch-extensionpoints"/>
> + </target>
> +
> +</project>
>
> Added: lucene/nutch/trunk/src/plugin/scoring-link/plugin.xml
> URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/scoring-link/plugin.xml?rev=723441&view=auto
> ==============================================================================
> --- lucene/nutch/trunk/src/plugin/scoring-link/plugin.xml (added)
> +++ lucene/nutch/trunk/src/plugin/scoring-link/plugin.xml Thu Dec 4 13:16:42 2008
> @@ -0,0 +1,39 @@
> +<?xml version="1.0" encoding="UTF-8"?>
> +<!--
> + Licensed to the Apache Software Foundation (ASF) under one or more
> + contributor license agreements. See the NOTICE file distributed with
> + this work for additional information regarding copyright ownership.
> + The ASF licenses this file to You under the Apache License, Version 2.0
> + (the "License"); you may not use this file except in compliance with
> + the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> +-->
> +<plugin
> + id="scoring-link"
> + name="Link Analysis Scoring Plug-in"
> + version="1.0.0"
> + provider-name="nutch.org">
> +
> +
> + <runtime>
> + <library name="scoring-link.jar">
> + <export name="*"/>
> + </library>
> + </runtime>
> +
> + <extension id="org.apache.nutch.scoring.link"
> + name="LinkAnalysisScoring"
> + point="org.apache.nutch.scoring.ScoringFilter">
> +
> + <implementation id="org.apache.nutch.scoring.link.LinkAnalysisScoringFilter"
> + class="org.apache.nutch.scoring.link.LinkAnalysisScoringFilter" />
> + </extension>
> +
> +</plugin>
>
> Added: lucene/nutch/trunk/src/plugin/scoring-link/src/java/org/apache/nutch/scoring/link/LinkAnalysisScoringFilter.java
> URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/scoring-link/src/java/org/apache/nutch/scoring/link/LinkAnalysisScoringFilter.java?rev=723441&view=auto
> ==============================================================================
> --- lucene/nutch/trunk/src/plugin/scoring-link/src/java/org/apache/nutch/scoring/link/LinkAnalysisScoringFilter.java (added)
> +++ lucene/nutch/trunk/src/plugin/scoring-link/src/java/org/apache/nutch/scoring/link/LinkAnalysisScoringFilter.java Thu Dec 4 13:16:42 2008
> @@ -0,0 +1,85 @@
> +package org.apache.nutch.scoring.link;
> +
> +import java.util.Collection;
> +import java.util.List;
> +import java.util.Map.Entry;
> +
> +import org.apache.hadoop.conf.Configuration;
> +import org.apache.hadoop.io.Text;
> +import org.apache.lucene.document.Document;
> +import org.apache.nutch.crawl.CrawlDatum;
> +import org.apache.nutch.crawl.Inlinks;
> +import org.apache.nutch.metadata.Nutch;
> +import org.apache.nutch.parse.Parse;
> +import org.apache.nutch.parse.ParseData;
> +import org.apache.nutch.protocol.Content;
> +import org.apache.nutch.scoring.ScoringFilter;
> +import org.apache.nutch.scoring.ScoringFilterException;
> +
> +public class LinkAnalysisScoringFilter
> + implements ScoringFilter {
> +
> + private Configuration conf;
> + private float scoreInjected = 0.001f;
> + private float normalizedScore = 1.00f;
> +
> + public LinkAnalysisScoringFilter() {
> +
> + }
> +
> + public Configuration getConf() {
> + return conf;
> + }
> +
> + public void setConf(Configuration conf) {
> + this.conf = conf;
> + normalizedScore = conf.getFloat("link.analyze.normalize.score", 1.00f);
> + scoreInjected = conf.getFloat("link.analyze.injected.score", 1.00f);
> + }
> +
> + public CrawlDatum distributeScoreToOutlinks(Text fromUrl,
> + ParseData parseData, Collection<Entry<Text, CrawlDatum>> targets,
> + CrawlDatum adjust, int allCount)
> + throws ScoringFilterException {
> + return adjust;
> + }
> +
> + public float generatorSortValue(Text url, CrawlDatum datum, float initSort)
> + throws ScoringFilterException {
> + return datum.getScore() * initSort;
> + }
> +
> + public float indexerScore(Text url, Document doc, CrawlDatum dbDatum,
> + CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore)
> + throws ScoringFilterException {
> + return (normalizedScore * dbDatum.getScore());
> + }
> +
> + public void initialScore(Text url, CrawlDatum datum)
> + throws ScoringFilterException {
> + datum.setScore(0.0f);
> + }
> +
> + public void injectedScore(Text url, CrawlDatum datum)
> + throws ScoringFilterException {
> + datum.setScore(scoreInjected);
> + }
> +
> + public void passScoreAfterParsing(Text url, Content content, Parse parse)
> + throws ScoringFilterException {
> + parse.getData().getContentMeta().set(Nutch.SCORE_KEY,
> + content.getMetadata().get(Nutch.SCORE_KEY));
> + }
> +
> + public void passScoreBeforeParsing(Text url, CrawlDatum datum, Content content)
> + throws ScoringFilterException {
> + content.getMetadata().set(Nutch.SCORE_KEY, "" + datum.getScore());
> + }
> +
> + public void updateDbScore(Text url, CrawlDatum old, CrawlDatum datum,
> + List<CrawlDatum> inlinked)
> + throws ScoringFilterException {
> + // nothing to do
> + }
> +
> +}
>
> Modified: lucene/nutch/trunk/src/test/org/apache/nutch/util/TestURLUtil.java
> URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/test/org/apache/nutch/util/TestURLUtil.java?rev=723441&r1=723440&r2=723441&view=diff
> ==============================================================================
> --- lucene/nutch/trunk/src/test/org/apache/nutch/util/TestURLUtil.java (original)
> +++ lucene/nutch/trunk/src/test/org/apache/nutch/util/TestURLUtil.java Thu Dec 4 13:16:42 2008
> @@ -22,14 +22,17 @@
> import junit.framework.TestCase;
>
> /** Test class for URLUtil */
> -public class TestURLUtil extends TestCase {
> +public class TestURLUtil
> + extends TestCase {
>
> @Override
> - protected void setUp() throws Exception {
> + protected void setUp()
> + throws Exception {
> super.setUp();
> }
>
> - public void testGetDomainName() throws Exception{
> + public void testGetDomainName()
> + throws Exception {
>
> URL url = null;
>
> @@ -51,7 +54,7 @@
> url = new URL("http://www.example.co.uk.com");
> assertEquals("uk.com", URLUtil.getDomainName(url));
>
> - //"nn" is not a tld
> + // "nn" is not a tld
> url = new URL("http://example.com.nn");
> assertEquals("nn", URLUtil.getDomainName(url));
>
> @@ -60,25 +63,26 @@
>
> url = new URL("http://www.edu.tr.xyz");
> assertEquals("xyz", URLUtil.getDomainName(url));
> -
> +
> url = new URL("http://www.example.c.se");
> assertEquals("example.c.se", URLUtil.getDomainName(url));
>
> - //plc.co.im is listed as a domain suffix
> + // plc.co.im is listed as a domain suffix
> url = new URL("http://www.example.plc.co.im");
> assertEquals("example.plc.co.im", URLUtil.getDomainName(url));
> -
> - //2000.hu is listed as a domain suffix
> +
> + // 2000.hu is listed as a domain suffix
> url = new URL("http://www.example.2000.hu");
> assertEquals("example.2000.hu", URLUtil.getDomainName(url));
> -
> - //test non-ascii
> +
> + // test non-ascii
> url = new URL("http://www.example.åæ¥.tw");
> assertEquals("example.åæ¥.tw", URLUtil.getDomainName(url));
> -
> +
> }
>
> - public void testGetDomainSuffix() throws Exception{
> + public void testGetDomainSuffix()
> + throws Exception {
> URL url = null;
>
> url = new URL("http://lucene.apache.org/nutch");
> @@ -96,7 +100,7 @@
> url = new URL("http://www.example.co.uk.com");
> assertEquals("com", URLUtil.getDomainSuffix(url).getDomain());
>
> - //"nn" is not a tld
> + // "nn" is not a tld
> url = new URL("http://example.com.nn");
> assertNull(URLUtil.getDomainSuffix(url));
>
> @@ -105,59 +109,108 @@
>
> url = new URL("http://www.edu.tr.xyz");
> assertNull(URLUtil.getDomainSuffix(url));
> -
> +
> url = new URL("http://subdomain.example.edu.tr");
> assertEquals("edu.tr", URLUtil.getDomainSuffix(url).getDomain());
> -
> +
> url = new URL("http://subdomain.example.presse.fr");
> assertEquals("presse.fr", URLUtil.getDomainSuffix(url).getDomain());
> -
> +
> url = new URL("http://subdomain.example.presse.tr");
> assertEquals("tr", URLUtil.getDomainSuffix(url).getDomain());
> -
> - //plc.co.im is listed as a domain suffix
> +
> + // plc.co.im is listed as a domain suffix
> url = new URL("http://www.example.plc.co.im");
> assertEquals("plc.co.im", URLUtil.getDomainSuffix(url).getDomain());
> -
> - //2000.hu is listed as a domain suffix
> +
> + // 2000.hu is listed as a domain suffix
> url = new URL("http://www.example.2000.hu");
> assertEquals("2000.hu", URLUtil.getDomainSuffix(url).getDomain());
> -
> - //test non-ascii
> +
> + // test non-ascii
> url = new URL("http://www.example.åæ¥.tw");
> assertEquals("åæ¥.tw", URLUtil.getDomainSuffix(url).getDomain());
> -
> +
> }
> -
> - public void testGetHostSegments() throws Exception{
> +
> + public void testGetHostSegments()
> + throws Exception {
> URL url;
> String[] segments;
> -
> +
> url = new URL("http://subdomain.example.edu.tr");
> segments = URLUtil.getHostSegments(url);
> assertEquals("subdomain", segments[0]);
> assertEquals("example", segments[1]);
> assertEquals("edu", segments[2]);
> assertEquals("tr", segments[3]);
> -
> +
> url = new URL("http://");
> segments = URLUtil.getHostSegments(url);
> assertEquals(1, segments.length);
> assertEquals("", segments[0]);
> -
> +
> url = new URL("http://140.211.11.130/foundation/contributing.html");
> segments = URLUtil.getHostSegments(url);
> assertEquals(1, segments.length);
> assertEquals("140.211.11.130", segments[0]);
> -
> - //test non-ascii
> +
> + // test non-ascii
> url = new URL("http://www.example.åæ¥.tw");
> segments = URLUtil.getHostSegments(url);
> assertEquals("www", segments[0]);
> assertEquals("example", segments[1]);
> assertEquals("åæ¥", segments[2]);
> assertEquals("tw", segments[3]);
> +
> + }
> +
> + public void testChooseRepr()
> + throws Exception {
>
> + String aDotCom = "http://www.a.com";
> + String bDotCom = "http://www.b.com";
> + String aSubDotCom = "http://www.news.a.com";
> + String aQStr = "http://www.a.com?y=1";
> + String aPath = "http://www.a.com/xyz/index.html";
> + String aPath2 = "http://www.a.com/abc/page.html";
> + String aPath3 = "http://www.news.a.com/abc/page.html";
> +
> + // 1) different domain them keep dest, temp or perm
> + // a.com -> b.com*
> + assertEquals(bDotCom, URLUtil.chooseRepr(aDotCom, bDotCom, true));
> + assertEquals(bDotCom, URLUtil.chooseRepr(aDotCom, bDotCom, false));
> +
> + // 2) permanent and root, keep src
> + // *a.com -> a.com?y=1 || *a.com -> a.com/xyz/index.html
> + assertEquals(aDotCom, URLUtil.chooseRepr(aDotCom, aQStr, false));
> + assertEquals(aDotCom, URLUtil.chooseRepr(aDotCom, aPath, false));
> +
> + //3) permanent and not root and dest root, keep dest
> + //a.com/xyz/index.html -> a.com*
> + assertEquals(aDotCom, URLUtil.chooseRepr(aPath, aDotCom, false));
> +
> + //4) permanent and neither root keep dest
> + // a.com/xyz/index.html -> a.com/abc/page.html*
> + assertEquals(aPath2, URLUtil.chooseRepr(aPath, aPath2, false));
> +
> + //5) temp and root and dest not root keep src
> + //*a.com -> a.com/xyz/index.html
> + assertEquals(aDotCom, URLUtil.chooseRepr(aDotCom, aPath, true));
> +
> + //6) temp and not root and dest root keep dest
> + // a.com/xyz/index.html -> a.com*
> + assertEquals(aDotCom, URLUtil.chooseRepr(aPath, aDotCom, true));
> +
> + //7) temp and neither root, keep shortest, if hosts equal by path else by hosts
> + // a.com/xyz/index.html -> a.com/abc/page.html*
> + // *www.a.com/xyz/index.html -> www.news.a.com/xyz/index.html
> + assertEquals(aPath2, URLUtil.chooseRepr(aPath, aPath2, true));
> + assertEquals(aPath, URLUtil.chooseRepr(aPath, aPath3, true));
> +
> + //8) temp and both root keep shortest sub domain
> + // *www.a.com -> www.news.a.com
> + assertEquals(aDotCom, URLUtil.chooseRepr(aDotCom, aSubDotCom, true));
> }
>
> }
>
>
>