You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by fe...@apache.org on 2011/11/16 16:16:11 UTC
svn commit: r1202736 - in /nutch/branches/nutchgora/src:
java/org/apache/nutch/crawl/ java/org/apache/nutch/util/
test/org/apache/nutch/crawl/
Author: ferdy
Date: Wed Nov 16 15:16:10 2011
New Revision: 1202736
URL: http://svn.apache.org/viewvc?rev=1202736&view=rev
Log:
NUTCH-1196 Update job should impose an upper limit on the number of inlinks (nutchgora)
Added:
nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/UrlWithScore.java (with props)
nutch/branches/nutchgora/src/test/org/apache/nutch/crawl/TestUrlWithScore.java (with props)
Modified:
nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdateMapper.java
nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdateReducer.java
nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
nutch/branches/nutchgora/src/java/org/apache/nutch/util/WebPageWritable.java
Modified: nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdateMapper.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdateMapper.java?rev=1202736&r1=1202735&r2=1202736&view=diff
==============================================================================
--- nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdateMapper.java (original)
+++ nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdateMapper.java Wed Nov 16 15:16:10 2011
@@ -24,6 +24,7 @@ import java.util.Map.Entry;
import org.apache.avro.util.Utf8;
import org.slf4j.Logger;
+import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.util.StringUtils;
import org.apache.nutch.scoring.ScoreDatum;
import org.apache.nutch.scoring.ScoringFilterException;
@@ -34,12 +35,17 @@ import org.apache.nutch.util.WebPageWrit
import org.apache.gora.mapreduce.GoraMapper;
public class DbUpdateMapper
-extends GoraMapper<String, WebPage, String, NutchWritable> {
+extends GoraMapper<String, WebPage, UrlWithScore, NutchWritable> {
public static final Logger LOG = DbUpdaterJob.LOG;
private ScoringFilters scoringFilters;
private final List<ScoreDatum> scoreData = new ArrayList<ScoreDatum>();
+
+ //reuse writables
+ private UrlWithScore urlWithScore = new UrlWithScore();
+ private NutchWritable nutchWritable = new NutchWritable();
+ private WebPageWritable pageWritable;
@Override
public void map(String key, WebPage page, Context context)
@@ -63,19 +69,26 @@ extends GoraMapper<String, WebPage, Stri
" exception:" + StringUtils.stringifyException(e));
}
- context.write(key,
- new NutchWritable(new WebPageWritable(context.getConfiguration(), page)));
+ urlWithScore.setUrl(key);
+ urlWithScore.setScore(Float.MAX_VALUE);
+ pageWritable.setWebPage(page);
+ nutchWritable.set(pageWritable);
+ context.write(urlWithScore, nutchWritable);
for (ScoreDatum scoreDatum : scoreData) {
String reversedOut = TableUtil.reverseUrl(scoreDatum.getUrl());
scoreDatum.setUrl(url);
- context.write(reversedOut, new NutchWritable(scoreDatum));
+ urlWithScore.setUrl(reversedOut);
+ urlWithScore.setScore(scoreDatum.getScore());
+ nutchWritable.set(scoreDatum);
+ context.write(urlWithScore, nutchWritable);
}
}
@Override
public void setup(Context context) {
scoringFilters = new ScoringFilters(context.getConfiguration());
+ pageWritable = new WebPageWritable(context.getConfiguration(), null);
}
}
Modified: nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdateReducer.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdateReducer.java?rev=1202736&r1=1202735&r2=1202736&view=diff
==============================================================================
--- nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdateReducer.java (original)
+++ nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdateReducer.java Wed Nov 16 15:16:10 2011
@@ -37,7 +37,7 @@ import org.apache.nutch.util.WebPageWrit
import org.apache.gora.mapreduce.GoraReducer;
public class DbUpdateReducer
-extends GoraReducer<String, NutchWritable, String, WebPage> {
+extends GoraReducer<UrlWithScore, NutchWritable, String, WebPage> {
public static final String CRAWLDB_ADDITIONS_ALLOWED = "db.update.additions.allowed";
@@ -49,6 +49,7 @@ extends GoraReducer<String, NutchWritabl
private FetchSchedule schedule;
private ScoringFilters scoringFilters;
private List<ScoreDatum> inlinkedScoreData = new ArrayList<ScoreDatum>();
+ private int maxLinks;
@Override
protected void setup(Context context) throws IOException, InterruptedException {
@@ -58,26 +59,32 @@ extends GoraReducer<String, NutchWritabl
maxInterval = conf.getInt("db.fetch.interval.max", 0 );
schedule = FetchScheduleFactory.getFetchSchedule(conf);
scoringFilters = new ScoringFilters(conf);
+ maxLinks = conf.getInt("db.update.max.inlinks", 10000);
}
@Override
- protected void reduce(String key, Iterable<NutchWritable> values,
+ protected void reduce(UrlWithScore key, Iterable<NutchWritable> values,
Context context) throws IOException, InterruptedException {
+ String keyUrl = key.getUrl().toString();
WebPage page = null;
inlinkedScoreData.clear();
-
+
for (NutchWritable nutchWritable : values) {
Writable val = nutchWritable.get();
if (val instanceof WebPageWritable) {
page = ((WebPageWritable) val).getWebPage();
} else {
inlinkedScoreData.add((ScoreDatum) val);
+ if (inlinkedScoreData.size() >= maxLinks) {
+ LOG.info("Limit reached, skipping further inlinks for " + keyUrl);
+ break;
+ }
}
}
String url;
try {
- url = TableUtil.unreverseUrl(key);
+ url = TableUtil.unreverseUrl(keyUrl);
} catch (Exception e) {
// this can happen because a newly discovered malformed link
// may slip by url filters
@@ -177,7 +184,7 @@ extends GoraReducer<String, NutchWritabl
Mark.UPDATEDB_MARK.putMark(page, mark);
}
- context.write(key, page);
+ context.write(keyUrl, page);
}
}
Modified: nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdaterJob.java?rev=1202736&r1=1202735&r2=1202736&view=diff
==============================================================================
--- nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdaterJob.java (original)
+++ nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/DbUpdaterJob.java Wed Nov 16 15:16:10 2011
@@ -16,21 +16,16 @@
******************************************************************************/
package org.apache.nutch.crawl;
-import java.io.IOException;
import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.RawComparator;
-import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.UrlWithScore.UrlOnlyPartitioner;
+import org.apache.nutch.crawl.UrlWithScore.UrlScoreComparator;
+import org.apache.nutch.crawl.UrlWithScore.UrlScoreComparator.UrlOnlyComparator;
import org.apache.nutch.metadata.Nutch;
import org.apache.nutch.scoring.ScoringFilters;
import org.apache.nutch.storage.StorageUtils;
@@ -39,7 +34,8 @@ import org.apache.nutch.util.NutchConfig
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.NutchTool;
import org.apache.nutch.util.ToolUtil;
-import org.apache.gora.mapreduce.StringComparator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class DbUpdaterJob extends NutchTool implements Tool {
@@ -85,10 +81,16 @@ public class DbUpdaterJob extends NutchT
ScoringFilters scoringFilters = new ScoringFilters(getConf());
HashSet<WebPage.Field> fields = new HashSet<WebPage.Field>(FIELDS);
fields.addAll(scoringFilters.getFields());
- // TODO: Figure out why this needs to be here
- currentJob.getConfiguration().setClass("mapred.output.key.comparator.class",
- StringComparator.class, RawComparator.class);
- StorageUtils.initMapperJob(currentJob, fields, String.class,
+
+ // Partition by {url}, sort by {url,score} and group by {url}.
+ // This ensures that the inlinks are sorted by score when they enter
+ // the reducer.
+
+ currentJob.setPartitionerClass(UrlOnlyPartitioner.class);
+ currentJob.setSortComparatorClass(UrlScoreComparator.class);
+ currentJob.setGroupingComparatorClass(UrlOnlyComparator.class);
+
+ StorageUtils.initMapperJob(currentJob, fields, UrlWithScore.class,
NutchWritable.class, DbUpdateMapper.class);
StorageUtils.initReducerJob(currentJob, DbUpdateReducer.class);
currentJob.waitForCompletion(true);
Added: nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/UrlWithScore.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/UrlWithScore.java?rev=1202736&view=auto
==============================================================================
--- nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/UrlWithScore.java (added)
+++ nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/UrlWithScore.java Wed Nov 16 15:16:10 2011
@@ -0,0 +1,196 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.nutch.crawl;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Comparator;
+
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapreduce.Partitioner;
+
+/**
+ * A writable comparable container for an url with score. Provides a
+ * {@link Partitioner} and {@link RawComparator} implementations for easy
+ * integration with jobs.
+ *
+ */
+public final class UrlWithScore implements WritableComparable<UrlWithScore> {
+ private static final Comparator<UrlWithScore> comp = new UrlScoreComparator();
+
+ private Text url;
+ private FloatWritable score;
+
+ /**
+ * Creates instance with empty url and zero score.
+ */
+ public UrlWithScore() {
+ url = new Text();
+ score = new FloatWritable();
+ }
+
+ /**
+ * Creates instance with provided writables.
+ *
+ * @param url
+ * @param score
+ */
+ public UrlWithScore(Text url, FloatWritable score) {
+ this.url = url;
+ this.score = score;
+ }
+
+ /**
+ * Creates instance with provided non-writable types.
+ *
+ * @param url
+ * @param score
+ */
+ public UrlWithScore(String url, float score) {
+ this.url = new Text(url);
+ this.score = new FloatWritable(score);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ url.write(out);
+ score.write(out);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ url.readFields(in);
+ score.readFields(in);
+ }
+
+ public Text getUrl() {
+ return url;
+ }
+
+ public void setUrl(Text url) {
+ this.url = url;
+ }
+
+ public void setUrl(String url) {
+ this.url.set(url);
+ }
+
+ public FloatWritable getScore() {
+ return score;
+ }
+
+ public void setScore(FloatWritable score) {
+ this.score = score;
+ }
+
+ public void setScore(float score) {
+ this.score.set(score);
+ }
+
+ @Override
+ public int compareTo(UrlWithScore other) {
+ return comp.compare(this, other);
+ }
+
+ @Override
+ public String toString() {
+ return "UrlWithScore [url=" + url + ", score=" + score + "]";
+ }
+
+
+ /**
+ * A partitioner by {url}.
+ */
+ public static final class UrlOnlyPartitioner extends
+ Partitioner<UrlWithScore, NutchWritable> {
+ @Override
+ public int getPartition(UrlWithScore key, NutchWritable val, int reduces) {
+ return (key.url.hashCode() & Integer.MAX_VALUE) % reduces;
+ }
+ }
+
+ /**
+ * Compares by {url,score}. Scores are sorted in descending order, that is
+ * from high scores to low.
+ */
+ public static final class UrlScoreComparator implements
+ RawComparator<UrlWithScore> {
+ private final WritableComparator textComp = new Text.Comparator();
+ private final WritableComparator floatComp = new FloatWritable.Comparator();
+
+ @Override
+ public int compare(UrlWithScore o1, UrlWithScore o2) {
+ int cmp = o1.getUrl().compareTo(o2.getUrl());
+ if (cmp != 0) {
+ return cmp;
+ }
+ //reverse order
+ return -o1.getScore().compareTo(o2.getScore());
+ }
+
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ try {
+ int deptLen1 = WritableUtils.decodeVIntSize(b1[s1])
+ + WritableComparator.readVInt(b1, s1);
+ int deptLen2 = WritableUtils.decodeVIntSize(b2[s2])
+ + WritableComparator.readVInt(b2, s2);
+ int cmp = textComp.compare(b1, s1, deptLen1, b2, s2, deptLen2);
+ if (cmp != 0) {
+ return cmp;
+ }
+ //reverse order
+ return -floatComp.compare(b1, s1 + deptLen1, l1 - deptLen1,
+ b2, s2 + deptLen2, l2 - deptLen2);
+ } catch (IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ /**
+ * Compares by {url}.
+ */
+ public static final class UrlOnlyComparator implements
+ RawComparator<UrlWithScore> {
+ private final WritableComparator textComp = new Text.Comparator();
+
+ @Override
+ public int compare(UrlWithScore o1, UrlWithScore o2) {
+ return o1.getUrl().compareTo(o2.getUrl());
+ }
+
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ try {
+ int deptLen1 = WritableUtils.decodeVIntSize(b1[s1])
+ + WritableComparator.readVInt(b1, s1);
+ int deptLen2 = WritableUtils.decodeVIntSize(b2[s2])
+ + WritableComparator.readVInt(b2, s2);
+ return textComp.compare(b1, s1, deptLen1, b2, s2, deptLen2);
+ } catch (IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
Propchange: nutch/branches/nutchgora/src/java/org/apache/nutch/crawl/UrlWithScore.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: nutch/branches/nutchgora/src/java/org/apache/nutch/util/WebPageWritable.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchgora/src/java/org/apache/nutch/util/WebPageWritable.java?rev=1202736&r1=1202735&r2=1202736&view=diff
==============================================================================
--- nutch/branches/nutchgora/src/java/org/apache/nutch/util/WebPageWritable.java (original)
+++ nutch/branches/nutchgora/src/java/org/apache/nutch/util/WebPageWritable.java Wed Nov 16 15:16:10 2011
@@ -53,5 +53,9 @@ implements Writable {
public WebPage getWebPage() {
return webPage;
}
+
+ public void setWebPage(WebPage webPage) {
+ this.webPage = webPage;
+ }
}
Added: nutch/branches/nutchgora/src/test/org/apache/nutch/crawl/TestUrlWithScore.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchgora/src/test/org/apache/nutch/crawl/TestUrlWithScore.java?rev=1202736&view=auto
==============================================================================
--- nutch/branches/nutchgora/src/test/org/apache/nutch/crawl/TestUrlWithScore.java (added)
+++ nutch/branches/nutchgora/src/test/org/apache/nutch/crawl/TestUrlWithScore.java Wed Nov 16 15:16:10 2011
@@ -0,0 +1,180 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.nutch.crawl;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.io.RawComparator;
+import org.apache.nutch.crawl.UrlWithScore;
+import org.apache.nutch.crawl.UrlWithScore.UrlOnlyPartitioner;
+import org.apache.nutch.crawl.UrlWithScore.UrlScoreComparator;
+import org.apache.nutch.crawl.UrlWithScore.UrlScoreComparator.UrlOnlyComparator;
+
+/**
+ * Tests {@link UrlWithScore} with serialization, partitioning and sorting.
+ */
+public class TestUrlWithScore extends TestCase {
+
+ public void testSerialization() throws IOException {
+ // create a key and test basic functionality
+ UrlWithScore keyOut = new UrlWithScore("http://example.org/", 1f);
+ assertEquals("http://example.org/", keyOut.getUrl().toString());
+ assertEquals(1f, keyOut.getScore().get(), 0.001);
+
+ // write to out
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(bos);
+ keyOut.write(out);
+
+ // read from in
+ UrlWithScore keyIn = new UrlWithScore();
+ ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
+ DataInputStream in = new DataInputStream(bis);
+ keyIn.readFields(in);
+ assertEquals(keyOut.getUrl().toString(), keyIn.getUrl().toString());
+ assertEquals(keyOut.getScore().get(), keyIn.getScore().get(), 0.001);
+
+ in.close();
+ out.close();
+ }
+
+ public void testPartitioner() throws IOException {
+ UrlOnlyPartitioner part = new UrlOnlyPartitioner();
+
+ UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
+ UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
+ UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
+ UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
+ UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
+
+ int numReduces = 7;
+
+ // keys 1 and 2 should be partitioned together
+ int partForKey1 = part.getPartition(k1, null, numReduces);
+ assertEquals(partForKey1, part.getPartition(k2, null, numReduces));
+ assertEquals(partForKey1, part.getPartition(k2, null, numReduces));
+
+ // keys 3, 4 and 5 should be partitioned together
+ int partForKey3 = part.getPartition(k3, null, numReduces);
+ assertEquals(partForKey3, part.getPartition(k4, null, numReduces));
+ assertEquals(partForKey3, part.getPartition(k5, null, numReduces));
+ }
+
+ public void testUrlOnlySorting() throws IOException {
+ UrlOnlyComparator comp = new UrlOnlyComparator();
+
+ UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
+ UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
+ UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
+ UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
+ UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
+
+ // k1 should be equal to k2
+ assertEquals(0, compareBothRegularAndRaw(comp, k1, k2));
+ // test symmetry
+ assertEquals(0, compareBothRegularAndRaw(comp, k2, k1));
+
+ // k1 is before k3, k4 and k5
+ assertEquals(-1, compareBothRegularAndRaw(comp, k1, k3));
+ assertEquals(-1, compareBothRegularAndRaw(comp, k1, k4));
+ assertEquals(-1, compareBothRegularAndRaw(comp, k1, k5));
+ // test symmetry
+ assertEquals(1, compareBothRegularAndRaw(comp, k3, k1));
+ assertEquals(1, compareBothRegularAndRaw(comp, k4, k1));
+ assertEquals(1, compareBothRegularAndRaw(comp, k5, k1));
+ }
+
+ public void testUrlScoreSorting() throws IOException {
+ UrlScoreComparator comp = new UrlScoreComparator();
+
+ UrlWithScore k1 = new UrlWithScore("http://example.org/1", 1f);
+ UrlWithScore k2 = new UrlWithScore("http://example.org/1", 2f);
+ UrlWithScore k3 = new UrlWithScore("http://example.org/2", 1f);
+ UrlWithScore k4 = new UrlWithScore("http://example.org/2", 2f);
+ UrlWithScore k5 = new UrlWithScore("http://example.org/2", 3f);
+
+ // k1 is after k2, because score is lower
+ assertEquals(1, comp.compare(k1, k2));
+ // test symmetry
+ assertEquals(-1, comp.compare(k2, k1));
+
+ // k1 is before k3, k4 and k5, because url is lower
+ assertEquals(-1, compareBothRegularAndRaw(comp, k1, k3));
+ assertEquals(-1, compareBothRegularAndRaw(comp, k1, k4));
+ assertEquals(-1, compareBothRegularAndRaw(comp, k1, k5));
+ // test symmetry
+ assertEquals(1, compareBothRegularAndRaw(comp, k3, k1));
+ assertEquals(1, compareBothRegularAndRaw(comp, k4, k1));
+ assertEquals(1, compareBothRegularAndRaw(comp, k5, k1));
+
+ // k3 after k4 and k4 after k5 and therefore k3 after k5 (transitivity)
+ assertEquals(1, compareBothRegularAndRaw(comp, k3, k4));
+ assertEquals(1, compareBothRegularAndRaw(comp, k4, k5));
+ assertEquals(1, compareBothRegularAndRaw(comp, k3, k5));
+ // test symmetry
+ assertEquals(-1, compareBothRegularAndRaw(comp, k4, k3));
+ assertEquals(-1, compareBothRegularAndRaw(comp, k5, k4));
+ assertEquals(-1, compareBothRegularAndRaw(comp, k5, k3));
+ }
+
+ /**
+ * Compares two keys using both regular and raw comparing. Checks if the two
+ * values are equal. Returns the compare value, but only when equal.
+ *
+ * @param comp
+ * @param k1
+ * @param k2
+ * @return The compare result. (When k1 != k2, assert failure kicks in)
+ * @throws IOException
+ */
+ private Object compareBothRegularAndRaw(RawComparator<UrlWithScore> comp,
+ UrlWithScore k1, UrlWithScore k2) throws IOException {
+ int regular = comp.compare(k1, k2);
+
+ byte[] bytes1 = extractBytes(k1);
+ byte[] bytes2 = extractBytes(k2);
+
+ int raw = comp.compare(bytes1, 0, bytes1.length, bytes2, 0, bytes2.length);
+
+ assertEquals("Regular compare should equal raw compare", regular, raw);
+
+ return regular;
+ }
+
+ /**
+ * Return the bytes for a key.
+ *
+ * @param k
+ * @return
+ * @throws IOException
+ */
+ private byte[] extractBytes(UrlWithScore k) throws IOException {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(bos);
+ k.write(out);
+ byte[] bytes = bos.toByteArray();
+ out.close();
+ return bytes;
+ }
+
+}
Propchange: nutch/branches/nutchgora/src/test/org/apache/nutch/crawl/TestUrlWithScore.java
------------------------------------------------------------------------------
svn:mime-type = text/plain