You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ab...@apache.org on 2008/03/19 11:46:04 UTC
svn commit: r638782 - in /lucene/nutch/trunk: ./
src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/
src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/
Author: ab
Date: Wed Mar 19 03:45:55 2008
New Revision: 638782
URL: http://svn.apache.org/viewvc?rev=638782&view=rev
Log:
NUTCH-620 BasicURLNormalizer should collapse runs of slashes with a single slash.
Modified:
lucene/nutch/trunk/CHANGES.txt
lucene/nutch/trunk/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java
lucene/nutch/trunk/src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java
Modified: lucene/nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?rev=638782&r1=638781&r2=638782&view=diff
==============================================================================
--- lucene/nutch/trunk/CHANGES.txt (original)
+++ lucene/nutch/trunk/CHANGES.txt Wed Mar 19 03:45:55 2008
@@ -242,6 +242,9 @@
88. NUTCH-598 - Remove deprecated use of ToolBase. Use generics in Hadoop API.
(Emmanuel Joke, dogacan, ab)
+89. NUTCH-620 - BasicURLNormalizer should collapse runs of slashes with a
+ single slash. (Mark DeSpain via ab)
+
Release 0.9 - 2007-04-02
Modified: lucene/nutch/trunk/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java?rev=638782&r1=638781&r2=638782&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java (original)
+++ lucene/nutch/trunk/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java Wed Mar 19 03:45:55 2008
@@ -43,6 +43,7 @@
};
private Rule relativePathRule = null;
private Rule leadingRelativePathRule = null;
+ private Rule adjacentSlashRule = null;
private Configuration conf;
@@ -64,6 +65,13 @@
compiler.compile("^(/\\.\\./)+", Perl5Compiler.READ_ONLY_MASK);
leadingRelativePathRule.substitution = new Perl5Substitution("/");
+ // this pattern tries to find spots like "xx//yy" in the url,
+ // which could be replaced by a "/"
+ adjacentSlashRule = new Rule();
+ adjacentSlashRule.pattern = (Perl5Pattern)
+ compiler.compile("/{2,}", Perl5Compiler.READ_ONLY_MASK);
+ adjacentSlashRule.substitution = new Perl5Substitution("/");
+
} catch (MalformedPatternException e) {
e.printStackTrace(LogUtil.getWarnStream(LOG));
throw new RuntimeException(e);
@@ -163,6 +171,13 @@
fileWorkCopy = Util.substitute
(matcher, leadingRelativePathRule.pattern,
leadingRelativePathRule.substitution, fileWorkCopy, 1);
+
+
+ // collapse adjacent slashes with "/"
+ fileWorkCopy = Util.substitute
+ (matcher, adjacentSlashRule.pattern,
+ adjacentSlashRule.substitution, fileWorkCopy, 1);
+
newLen = fileWorkCopy.length();
}
Modified: lucene/nutch/trunk/src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java?rev=638782&r1=638781&r2=638782&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java (original)
+++ lucene/nutch/trunk/src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java Wed Mar 19 03:45:55 2008
@@ -90,12 +90,22 @@
"http://foo.com/foo.html" );
normalizeTest("http://foo.com/foo.foo/../foo.html",
"http://foo.com/foo.html" );
+ normalizeTest("http://foo.com//aa/bb/foo.html",
+ "http://foo.com/aa/bb/foo.html" );
+ normalizeTest("http://foo.com/aa//bb/foo.html",
+ "http://foo.com/aa/bb/foo.html" );
+ normalizeTest("http://foo.com/aa/bb//foo.html",
+ "http://foo.com/aa/bb/foo.html" );
+ normalizeTest("http://foo.com//aa//bb//foo.html",
+ "http://foo.com/aa/bb/foo.html" );
+ normalizeTest("http://foo.com////aa////bb////foo.html",
+ "http://foo.com/aa/bb/foo.html" );
}
private void normalizeTest(String weird, String normal) throws Exception {
assertEquals(normal, normalizer.normalize(weird, URLNormalizers.SCOPE_DEFAULT));
}
-
+
public static void main(String[] args) throws Exception {
new TestBasicURLNormalizer("test").testNormalizer();
}