You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by th...@apache.org on 2016/07/05 22:49:07 UTC
[23/69] [abbrv] [partial] nutch git commit: Re arranged the source
code as per maven conventions for build
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-querystring/src/main/java/org/apache/nutch/net/urlnormalizer/querystring/QuerystringURLNormalizer.java
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-querystring/src/main/java/org/apache/nutch/net/urlnormalizer/querystring/QuerystringURLNormalizer.java b/nutch-plugins/urlnormalizer-querystring/src/main/java/org/apache/nutch/net/urlnormalizer/querystring/QuerystringURLNormalizer.java
new file mode 100644
index 0000000..1de03a3
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-querystring/src/main/java/org/apache/nutch/net/urlnormalizer/querystring/QuerystringURLNormalizer.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.net.urlnormalizer.querystring;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.net.URLNormalizer;
+import org.apache.nutch.plugin.Extension;
+import org.apache.nutch.plugin.PluginRepository;
+import org.apache.nutch.util.URLUtil;
+
+/**
+ * URL normalizer plugin for normalizing query strings but sorting query string
+ * parameters. Not sorting query strings can lead to large amounts of duplicate
+ * URL's such as ?a=x&b=y vs b=y&a=x.
+ *
+ */
+public class QuerystringURLNormalizer implements URLNormalizer {
+
+ private Configuration conf;
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(QuerystringURLNormalizer.class);
+
+ public QuerystringURLNormalizer() {
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ public String normalize(String urlString, String scope)
+ throws MalformedURLException {
+ URL url = new URL(urlString);
+
+ String queryString = url.getQuery();
+
+ if (queryString == null) {
+ return urlString;
+ }
+
+ List<String> queryStringParts = Arrays.asList(queryString.split("&"));
+ Collections.sort(queryStringParts);
+
+ StringBuilder sb = new StringBuilder();
+
+ sb.append(url.getProtocol());
+ sb.append("://");
+ sb.append(url.getHost());
+ if (url.getPort() > -1) {
+ sb.append(":");
+ sb.append(url.getPort());
+ }
+ sb.append(url.getPath());
+ sb.append("?");
+ sb.append(StringUtils.join(queryStringParts, "&"));
+ if (url.getRef() != null) {
+ sb.append("#");
+ sb.append(url.getRef());
+ }
+
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-querystring/src/main/java/org/apache/nutch/net/urlnormalizer/querystring/package-info.java
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-querystring/src/main/java/org/apache/nutch/net/urlnormalizer/querystring/package-info.java b/nutch-plugins/urlnormalizer-querystring/src/main/java/org/apache/nutch/net/urlnormalizer/querystring/package-info.java
new file mode 100644
index 0000000..005fbca
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-querystring/src/main/java/org/apache/nutch/net/urlnormalizer/querystring/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * URL normalizer which sort the elements in the query part to avoid duplicates
+ * by permutations.
+ */
+package org.apache.nutch.net.urlnormalizer.querystring;
+
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-querystring/src/test/org/apache/nutch/net/urlnormalizer/querystring/TestQuerystringURLNormalizer.java
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-querystring/src/test/org/apache/nutch/net/urlnormalizer/querystring/TestQuerystringURLNormalizer.java b/nutch-plugins/urlnormalizer-querystring/src/test/org/apache/nutch/net/urlnormalizer/querystring/TestQuerystringURLNormalizer.java
new file mode 100644
index 0000000..b85c55d
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-querystring/src/test/org/apache/nutch/net/urlnormalizer/querystring/TestQuerystringURLNormalizer.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.net.urlnormalizer.querystring;
+
+import java.net.MalformedURLException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.net.URLNormalizers;
+import org.apache.nutch.util.NutchConfiguration;
+
+import junit.framework.TestCase;
+
+public class TestQuerystringURLNormalizer extends TestCase {
+
+ public void testQuerystringURLNormalizer() throws Exception {
+ Configuration conf = NutchConfiguration.create();
+
+ QuerystringURLNormalizer normalizer = new QuerystringURLNormalizer();
+ normalizer.setConf(conf);
+
+ assertEquals("http://example.com/?a=b&c=d", normalizer.normalize(
+ "http://example.com/?c=d&a=b", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.com/a/b/c", normalizer.normalize(
+ "http://example.com/a/b/c", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.com:1234/a/b/c", normalizer.normalize(
+ "http://example.com:1234/a/b/c", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.com:1234/a/b/c#ref", normalizer.normalize(
+ "http://example.com:1234/a/b/c#ref", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.com:1234/a/b/c?a=b&c=d#ref",
+ normalizer.normalize("http://example.com:1234/a/b/c?c=d&a=b#ref",
+ URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.com/?a=b&a=c&c=d", normalizer.normalize(
+ "http://example.com/?c=d&a=b&a=c", URLNormalizers.SCOPE_DEFAULT));
+ }
+}
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-regex/build.xml
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-regex/build.xml b/nutch-plugins/urlnormalizer-regex/build.xml
new file mode 100644
index 0000000..76875ec
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-regex/build.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project name="urlnormalizer-regex" default="jar-core">
+
+ <import file="../build-plugin.xml"/>
+
+ <!-- Deploy Unit test dependencies -->
+ <target name="deps-test">
+ <ant target="deploy" inheritall="false" dir="../nutch-extensionpoints"/>
+ </target>
+
+ <!-- for junit test -->
+ <mkdir dir="${build.test}/data"/>
+ <copy todir="${build.test}/data">
+ <fileset dir="sample" includes="**/*.xml, **/*.test"/>
+ </copy>
+
+
+</project>
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-regex/ivy.xml
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-regex/ivy.xml b/nutch-plugins/urlnormalizer-regex/ivy.xml
new file mode 100644
index 0000000..1a86d68
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-regex/ivy.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../..//ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-regex/plugin.xml
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-regex/plugin.xml b/nutch-plugins/urlnormalizer-regex/plugin.xml
new file mode 100644
index 0000000..e75096f
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-regex/plugin.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<plugin
+ id="urlnormalizer-regex"
+ name="Regex URL Normalizer"
+ version="1.0.0"
+ provider-name="nutch.org">
+
+ <runtime>
+ <library name="urlnormalizer-regex.jar">
+ <export name="*"/>
+ </library>
+ </runtime>
+
+ <requires>
+ <import plugin="nutch-extensionpoints"/>
+ </requires>
+
+ <extension id="org.apache.nutch.net.urlnormalizer.regex"
+ name="Nutch Regex URL Normalizer"
+ point="org.apache.nutch.net.URLNormalizer">
+ <implementation id="RegexURLNormalizer"
+ class="org.apache.nutch.net.urlnormalizer.regex.RegexURLNormalizer"/>
+ </extension>
+
+</plugin>
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-regex/pom.xml
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-regex/pom.xml b/nutch-plugins/urlnormalizer-regex/pom.xml
new file mode 100644
index 0000000..bfb056e
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-regex/pom.xml
@@ -0,0 +1,38 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nutch</groupId>
+ <artifactId>nutch-plugins</artifactId>
+ <version>1.13-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <artifactId>urlnormalizer-regex</artifactId>
+ <packaging>jar</packaging>
+
+ <name>urlnormalizer-regex</name>
+ <url>http://nutch.apache.org</url>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+</project>
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-default.test
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-default.test b/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-default.test
new file mode 100644
index 0000000..7867ad8
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-default.test
@@ -0,0 +1,84 @@
+# test simple removal of session id, keeping parameters before and after
+http://foo.com/foo.php?PHPSESSID=cdc993a493e899bed04f4d0c8a462a03 http://foo.com/foo.php
+http://foo.com/foo.php?f=2&PHPSESSID=cdc993a493e899bed04f4d0c8a462a03 http://foo.com/foo.php?f=2
+http://foo.com/foo.php?f=2&PHPSESSID=cdc993a493e899bed04f4d0c8a462a03&q=3 http://foo.com/foo.php?f=2&q=3
+http://foo.com/foo.php?PHPSESSID=cdc993a493e899bed04f4d0c8a462a03&f=2 http://foo.com/foo.php?f=2
+
+# test removal of different session ids including removal of ; in jsessionid
+http://www.foo.com/foo.php?Bv_SessionID=fassassddsajkl http://www.foo.com/foo.php
+http://www.foo.com/foo.php?Bv_SessionID=fassassddsajkl&x=y http://www.foo.com/foo.php?x=y
+http://www.foo.com/foo.html;jsessionid=1E6FEC0D14D044541DD84D2D013D29ED http://www.foo.com/foo.html
+http://www.foo.com/foo.html?param=1&another=2;jsessionid=1E6FEC0D14D044541DD84D2D013D29ED http://www.foo.com/foo.html?param=1&another=2
+http://www.foo.com/foo.html;jsessionid=1E6FEC0D14D044541DD84D2D013D29ED?param=1&another=2 http://www.foo.com/foo.html?param=1&another=2
+http://www.foo.com/foo.php?&x=1&sid=xyz&something=1 http://www.foo.com/foo.php?x=1&something=1
+http://www.foo.com/foo.html?_sessionID=824A6C0A13a7e11205wxN28F44E3 http://www.foo.com/foo.html
+http://www.foo.com/foo.php?_sessionid=qmyrcedt&outputformat=html&path=/3_images/foo http://www.foo.com/foo.php?outputformat=html&path=/3_images/foo
+http://www.foo.com/foo.php?_pid=2&_spid=0&lang=en&_sessionid=e36902d5bb2d0d922fc24b43 http://www.foo.com/foo.php?_pid=2&_spid=0&lang=en
+http://www.foo.com/foo.php?app=content&content=overview&lang=en&_sid=587fba8f825b05844526519fdb7d75c8&b=35&m=47 http://www.foo.com/foo.php?app=content&content=overview&lang=en&b=35&m=47
+# but NewsId (and similar) is not a session id (NUTCH-706, NUTCH-1328)
+http://www.foo.com/fa/newsdetail.aspx?NewsID=1567539 http://www.foo.com/fa/newsdetail.aspx?NewsID=1567539
+http://www.foo.com/home.cfm?language=en&country=uk&addressid=250646&pagingpos=0 http://www.foo.com/home.cfm?language=en&country=uk&addressid=250646&pagingpos=0
+
+# test removal default pages
+http://www.foo.com/home/index.html http://www.foo.com/home/
+http://www.foo.com/index.html http://www.foo.com/
+http://www.foo.com/index.htm http://www.foo.com/
+http://www.foo.com/index.asp http://www.foo.com/
+http://www.foo.com/index.aspx http://www.foo.com/
+http://www.foo.com/index.php http://www.foo.com/
+http://www.foo.com/index.php3 http://www.foo.com/
+http://www.foo.com/default.html http://www.foo.com/
+http://www.foo.com/default.htm http://www.foo.com/
+http://www.foo.com/default.asp http://www.foo.com/
+http://www.foo.com/default.aspx http://www.foo.com/
+http://www.foo.com/default.php http://www.foo.com/
+http://www.foo.com/default.php3 http://www.foo.com/
+http://www.foo.com/something.php3 http://www.foo.com/something.php3
+http://www.foo.com/something.html http://www.foo.com/something.html
+http://www.foo.com/something.asp http://www.foo.com/something.asp
+http://www.foo.com/index.phtml http://www.foo.com/
+http://www.foo.com/index.cfm http://www.foo.com/
+http://www.foo.com/index.cgi http://www.foo.com/
+http://www.foo.com/index.HTML http://www.foo.com/
+http://www.foo.com/index.Htm http://www.foo.com/
+http://www.foo.com/index.ASP http://www.foo.com/
+http://www.foo.com/index.jsp http://www.foo.com/
+http://www.foo.com/index.jsf http://www.foo.com/
+http://www.foo.com/index.jspx http://www.foo.com/
+http://www.foo.com/index.jspfx http://www.foo.com/index.jspfx
+http://www.foo.com/index.jspa http://www.foo.com/
+http://www.foo.com/index.jsps http://www.foo.com/index.jsps
+http://www.foo.com/index.aspX http://www.foo.com/
+http://www.foo.com/index.PhP http://www.foo.com/
+http://www.foo.com/index.PhP4 http://www.foo.com/
+http://www.foo.com/default.HTml http://www.foo.com/
+http://www.foo.com/default.HTm http://www.foo.com/
+http://www.foo.com/default.ASp http://www.foo.com/
+http://www.foo.com/default.AspX http://www.foo.com/
+http://www.foo.com/default.PHP http://www.foo.com/
+http://www.foo.com/default.PHP3 http://www.foo.com/
+http://www.foo.com/index.phtml http://www.foo.com/
+http://www.foo.com/index.cfm http://www.foo.com/
+http://www.foo.com/index.cgi http://www.foo.com/
+
+# ensure keeping non-default pages
+http://www.foo.com/foo.php3 http://www.foo.com/foo.php3
+http://www.foo.com/foo.html http://www.foo.com/foo.html
+http://www.foo.com/foo.asp http://www.foo.com/foo.asp
+
+# test removal of interpage anchors and keeping query string
+http://www.foo.com/foo.html#something http://www.foo.com/foo.html
+http://www.foo.com/foo.html#something?x=y http://www.foo.com/foo.html?x=y
+
+# test general cleaning of bad urls
+http://www.foo.com/foo.html?&x=y http://www.foo.com/foo.html?x=y
+http://www.foo.com/foo.html?x=y&&&z=a http://www.foo.com/foo.html?x=y&z=a
+http://www.foo.com/foo.html? http://www.foo.com/foo.html
+
+# remove double slashes but keep 2 slashes after protocol
+http://www.foo.com//path//foo.html http://www.foo.com/path/foo.html
+https://www.foo.com//path//foo.html https://www.foo.com/path/foo.html
+
+# normalize file: protocol prefix (keep one slash)
+file:///path//foo.html file:/path/foo.html
+file:/path//foo.html file:/path/foo.html
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-default.xml
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-default.xml b/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-default.xml
new file mode 100644
index 0000000..4d6eabc
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-default.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0"?>
+<!-- This is the configuration file for the RegexUrlNormalize Class.
+ This is intended so that users can specify substitutions to be
+ done on URLs. The regex engine that is used is Perl5 compatible.
+ The rules are applied to URLs in the order they occur in this file. -->
+
+<!-- WATCH OUT: an xml parser reads this file an ampersands must be
+ expanded to & -->
+
+<!-- The following rules show how to strip out session IDs, default pages,
+ interpage anchors, etc. Order does matter! -->
+<regex-normalize>
+
+<!-- removes session ids from urls (such as jsessionid and PHPSESSID) -->
+<regex>
+ <pattern>(?i)(;?\b_?(l|j|bv_)?(sid|phpsessid|sessionid)=.*?)(\?|&|#|$)</pattern>
+ <substitution>$4</substitution>
+</regex>
+
+<!-- changes default pages into standard for /index.html, etc. into / -->
+<!-- these are commented in the default file but uncommented here for testing -->
+<regex>
+ <pattern>/((?i)index|default)\.((?i)js[pf]{1}?[afx]?|cgi|cfm|asp[x]?|[psx]?htm[l]?|php[3456]?)(\?|&|#|$)</pattern>
+ <substitution>/$3</substitution>
+</regex>
+
+<!-- removes interpage href anchors such as site.com#location -->
+<regex>
+ <pattern>#.*?(\?|&|$)</pattern>
+ <substitution>$1</substitution>
+</regex>
+
+<!-- cleans ?&var=value into ?var=value -->
+<regex>
+ <pattern>\?&</pattern>
+ <substitution>\?</substitution>
+</regex>
+
+<!-- cleans multiple sequential ampersands into a single ampersand -->
+<regex>
+ <pattern>&{2,}</pattern>
+ <substitution>&</substitution>
+</regex>
+
+<!-- removes trailing ?, ampersands, . -->
+<regex>
+ <pattern>[\?&\.]$</pattern>
+ <substitution></substitution>
+</regex>
+
+<!-- normalize file:/// protocol prefix: -->
+<!-- keep one single slash (NUTCH-1483) -->
+<regex>
+ <pattern>^file://+</pattern>
+ <substitution>file:/</substitution>
+</regex>
+
+<!-- removes duplicate slashes but -->
+<!-- * allow 2 slashes after colon ':' (indicating protocol) -->
+<regex>
+ <pattern>(?<!:)/{2,}</pattern>
+ <substitution>/</substitution>
+</regex>
+
+</regex-normalize>
+
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-scope1.test
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-scope1.test b/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-scope1.test
new file mode 100644
index 0000000..9d92880
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-scope1.test
@@ -0,0 +1,8 @@
+# test removal of subdomains
+http://www.foo.bar.com/ http://bar.com/
+
+# test removal of url path
+http://www.foo.bar.com/foo.php?PHPSESSID=cdc993a493e899bed04f4d0c8a462a03 http://bar.com/
+
+# test removal of urls in arguments
+https://www.foo.bar.com/foo.php?url=http://www.example.com/test.php https://bar.com/
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-scope1.xml
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-scope1.xml b/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-scope1.xml
new file mode 100644
index 0000000..3698968
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-regex/sample/regex-normalize-scope1.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<!-- This is the configuration file for the RegexUrlNormalize Class.
+ This is intended so that users can specify substitutions to be
+ done on URLs. The regex engine that is used is Perl5 compatible.
+ The rules are applied to URLs in the order they occur in this file. -->
+
+<!-- WATCH OUT: an xml parser reads this file an ampersands must be
+ expanded to & -->
+
+<!--
+ The following rules show how to reduce urls so that
+ urls from the same domain are identical. This is useful
+ e.g. when calculating host counts, or splitting fetchlists.
+-->
+<regex-normalize>
+<regex>
+ <pattern>(^[a-z]{3,5}://)([\w]+\.)*?(\w+\.\w+)[/$].*</pattern>
+ <substitution>$1$3/</substitution>
+</regex>
+</regex-normalize>
+
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-regex/src/main/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-regex/src/main/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java b/nutch-plugins/urlnormalizer-regex/src/main/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java
new file mode 100644
index 0000000..363da18
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-regex/src/main/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java
@@ -0,0 +1,324 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.net.urlnormalizer.regex;
+
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.net.MalformedURLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.nutch.net.URLNormalizer;
+import org.apache.nutch.net.URLNormalizers;
+import org.apache.nutch.util.NutchConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+import org.xml.sax.InputSource;
+
+/**
+ * Allows users to do regex substitutions on all/any URLs that are encountered,
+ * which is useful for stripping session IDs from URLs.
+ *
+ * <p>
+ * This class uses the <tt>urlnormalizer.regex.file</tt> property. It should be
+ * set to the file name of an xml file which should contain the patterns and
+ * substitutions to be done on encountered URLs.
+ * </p>
+ * <p>
+ * This class also supports different rules depending on the scope. Please see
+ * the javadoc in {@link org.apache.nutch.net.URLNormalizers} for more details.
+ * </p>
+ *
+ * @author Luke Baker
+ * @author Andrzej Bialecki
+ */
+public class RegexURLNormalizer extends Configured implements URLNormalizer {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(RegexURLNormalizer.class);
+
+ /**
+ * Class which holds a compiled pattern and its corresponding substition
+ * string.
+ */
+ private static class Rule {
+ public Pattern pattern;
+
+ public String substitution;
+ }
+
+ private ThreadLocal<HashMap<String, List<Rule>>> scopedRulesThreadLocal = new ThreadLocal<HashMap<String, List<Rule>>>() {
+ protected java.util.HashMap<String, java.util.List<Rule>> initialValue() {
+ return new HashMap<String, List<Rule>>();
+ };
+ };
+
+ public HashMap<String, List<Rule>> getScopedRules() {
+ return scopedRulesThreadLocal.get();
+ }
+
+ private List<Rule> defaultRules;
+
+ private static final List<Rule> EMPTY_RULES = Collections.emptyList();
+
+ /**
+ * The default constructor which is called from UrlNormalizerFactory
+ * (normalizerClass.newInstance()) in method: getNormalizer()*
+ */
+ public RegexURLNormalizer() {
+ super(null);
+ }
+
+ public RegexURLNormalizer(Configuration conf) {
+ super(conf);
+ }
+
+ /**
+ * Constructor which can be passed the file name, so it doesn't look in the
+ * configuration files for it.
+ */
+ public RegexURLNormalizer(Configuration conf, String filename)
+ throws IOException, PatternSyntaxException {
+ super(conf);
+ List<Rule> rules = readConfigurationFile(filename);
+ if (rules != null) {
+ defaultRules = rules;
+ }
+ }
+
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ if (conf == null)
+ return;
+ // the default constructor was called
+
+ String filename = getConf().get("urlnormalizer.regex.file");
+ String stringRules = getConf().get("urlnormalizer.regex.rules");
+ Reader reader = null;
+ if (stringRules != null) {
+ reader = new StringReader(stringRules);
+ } else {
+ reader = getConf().getConfResourceAsReader(filename);
+ }
+ List<Rule> rules = null;
+ if (reader == null) {
+ LOG.warn("Can't load the default rules! ");
+ rules = EMPTY_RULES;
+ } else {
+ try {
+ rules = readConfiguration(reader);
+ } catch (Exception e) {
+ LOG.warn("Couldn't read default config: " + e);
+ rules = EMPTY_RULES;
+ }
+ }
+ defaultRules = rules;
+ }
+
+ // used in JUnit test.
+ void setConfiguration(Reader reader, String scope) {
+ List<Rule> rules = readConfiguration(reader);
+ getScopedRules().put(scope, rules);
+ LOG.debug("Set config for scope '" + scope + "': " + rules.size()
+ + " rules.");
+ }
+
+ /**
+ * This function does the replacements by iterating through all the regex
+ * patterns. It accepts a string url as input and returns the altered string.
+ */
+ public String regexNormalize(String urlString, String scope) {
+ HashMap<String, List<Rule>> scopedRules = getScopedRules();
+ List<Rule> curRules = scopedRules.get(scope);
+ if (curRules == null) {
+ // try to populate
+ String configFile = getConf().get("urlnormalizer.regex.file." + scope);
+ if (configFile != null) {
+ LOG.debug("resource for scope '" + scope + "': " + configFile);
+ try {
+ Reader reader = getConf().getConfResourceAsReader(configFile);
+ curRules = readConfiguration(reader);
+ scopedRules.put(scope, curRules);
+ } catch (Exception e) {
+ LOG.warn("Couldn't load resource '" + configFile + "': " + e);
+ }
+ }
+ if (curRules == EMPTY_RULES || curRules == null) {
+ LOG.info("can't find rules for scope '" + scope + "', using default");
+ scopedRules.put(scope, EMPTY_RULES);
+ }
+ }
+ if (curRules == EMPTY_RULES || curRules == null) {
+ curRules = defaultRules;
+ }
+ Iterator<Rule> i = curRules.iterator();
+ while (i.hasNext()) {
+ Rule r = (Rule) i.next();
+
+ Matcher matcher = r.pattern.matcher(urlString);
+
+ urlString = matcher.replaceAll(r.substitution);
+ }
+ return urlString;
+ }
+
+ public String normalize(String urlString, String scope)
+ throws MalformedURLException {
+ return regexNormalize(urlString, scope);
+ }
+
+ /** Reads the configuration file and populates a List of Rules. */
+ private List<Rule> readConfigurationFile(String filename) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("loading " + filename);
+ }
+ try {
+ FileReader reader = new FileReader(filename);
+ return readConfiguration(reader);
+ } catch (Exception e) {
+ LOG.error("Error loading rules from '" + filename + "': " + e);
+ return EMPTY_RULES;
+ }
+ }
+
+ private List<Rule> readConfiguration(Reader reader) {
+ List<Rule> rules = new ArrayList<Rule>();
+ try {
+
+ // borrowed heavily from code in Configuration.java
+ Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder()
+ .parse(new InputSource(reader));
+ Element root = doc.getDocumentElement();
+ if ((!"regex-normalize".equals(root.getTagName()))
+ && (LOG.isErrorEnabled())) {
+ LOG.error("bad conf file: top-level element not <regex-normalize>");
+ }
+ NodeList regexes = root.getChildNodes();
+ for (int i = 0; i < regexes.getLength(); i++) {
+ Node regexNode = regexes.item(i);
+ if (!(regexNode instanceof Element))
+ continue;
+ Element regex = (Element) regexNode;
+ if ((!"regex".equals(regex.getTagName())) && (LOG.isWarnEnabled())) {
+ LOG.warn("bad conf file: element not <regex>");
+ }
+ NodeList fields = regex.getChildNodes();
+ String patternValue = null;
+ String subValue = null;
+ for (int j = 0; j < fields.getLength(); j++) {
+ Node fieldNode = fields.item(j);
+ if (!(fieldNode instanceof Element))
+ continue;
+ Element field = (Element) fieldNode;
+ if ("pattern".equals(field.getTagName()) && field.hasChildNodes())
+ patternValue = ((Text) field.getFirstChild()).getData();
+ if ("substitution".equals(field.getTagName())
+ && field.hasChildNodes())
+ subValue = ((Text) field.getFirstChild()).getData();
+ if (!field.hasChildNodes())
+ subValue = "";
+ }
+ if (patternValue != null && subValue != null) {
+ Rule rule = new Rule();
+ try {
+ rule.pattern = Pattern.compile(patternValue);
+ } catch (PatternSyntaxException e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("skipped rule: " + patternValue + " -> " + subValue
+ + " : invalid regular expression pattern: " + e);
+ }
+ continue;
+ }
+ rule.substitution = subValue;
+ rules.add(rule);
+ }
+ }
+ } catch (Exception e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("error parsing conf file: " + e);
+ }
+ return EMPTY_RULES;
+ }
+ if (rules.size() == 0)
+ return EMPTY_RULES;
+ return rules;
+ }
+
+ /** Spits out patterns and substitutions that are in the configuration file. */
+ public static void main(String args[]) throws PatternSyntaxException,
+ IOException {
+ RegexURLNormalizer normalizer = new RegexURLNormalizer();
+ normalizer.setConf(NutchConfiguration.create());
+ HashMap<String, List<Rule>> scopedRules = normalizer.getScopedRules();
+ Iterator<Rule> i = normalizer.defaultRules.iterator();
+ System.out.println("* Rules for 'DEFAULT' scope:");
+ while (i.hasNext()) {
+ Rule r = i.next();
+ System.out.print(" " + r.pattern.pattern() + " -> ");
+ System.out.println(r.substitution);
+ }
+ // load the scope
+ if (args.length > 1) {
+ normalizer.normalize("http://test.com", args[1]);
+ }
+ if (scopedRules.size() > 1) {
+ Iterator<String> it = scopedRules.keySet().iterator();
+ while (it.hasNext()) {
+ String scope = it.next();
+ if (URLNormalizers.SCOPE_DEFAULT.equals(scope))
+ continue;
+ System.out.println("* Rules for '" + scope + "' scope:");
+ i = ((List<Rule>) scopedRules.get(scope)).iterator();
+ while (i.hasNext()) {
+ Rule r = (Rule) i.next();
+ System.out.print(" " + r.pattern.pattern() + " -> ");
+ System.out.println(r.substitution);
+ }
+ }
+ }
+ if (args.length > 0) {
+ System.out.println("\n---------- Normalizer test -----------");
+ String scope = URLNormalizers.SCOPE_DEFAULT;
+ if (args.length > 1)
+ scope = args[1];
+ System.out.println("Scope: " + scope);
+ System.out.println("Input url: '" + args[0] + "'");
+ System.out.println("Output url: '" + normalizer.normalize(args[0], scope)
+ + "'");
+ }
+ System.exit(0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-regex/src/main/java/org/apache/nutch/net/urlnormalizer/regex/package-info.java
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-regex/src/main/java/org/apache/nutch/net/urlnormalizer/regex/package-info.java b/nutch-plugins/urlnormalizer-regex/src/main/java/org/apache/nutch/net/urlnormalizer/regex/package-info.java
new file mode 100644
index 0000000..04562c3
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-regex/src/main/java/org/apache/nutch/net/urlnormalizer/regex/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * URL normalizer with configurable rules based on regular expressions
+ * ({@link java.util.regex.Pattern}).
+ */
+package org.apache.nutch.net.urlnormalizer.regex;
+
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java b/nutch-plugins/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java
new file mode 100644
index 0000000..cbf6c64
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.net.urlnormalizer.regex;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.*;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.net.URLNormalizers;
+import org.apache.nutch.util.NutchConfiguration;
+
+/** Unit tests for RegexUrlNormalizer. */
+public class TestRegexURLNormalizer {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestRegexURLNormalizer.class);
+
+ private RegexURLNormalizer normalizer;
+ private Configuration conf;
+ private Map<String, NormalizedURL[]> testData = new HashMap<String, NormalizedURL[]>();
+
+ // This system property is defined in ./src/plugin/build-plugin.xml
+ private String sampleDir = System.getProperty("test.data", ".");
+
+ // Make sure sample files are copied to "test.data" as specified in
+ // ./src/plugin/urlnormalizer-regex/build.xml during plugin compilation.
+
+ public TestRegexURLNormalizer() throws IOException {
+ normalizer = new RegexURLNormalizer();
+ conf = NutchConfiguration.create();
+ normalizer.setConf(conf);
+ File[] configs = new File(sampleDir).listFiles(new FileFilter() {
+ public boolean accept(File f) {
+ if (f.getName().endsWith(".xml")
+ && f.getName().startsWith("regex-normalize-"))
+ return true;
+ return false;
+ }
+ });
+ for (int i = 0; i < configs.length; i++) {
+ try {
+ FileReader reader = new FileReader(configs[i]);
+ String cname = configs[i].getName();
+ cname = cname.substring(16, cname.indexOf(".xml"));
+ normalizer.setConfiguration(reader, cname);
+ NormalizedURL[] urls = readTestFile(cname);
+ testData.put(cname, urls);
+ } catch (Exception e) {
+ LOG.warn("Could load config from '" + configs[i] + "': " + e.toString());
+ }
+ }
+ }
+
+ @Test
+ public void testNormalizerDefault() throws Exception {
+ normalizeTest((NormalizedURL[]) testData.get(URLNormalizers.SCOPE_DEFAULT),
+ URLNormalizers.SCOPE_DEFAULT);
+ }
+
+ @Test
+ public void testNormalizerScope() throws Exception {
+ Iterator<String> it = testData.keySet().iterator();
+ while (it.hasNext()) {
+ String scope = it.next();
+ normalizeTest((NormalizedURL[]) testData.get(scope), scope);
+ }
+ }
+
+ private void normalizeTest(NormalizedURL[] urls, String scope)
+ throws Exception {
+ for (int i = 0; i < urls.length; i++) {
+ String url = urls[i].url;
+ String normalized = normalizer.normalize(urls[i].url, scope);
+ String expected = urls[i].expectedURL;
+ LOG.info("scope: " + scope + " url: " + url + " | normalized: "
+ + normalized + " | expected: " + expected);
+ Assert.assertEquals(urls[i].expectedURL, normalized);
+ }
+ }
+
+ private void bench(int loops, String scope) {
+ long start = System.currentTimeMillis();
+ try {
+ NormalizedURL[] expected = (NormalizedURL[]) testData.get(scope);
+ if (expected == null)
+ return;
+ for (int i = 0; i < loops; i++) {
+ normalizeTest(expected, scope);
+ }
+ } catch (Exception e) {
+ Assert.fail(e.toString());
+ }
+ LOG.info("bench time (" + loops + ") "
+ + (System.currentTimeMillis() - start) + "ms");
+ }
+
+ private static class NormalizedURL {
+ String url;
+ String expectedURL;
+
+ public NormalizedURL(String line) {
+ String[] fields = line.split("\\s+");
+ url = fields[0];
+ expectedURL = fields[1];
+ }
+ }
+
+ private NormalizedURL[] readTestFile(String scope) throws IOException {
+ File f = new File(sampleDir, "regex-normalize-" + scope + ".test");
+ @SuppressWarnings("resource")
+ BufferedReader in = new BufferedReader(new InputStreamReader(
+ new FileInputStream(f), "UTF-8"));
+ List<NormalizedURL> list = new ArrayList<NormalizedURL>();
+ String line;
+ while ((line = in.readLine()) != null) {
+ if (line.trim().length() == 0 || line.startsWith("#")
+ || line.startsWith(" "))
+ continue;
+ list.add(new NormalizedURL(line));
+ }
+ return (NormalizedURL[]) list.toArray(new NormalizedURL[list.size()]);
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length == 0) {
+ System.err.println("TestRegexURLNormalizer [-bench <iter>] <scope>");
+ System.exit(-1);
+ }
+ boolean bench = false;
+ int iter = -1;
+ String scope = null;
+ for (int i = 0; i < args.length; i++) {
+ if (args[i].equals("-bench")) {
+ bench = true;
+ iter = Integer.parseInt(args[++i]);
+ } else
+ scope = args[i];
+ }
+ if (scope == null) {
+ System.err.println("Missing required scope name.");
+ System.exit(-1);
+ }
+ if (bench && iter < 0) {
+ System.err.println("Invalid number of iterations: " + iter);
+ System.exit(-1);
+ }
+ TestRegexURLNormalizer test = new TestRegexURLNormalizer();
+ NormalizedURL[] urls = (NormalizedURL[]) test.testData.get(scope);
+ if (urls == null) {
+ LOG.warn("Missing test data for scope '" + scope
+ + "', using default scope.");
+ scope = URLNormalizers.SCOPE_DEFAULT;
+ urls = (NormalizedURL[]) test.testData.get(scope);
+ }
+ if (bench) {
+ test.bench(iter, scope);
+ } else {
+ test.normalizeTest(urls, scope);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-slash/build.xml
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-slash/build.xml b/nutch-plugins/urlnormalizer-slash/build.xml
new file mode 100644
index 0000000..29b2262
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-slash/build.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project name="urlnormalizer-slash" default="jar-core">
+
+ <import file="../build-plugin.xml"/>
+
+ <!-- for junit test -->
+ <mkdir dir="${build.test}/data"/>
+ <copy todir="${build.test}/data">
+ <fileset dir="data" />
+ </copy>
+</project>
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-slash/data/slashes.txt
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-slash/data/slashes.txt b/nutch-plugins/urlnormalizer-slash/data/slashes.txt
new file mode 100644
index 0000000..d3bd70a
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-slash/data/slashes.txt
@@ -0,0 +1,7 @@
+# Both domains have duplicate URL's, some with slashes and some without
+
+# We prefer this domain with slashes
+www.example.org +
+
+# ..but this domain without
+www.example.net -
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-slash/ivy.xml
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-slash/ivy.xml b/nutch-plugins/urlnormalizer-slash/ivy.xml
new file mode 100644
index 0000000..0a363f7
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-slash/ivy.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="${nutch.root}/ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-slash/plugin.xml
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-slash/plugin.xml b/nutch-plugins/urlnormalizer-slash/plugin.xml
new file mode 100644
index 0000000..db820ed
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-slash/plugin.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<plugin
+ id="urlnormalizer-slash"
+ name="Slash URL Normalizer"
+ version="1.0.0"
+ provider-name="nutch.org">
+
+ <runtime>
+ <library name="urlnormalizer-slash.jar">
+ <export name="*"/>
+ </library>
+ </runtime>
+
+ <requires>
+ <import plugin="nutch-extensionpoints"/>
+ </requires>
+
+ <extension id="org.apache.nutch.net.urlnormalizer.slash"
+ name="Nutch Slash URL Normalizer"
+ point="org.apache.nutch.net.URLNormalizer">
+ <implementation id="SlashURLNormalizer"
+ class="org.apache.nutch.net.urlnormalizer.slash.SlashURLNormalizer">
+ <parameter name="file" value="slashes.txt"/>
+ </implementation>
+ </extension>
+
+</plugin>
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-slash/pom.xml
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-slash/pom.xml b/nutch-plugins/urlnormalizer-slash/pom.xml
new file mode 100644
index 0000000..0ac618f
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-slash/pom.xml
@@ -0,0 +1,38 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.nutch</groupId>
+ <artifactId>nutch-plugins</artifactId>
+ <version>1.13-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <artifactId>urlnormalizer-slash</artifactId>
+ <packaging>jar</packaging>
+
+ <name>urlnormalizer-slash</name>
+ <url>http://nutch.apache.org</url>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+</project>
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-slash/src/main/java/org/apache/nutch/net/urlnormalizer/slash/SlashURLNormalizer.java
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-slash/src/main/java/org/apache/nutch/net/urlnormalizer/slash/SlashURLNormalizer.java b/nutch-plugins/urlnormalizer-slash/src/main/java/org/apache/nutch/net/urlnormalizer/slash/SlashURLNormalizer.java
new file mode 100644
index 0000000..4554cf0
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-slash/src/main/java/org/apache/nutch/net/urlnormalizer/slash/SlashURLNormalizer.java
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.net.urlnormalizer.slash;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.net.URLNormalizer;
+import org.apache.nutch.plugin.Extension;
+import org.apache.nutch.plugin.PluginRepository;
+import org.apache.nutch.util.URLUtil;
+
+/**
+ * @author markus@openindex.io
+ */
+public class SlashURLNormalizer implements URLNormalizer {
+
+ private Configuration conf;
+
+ private static final Logger LOG = LoggerFactory.getLogger(SlashURLNormalizer.class);
+
+ private static final char QUESTION_MARK = '?';
+ private static final char SLASH = '/';
+ private static final char DOT = '.';
+ private static final String PROTOCOL_DELIMITER = "://";
+
+ private static String attributeFile = null;
+ private String slashesFile = null;
+
+ // We record a map of hosts and boolean, the boolean denotes whether the host should
+ // have slashes after URL paths. True means slash, false means remove the slash
+ private static final Map<String,Boolean> slashesMap = new HashMap<String,Boolean>();
+
+ public SlashURLNormalizer() {}
+
+ public SlashURLNormalizer(String slashesFile) {
+ this.slashesFile = slashesFile;
+ }
+
+ private synchronized void readConfiguration(Reader configReader) throws IOException {
+ if (slashesMap.size() > 0) {
+ return;
+ }
+
+ BufferedReader reader = new BufferedReader(configReader);
+ String line, host;
+ String rule;
+ int delimiterIndex;
+
+ while ((line = reader.readLine()) != null) {
+ if (StringUtils.isNotBlank(line) && !line.startsWith("#")) {
+ line.trim();
+ delimiterIndex = line.indexOf(" ");
+ // try tabulator
+ if (delimiterIndex == -1) {
+ delimiterIndex = line.indexOf("\t");
+ }
+
+ host = line.substring(0, delimiterIndex);
+ rule = line.substring(delimiterIndex + 1).trim();
+
+ if (rule.equals("+")) {
+ slashesMap.put(host, true);
+ } else {
+ slashesMap.put(host, false);
+ }
+ }
+ }
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+
+ // get the extensions for domain urlfilter
+ String pluginName = "urlnormalizer-slash";
+ Extension[] extensions = PluginRepository.get(conf).getExtensionPoint(
+ URLNormalizer.class.getName()).getExtensions();
+ for (int i = 0; i < extensions.length; i++) {
+ Extension extension = extensions[i];
+ if (extension.getDescriptor().getPluginId().equals(pluginName)) {
+ attributeFile = extension.getAttribute("file");
+ break;
+ }
+ }
+
+ // handle blank non empty input
+ if (attributeFile != null && attributeFile.trim().equals("")) {
+ attributeFile = null;
+ }
+
+ if (attributeFile != null) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Attribute \"file\" is defined for plugin " + pluginName
+ + " as " + attributeFile);
+ }
+ }
+ else {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn("Attribute \"file\" is not defined in plugin.xml for plugin "
+ + pluginName);
+ }
+ }
+
+ // domain file and attribute "file" take precedence if defined
+ String file = conf.get("urlnormalizer.slashes.file");
+ String stringRules = conf.get("urlnormalizer.slashes.rules");
+ if (slashesFile != null) {
+ file = slashesFile;
+ }
+ else if (attributeFile != null) {
+ file = attributeFile;
+ }
+ Reader reader = null;
+ if (stringRules != null) { // takes precedence over files
+ reader = new StringReader(stringRules);
+ } else {
+ reader = conf.getConfResourceAsReader(file);
+ }
+ try {
+ if (reader == null) {
+ reader = new FileReader(file);
+ }
+ readConfiguration(reader);
+ }
+ catch (IOException e) {
+ LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ }
+
+ public String normalize(String url, String scope) throws MalformedURLException {
+ return normalize(url, null, scope);
+ }
+
+ public String normalize(String url, CrawlDatum crawlDatum, String scope) throws MalformedURLException {
+ // Get URL repr.
+ URL u = new URL(url);
+
+ // Get the host
+ String host = u.getHost();
+
+ // Do we have a rule for this host?
+ if (slashesMap.containsKey(host)) {
+ // Yes, separate the path and optional querystring
+ String protocol = u.getProtocol();
+ String path = u.getPath();
+
+ // Don't do anything to root URL's
+ // / is always set by basic normalizer
+ if (path.length() > 1) {
+ String queryString = u.getQuery();
+
+ // Get the rule
+ boolean rule = slashesMap.get(host);
+
+ // Does it have a trailing slash
+ int lastIndexOfSlash = path.lastIndexOf(SLASH);
+ boolean trailingSlash = (lastIndexOfSlash == path.length() - 1);
+
+ // Do we need to add a trailing slash?
+ if (!trailingSlash && rule) {
+ // Only add a trailing slash if this path doesn't appear to have an extension/suffix such as .html
+ int lastIndexOfDot = path.lastIndexOf(DOT);
+ if (path.length() < 6 || lastIndexOfDot == -1 || lastIndexOfDot < path.length() - 6) {
+ StringBuilder buffer = new StringBuilder(protocol);
+ buffer.append(PROTOCOL_DELIMITER);
+ buffer.append(host);
+ buffer.append(path);
+ buffer.append(SLASH);
+ if (queryString != null) {
+ buffer.append(QUESTION_MARK);
+ buffer.append(queryString);
+ }
+ url = buffer.toString();
+ }
+ }
+
+ // Do we need to remove a trailing slash?
+ else if (trailingSlash && !rule) {
+ StringBuilder buffer = new StringBuilder(protocol);
+ buffer.append(PROTOCOL_DELIMITER);
+ buffer.append(host);
+ buffer.append(path.substring(0, lastIndexOfSlash));
+ if (queryString != null) {
+ buffer.append(QUESTION_MARK);
+ buffer.append(queryString);
+ }
+ url = buffer.toString();
+ }
+ }
+ }
+
+ return url;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/nutch-plugins/urlnormalizer-slash/src/test/org/apache/nutch/net/urlnormalizer/slash/TestSlashURLNormalizer.java
----------------------------------------------------------------------
diff --git a/nutch-plugins/urlnormalizer-slash/src/test/org/apache/nutch/net/urlnormalizer/slash/TestSlashURLNormalizer.java b/nutch-plugins/urlnormalizer-slash/src/test/org/apache/nutch/net/urlnormalizer/slash/TestSlashURLNormalizer.java
new file mode 100644
index 0000000..c3585e4
--- /dev/null
+++ b/nutch-plugins/urlnormalizer-slash/src/test/org/apache/nutch/net/urlnormalizer/slash/TestSlashURLNormalizer.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.net.urlnormalizer.slash;
+
+import java.net.MalformedURLException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.net.URLNormalizers;
+import org.apache.nutch.util.NutchConfiguration;
+
+import junit.framework.TestCase;
+
+public class TestSlashURLNormalizer extends TestCase {
+
+ private final static String SEPARATOR = System.getProperty("file.separator");
+ private final static String SAMPLES = System.getProperty("test.data", ".");
+
+ public void testSlashURLNormalizer() throws Exception {
+ Configuration conf = NutchConfiguration.create();
+
+ String slashesFile = SAMPLES + SEPARATOR + "slashes.txt";
+ SlashURLNormalizer normalizer = new SlashURLNormalizer(slashesFile);
+ normalizer.setConf(conf);
+
+ // No change
+ assertEquals("http://example.org/", normalizer.normalize("http://example.org/", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.net/", normalizer.normalize("http://example.net/", URLNormalizers.SCOPE_DEFAULT));
+
+ // Don't touch base URL's
+ assertEquals("http://example.org", normalizer.normalize("http://example.org", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.net", normalizer.normalize("http://example.net", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.org/", normalizer.normalize("http://example.org/", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.net/", normalizer.normalize("http://example.net/", URLNormalizers.SCOPE_DEFAULT));
+
+ // Change
+ assertEquals("http://www.example.org/page/", normalizer.normalize("http://www.example.org/page", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://www.example.net/path/to/something", normalizer.normalize("http://www.example.net/path/to/something/", URLNormalizers.SCOPE_DEFAULT));
+
+ // No change
+ assertEquals("http://example.org/buh/", normalizer.normalize("http://example.org/buh/", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.net/blaat", normalizer.normalize("http://example.net/blaat", URLNormalizers.SCOPE_DEFAULT));
+
+ // No change
+ assertEquals("http://example.nl/buh/", normalizer.normalize("http://example.nl/buh/", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://example.de/blaat", normalizer.normalize("http://example.de/blaat", URLNormalizers.SCOPE_DEFAULT));
+
+ // Change
+ assertEquals("http://www.example.org/page/?a=b&c=d", normalizer.normalize("http://www.example.org/page?a=b&c=d", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://www.example.net/path/to/something?a=b&c=d", normalizer.normalize("http://www.example.net/path/to/something/?a=b&c=d", URLNormalizers.SCOPE_DEFAULT));
+
+ // No change
+ assertEquals("http://www.example.org/noise.mp3", normalizer.normalize("http://www.example.org/noise.mp3", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://www.example.org/page.html", normalizer.normalize("http://www.example.org/page.html", URLNormalizers.SCOPE_DEFAULT));
+ assertEquals("http://www.example.org/page.shtml", normalizer.normalize("http://www.example.org/page.shtml", URLNormalizers.SCOPE_DEFAULT));
+
+ // Change
+ assertEquals("http://www.example.org/this.is.not.an_extension/", normalizer.normalize("http://www.example.org/this.is.not.an_extension", URLNormalizers.SCOPE_DEFAULT));
+ }
+}
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b92375c..8cffbc2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -10,7 +10,7 @@
<packaging>pom</packaging>
<properties>
-
+ <junit.version>4.12</junit.version>
</properties>
<modules>
<module>nutch-core</module>
@@ -26,6 +26,26 @@
<target>1.7</target>
</configuration>
</plugin>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>3.0.0</version>
+ <configuration>
+ <filesets>
+ <fileset>
+ <directory>runtime/</directory>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ </filesets>
+ </configuration>
+ </plugin>
</plugins>
</build>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/nutch/blob/ffa16784/src/plugin/build-plugin.xml
----------------------------------------------------------------------
diff --git a/src/plugin/build-plugin.xml b/src/plugin/build-plugin.xml
deleted file mode 100755
index c759d5f..0000000
--- a/src/plugin/build-plugin.xml
+++ /dev/null
@@ -1,255 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- Imported by plugin build.xml files to define default targets. -->
-<project xmlns:ivy="antlib:org.apache.ivy.ant">
-
- <property name="name" value="${ant.project.name}"/>
- <property name="root" value="${basedir}"/>
-
- <!-- load plugin-specific properties first -->
- <property file="${user.home}/${name}.build.properties" />
- <property file="${root}/build.properties" />
-
- <property name="nutch.root" location="${root}/../../../"/>
-
- <property name="src.dir" location="${root}/src/java"/>
- <property name="src.test" location="${root}/src/test"/>
-
- <available file="${src.test}" type="dir" property="test.available"/>
-
- <property name="conf.dir" location="${nutch.root}/conf"/>
-
- <property name="build.dir" location="${nutch.root}/build/${name}"/>
- <property name="build.classes" location="${build.dir}/classes"/>
- <property name="build.test" location="${build.dir}/test"/>
- <property name="build.test.lib" location="${build.test}/lib"/>
-
- <property name="deploy.dir" location="${nutch.root}/build/plugins/${name}"/>
-
- <!-- load nutch defaults last so that they can be overridden above -->
- <property file="${nutch.root}/default.properties" />
-
- <ivy:settings id="ivy.instance" file="${nutch.root}/ivy/ivysettings.xml" />
-
- <path id="plugin.deps"/>
-
- <fileset id="lib.jars" dir="${root}" includes="lib/*.jar"/>
-
- <!-- the normal classpath -->
- <path id="classpath">
- <pathelement location="${build.classes}"/>
- <fileset refid="lib.jars"/>
- <pathelement location="${nutch.root}/build/classes"/>
- <fileset dir="${nutch.root}/build/lib">
- <include name="*.jar" />
- </fileset>
- <path refid="plugin.deps"/>
- <fileset dir="${deploy.dir}">
- <include name="*.jar" />
- </fileset>
- </path>
-
- <!-- the unit test classpath -->
- <path id="test.classpath">
- <pathelement location="${build.test}" />
- <pathelement location="${nutch.root}/build/test/classes"/>
- <pathelement location="${nutch.root}/src/test"/>
- <pathelement location="${conf.dir}"/>
- <pathelement location="${nutch.root}/build"/>
- <!-- test dependencies specific to current plugin -->
- <fileset dir="${build.test.lib}">
- <include name="*.jar" />
- </fileset>
- <!-- global test dependencies -->
- <fileset dir="${nutch.root}/build/test/lib">
- <include name="*.jar" />
- </fileset>
- <path refid="classpath"/>
- </path>
-
- <!-- ====================================================== -->
- <!-- Stuff needed by all targets -->
- <!-- ====================================================== -->
- <target name="init">
- <mkdir dir="${build.dir}"/>
- <mkdir dir="${build.classes}"/>
- <mkdir dir="${build.test}"/>
- <mkdir dir="${build.test.lib}"/>
- <mkdir dir="${deploy.dir}"/>
-
- <antcall target="init-plugin"/>
- </target>
-
- <!-- to be overridden by sub-projects -->
- <target name="init-plugin"/>
-
- <!--
- ! Used to build plugin compilation dependencies
- ! (to be overridden by plugins)
- !-->
- <target name="deps-jar"/>
-
- <!--
- ! Used to deploy plugin runtime dependencies
- ! (to be overridden by plugins)
- !-->
- <target name="deps-test"/>
-
- <!--
- ! Used to compile test for plugin runtime dependencies
- ! (to be overridden by plugins)
- !-->
- <target name="deps-test-compile"/>
-
- <!-- ====================================================== -->
- <!-- Compile the Java files -->
- <!-- ====================================================== -->
- <target name="compile" depends="init,deps-jar, resolve-default">
- <echo message="Compiling plugin: ${name}"/>
- <javac
- encoding="${build.encoding}"
- srcdir="${src.dir}"
- includes="**/*.java"
- destdir="${build.classes}"
- debug="${javac.debug}"
- optimize="${javac.optimize}"
- target="${javac.version}"
- source="${javac.version}"
- deprecation="${javac.deprecation}">
- <classpath refid="classpath"/>
- </javac>
- </target>
-
- <target name="compile-core">
- <ant target="compile-core" inheritall="false" dir="${nutch.root}"/>
- <ant target="compile"/>
- </target>
-
- <!-- ================================================================== -->
- <!-- Make plugin .jar -->
- <!-- ================================================================== -->
- <!-- -->
- <!-- ================================================================== -->
- <target name="jar" depends="compile">
- <jar
- jarfile="${build.dir}/${name}.jar"
- basedir="${build.classes}"
- />
- </target>
-
- <target name="jar-core" depends="compile-core">
- <jar
- jarfile="${build.dir}/${name}.jar"
- basedir="${build.classes}"
- />
- </target>
-
- <!-- ================================================================== -->
- <!-- Deploy plugin to ${deploy.dir} -->
- <!-- ================================================================== -->
- <!-- -->
- <!-- ================================================================== -->
- <target name="deploy" depends="jar, deps-test">
- <mkdir dir="${deploy.dir}"/>
- <copy file="plugin.xml" todir="${deploy.dir}"
- preservelastmodified="true"/>
- <available property="lib-available"
- file="${build.dir}/${name}.jar"/>
- <antcall target="copy-generated-lib"/>
- <copy todir="${deploy.dir}" flatten="true">
- <fileset refid="lib.jars"/>
- </copy>
- </target>
-
- <target name="copy-generated-lib" if="lib-available">
- <copy file="${build.dir}/${name}.jar" todir="${deploy.dir}" failonerror="false"/>
- </target>
-
- <!-- ================================================================== -->
- <!-- Compile test code -->
- <!-- ================================================================== -->
- <target name="compile-test" depends="compile, deps-test-compile" if="test.available">
- <javac
- encoding="${build.encoding}"
- srcdir="${src.test}"
- includes="**/*.java"
- destdir="${build.test}"
- debug="${javac.debug}"
- optimize="${javac.optimize}"
- target="${javac.version}"
- source="${javac.version}"
- deprecation="${javac.deprecation}">
- <classpath refid="test.classpath"/>
- </javac>
- </target>
-
- <!-- ================================================================== -->
- <!-- Run unit tests -->
- <!-- ================================================================== -->
- <target name="test" depends="compile-test, deploy" if="test.available">
- <echo message="Testing plugin: ${name}"/>
-
- <junit printsummary="yes" haltonfailure="no" fork="yes"
- errorProperty="tests.failed" failureProperty="tests.failed">
- <sysproperty key="test.data" value="${build.test}/data"/>
- <sysproperty key="test.input" value="${root}/data"/>
- <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
- <classpath refid="test.classpath"/>
- <formatter type="${test.junit.output.format}" />
- <batchtest todir="${build.test}" unless="testcase">
- <fileset dir="${src.test}"
- includes="**/Test*.java" excludes="**/${test.exclude}.java" />
- </batchtest>
- <batchtest todir="${build.test}" if="testcase">
- <fileset dir="${src.test}" includes="**/${testcase}.java"/>
- </batchtest>
- </junit>
-
- <fail if="tests.failed">Tests failed!</fail>
-
- </target>
-
- <!-- target: resolve ================================================= -->
- <target name="resolve-default" depends="clean-lib" description="resolve and retrieve dependencies with ivy">
- <ivy:resolve file="ivy.xml" conf="default" log="download-only"/>
- <ivy:retrieve pattern="${deploy.dir}/[artifact]-[revision].[ext]" symlink="false" log="quiet"/>
- </target>
-
- <target name="resolve-test" depends="clean-lib" description="resolve and retrieve dependencies with ivy">
- <ivy:resolve file="ivy.xml" conf="test" log="download-only"/>
- <ivy:retrieve pattern="${build.test.lib}/[artifact]-[revision].[ext]" symlink="false" log="quiet"/>
- </target>
-
- <!-- ================================================================== -->
- <!-- Clean. Delete the build files, and their directories -->
- <!-- ================================================================== -->
- <!-- target: clean =================================================== -->
- <target name="clean" depends="clean-build, clean-lib" description="--> clean the project" />
-
- <!-- target: clean-lib =============================================== -->
- <target name="clean-lib" description="--> clean the project libraries directory (dependencies)">
- <delete includeemptydirs="true" dir="${build.lib.dir}"/>
- </target>
-
- <!-- target: clean-build ============================================= -->
- <target name="clean-build" description="--> clean the project built files">
- <delete includeemptydirs="true" dir="${build.dir}"/>
- <delete includeemptydirs="true" dir="${deploy.dir}"/>
- </target>
-
-</project>