You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ma...@apache.org on 2012/02/07 14:25:47 UTC
svn commit: r1241460 - in /nutch/trunk: ./ src/plugin/ src/plugin/headings/
src/plugin/headings/src/ src/plugin/headings/src/java/
src/plugin/headings/src/java/org/ src/plugin/headings/src/java/org/apache/
src/plugin/headings/src/java/org/apache/nutch/...
Author: markus
Date: Tue Feb 7 13:25:46 2012
New Revision: 1241460
URL: http://svn.apache.org/viewvc?rev=1241460&view=rev
Log:
NUTCH-1005 Parse headings plugin
Added:
nutch/trunk/src/plugin/headings/
nutch/trunk/src/plugin/headings/build.xml
nutch/trunk/src/plugin/headings/ivy.xml
nutch/trunk/src/plugin/headings/plugin.xml
nutch/trunk/src/plugin/headings/src/
nutch/trunk/src/plugin/headings/src/java/
nutch/trunk/src/plugin/headings/src/java/org/
nutch/trunk/src/plugin/headings/src/java/org/apache/
nutch/trunk/src/plugin/headings/src/java/org/apache/nutch/
nutch/trunk/src/plugin/headings/src/java/org/apache/nutch/parse/
nutch/trunk/src/plugin/headings/src/java/org/apache/nutch/parse/headings/
nutch/trunk/src/plugin/headings/src/java/org/apache/nutch/parse/headings/HeadingsParseFilter.java
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/plugin/build.xml
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1241460&r1=1241459&r2=1241460&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue Feb 7 13:25:46 2012
@@ -1,5 +1,7 @@
Nutch Change Log
+* NUTCH-1005 Parse headings plugin (markus)
+
* NUTCH-1264 Configurable indexing plugin index-metadata (jnioche)
* NUTCH-1242 Allow disabling of URL Filters in ParseSegment (Edward Drapkin via markus)
Modified: nutch/trunk/src/plugin/build.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/build.xml?rev=1241460&r1=1241459&r2=1241460&view=diff
==============================================================================
--- nutch/trunk/src/plugin/build.xml (original)
+++ nutch/trunk/src/plugin/build.xml Tue Feb 7 13:25:46 2012
@@ -28,6 +28,7 @@
<target name="deploy">
<ant dir="creativecommons" target="deploy"/>
<ant dir="feed" target="deploy"/>
+ <ant dir="headings" target="deploy"/>
<ant dir="index-basic" target="deploy"/>
<ant dir="index-anchor" target="deploy"/>
<ant dir="index-more" target="deploy"/>
@@ -100,6 +101,7 @@
<target name="clean">
<ant dir="creativecommons" target="clean"/>
<ant dir="feed" target="clean"/>
+ <ant dir="headings" target="clean"/>
<ant dir="index-basic" target="clean"/>
<ant dir="index-anchor" target="clean"/>
<ant dir="index-more" target="clean"/>
Added: nutch/trunk/src/plugin/headings/build.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/headings/build.xml?rev=1241460&view=auto
==============================================================================
--- nutch/trunk/src/plugin/headings/build.xml (added)
+++ nutch/trunk/src/plugin/headings/build.xml Tue Feb 7 13:25:46 2012
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project name="headings" default="jar-core">
+
+ <import file="../build-plugin.xml"/>
+
+</project>
Added: nutch/trunk/src/plugin/headings/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/headings/ivy.xml?rev=1241460&view=auto
==============================================================================
--- nutch/trunk/src/plugin/headings/ivy.xml (added)
+++ nutch/trunk/src/plugin/headings/ivy.xml Tue Feb 7 13:25:46 2012
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="${nutch.root}/ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/trunk/src/plugin/headings/plugin.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/headings/plugin.xml?rev=1241460&view=auto
==============================================================================
--- nutch/trunk/src/plugin/headings/plugin.xml (added)
+++ nutch/trunk/src/plugin/headings/plugin.xml Tue Feb 7 13:25:46 2012
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<plugin
+ id="headings"
+ name="Headings Parse Filter"
+ version="1.0.0"
+ provider-name="nutch.org">
+
+
+ <runtime>
+ <library name="headings.jar">
+ <export name="*"/>
+ </library>
+ </runtime>
+
+ <requires>
+ <import plugin="nutch-extensionpoints"/>
+ </requires>
+
+ <extension id="org.apache.nutch.parse.headings"
+ name="Nutch Headings Parse Filter"
+ point="org.apache.nutch.parse.HtmlParseFilter">
+
+ <implementation id="HeadingsParseFilter"
+ class="org.apache.nutch.parse.headings.HeadingsParseFilter">
+ </implementation>
+
+ </extension>
+
+</plugin>
Added: nutch/trunk/src/plugin/headings/src/java/org/apache/nutch/parse/headings/HeadingsParseFilter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/headings/src/java/org/apache/nutch/parse/headings/HeadingsParseFilter.java?rev=1241460&view=auto
==============================================================================
--- nutch/trunk/src/plugin/headings/src/java/org/apache/nutch/parse/headings/HeadingsParseFilter.java (added)
+++ nutch/trunk/src/plugin/headings/src/java/org/apache/nutch/parse/headings/HeadingsParseFilter.java Tue Feb 7 13:25:46 2012
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.parse.headings;
+
+// Nutch imports
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.parse.HTMLMetaTags;
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.parse.HtmlParseFilter;
+import org.apache.nutch.parse.ParseResult;
+import org.apache.nutch.protocol.Content;
+import org.apache.nutch.util.NodeWalker;
+
+// W3C imports
+import org.w3c.dom.*;
+
+/**
+ * HtmlParseFilter to retrieve h1 and h2 values from the DOM.
+ */
+public class HeadingsParseFilter implements HtmlParseFilter {
+
+ private Configuration conf;
+ private DocumentFragment doc;
+ private String[] headings;
+
+ public ParseResult filter(Content content, ParseResult parseResult, HTMLMetaTags metaTags, DocumentFragment doc) {
+ this.doc = doc;
+
+ String heading;
+ Parse parse = parseResult.get(content.getUrl());
+
+ for (int i = 0 ; headings != null && i < headings.length ; i++ ) {
+ heading = getElement(headings[i]);
+
+ if (heading != null) {
+ parse.getData().getParseMeta().set(headings[i], heading.trim());
+ }
+ }
+
+ return parseResult;
+ }
+
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+
+ headings = conf.getStrings("headings");
+ }
+
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ /**
+ * Finds the specified element and returns its value
+ */
+ protected String getElement(String element) {
+ NodeWalker walker = new NodeWalker(doc);
+
+ while (walker.hasNext()) {
+ Node currentNode = walker.nextNode();
+
+ if (currentNode.getNodeType() == Node.ELEMENT_NODE) {
+ if (element.equalsIgnoreCase(currentNode.getNodeName())) {
+ return getNodeValue(currentNode);
+ }
+ }
+ }
+
+ // Seems nothing is found
+ return null;
+ }
+
+ /**
+ * Returns the text value of the specified Node and child nodes
+ */
+ protected static String getNodeValue(Node node) {
+ StringBuffer buffer = new StringBuffer();
+
+ NodeList children = node.getChildNodes();
+
+ for (int i = 0; i < children.getLength(); i++) {
+ if (children.item(i).getNodeType() == Node.TEXT_NODE) {
+ buffer.append(children.item(i).getNodeValue());
+ }
+ }
+
+ return buffer.toString();
+ }
+
+}