svn commit: r965806 - in /nutch/branches/nutchbase/src/plugin: build.xml microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java

[jn...@apache.org]

Author: jnioche
Date: Tue Jul 20 11:18:18 2010
New Revision: 965806

URL: http://svn.apache.org/viewvc?rev=965806&view=rev
Log:
ported microformat plugin

Modified:
    nutch/branches/nutchbase/src/plugin/build.xml
    nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java
    nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java

Modified: nutch/branches/nutchbase/src/plugin/build.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/build.xml?rev=965806&r1=965805&r2=965806&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/build.xml (original)
+++ nutch/branches/nutchbase/src/plugin/build.xml Tue Jul 20 11:18:18 2010
@@ -35,6 +35,7 @@
      <ant dir="lib-nekohtml" target="deploy"/>
      <ant dir="lib-regex-filter" target="deploy"/>
      <ant dir="lib-xml" target="deploy"/>
+     <ant dir="microformats-reltag" target="deploy"/>
      <ant dir="nutch-extensionpoints" target="deploy"/>
      <ant dir="protocol-ftp" target="deploy"/>
      <ant dir="protocol-file" target="deploy"/>
@@ -57,7 +58,6 @@
      <ant dir="urlnormalizer-regex" target="deploy"/>
      <!--
      <ant dir="feed" target="deploy"/>
-     <ant dir="microformats-reltag" target="deploy"/>
      <ant dir="parse-ext" target="deploy"/>
      <ant dir="parse-swf" target="deploy"/>
      <ant dir="parse-zip" target="deploy"/>

Modified: nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java?rev=965806&r1=965805&r2=965806&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java (original)
+++ nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java Tue Jul 20 11:18:18 2010
@@ -16,63 +16,64 @@
  */
 package org.apache.nutch.microformats.reltag;
 
-
 // Nutch imports
-import org.apache.nutch.crawl.CrawlDatum;
-import org.apache.nutch.crawl.Inlinks;
-import org.apache.nutch.indexer.IndexingFilter;
-import org.apache.nutch.indexer.IndexingException;
-import org.apache.nutch.indexer.NutchDocument;
-import org.apache.hadoop.io.Text;
-import org.apache.nutch.parse.Parse;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.HashSet;
 
-// Hadoop imports
+import org.apache.avro.util.Utf8;
 import org.apache.hadoop.conf.Configuration;
-
+import org.apache.nutch.indexer.IndexingException;
+import org.apache.nutch.indexer.IndexingFilter;
+import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.storage.WebPage.Field;
 
 /**
- * An {@link org.apache.nutch.indexer.IndexingFilter} that 
- * add <code>tag</code> field(s) to the document.
- *
+ * An {@link org.apache.nutch.indexer.IndexingFilter} that add <code>tag</code>
+ * field(s) to the document.
+ * 
  * @see <a href="http://www.microformats.org/wiki/rel-tag">
  *      http://www.microformats.org/wiki/rel-tag</a>
  * @author J&eacute;r&ocirc;me Charron
  */
 public class RelTagIndexingFilter implements IndexingFilter {
-  
 
-  private Configuration conf;
+	private Configuration conf;
+
+	private static final Collection<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();
 
+	static {
+		FIELDS.add(WebPage.Field.BASE_URL);
+		FIELDS.add(WebPage.Field.METADATA);
+	}
+
+	@Override
+	public Collection<Field> getFields() {
+		return FIELDS;
+	}
+
+	public void setConf(Configuration conf) {
+		this.conf = conf;
+	}
+
+	public Configuration getConf() {
+		return this.conf;
+	}
+
+	@Override
+	public NutchDocument filter(NutchDocument doc, String url, WebPage page)
+			throws IndexingException {
+		// Check if some Rel-Tags found, possibly put there by RelTagParser
+		ByteBuffer bb = page.getFromMetadata(new Utf8(RelTagParser.REL_TAG));
+		
+		if (bb != null) {
+			String[] tags = new String(bb.array()).split("\t");
+			for (int i = 0; i < tags.length; i++) {
+				doc.add("tag", tags[i]);
+			}
+		}
 
-  // Inherited JavaDoc
-  public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks)
-    throws IndexingException {
-
-    // Check if some Rel-Tags found, possibly put there by RelTagParser
-    String[] tags = parse.getData().getParseMeta().getValues(RelTagParser.REL_TAG);
-    if (tags != null) {
-      for (int i=0; i<tags.length; i++) {
-        doc.add("tag", tags[i]);
-      }
-    }
-
-    return doc;
-  }
-
-  /* ----------------------------- *
-   * <implementation:Configurable> *
-   * ----------------------------- */
-  
-  public void setConf(Configuration conf) {
-    this.conf = conf;
-  }
-
-  public Configuration getConf() {
-    return this.conf;
-  }
-  
-  /* ------------------------------ *
-   * </implementation:Configurable> *
-   * ------------------------------ */
-  
+		return doc;
+	}
 }

Modified: nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java?rev=965806&r1=965805&r2=965806&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java (original)
+++ nutch/branches/nutchbase/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java Tue Jul 20 11:18:18 2010
@@ -19,142 +19,141 @@ package org.apache.nutch.microformats.re
 // JDK imports
 import java.net.URL;
 import java.net.URLDecoder;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
 import java.util.TreeSet;
-import org.w3c.dom.DocumentFragment;
-import org.w3c.dom.NamedNodeMap;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
 
-// Commons Logging imports
+import org.apache.avro.util.Utf8;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
-// Nutch imports
-import org.apache.nutch.metadata.Metadata;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.nutch.parse.HTMLMetaTags;
-import org.apache.nutch.parse.Parse;
-import org.apache.nutch.parse.ParseResult;
 import org.apache.nutch.parse.HtmlParseFilter;
-import org.apache.nutch.protocol.Content;
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.storage.WebPage.Field;
 import org.apache.nutch.util.StringUtil;
-
-// Hadoop imports
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-
+import org.w3c.dom.DocumentFragment;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
 
 /**
  * Adds microformat rel-tags of document if found.
- *
+ * 
  * @see <a href="http://www.microformats.org/wiki/rel-tag">
  *      http://www.microformats.org/wiki/rel-tag</a>
  * @author J&eacute;r&ocirc;me Charron
  */
 public class RelTagParser implements HtmlParseFilter {
-  
-  public final static Log LOG = LogFactory.getLog(RelTagParser.class);
 
-  public final static String REL_TAG = "Rel-Tag";
-  
-  
-  private Configuration conf = null;
-  
-  
-  /**
-   * Scan the HTML document looking at possible rel-tags
-   */
-  public ParseResult filter(Content content, ParseResult parseResult,
-    HTMLMetaTags metaTags, DocumentFragment doc) {
-    
-    // get parse obj
-    Parse parse = parseResult.get(content.getUrl());
-    // Trying to find the document's rel-tags
-    Parser parser = new Parser(doc);
-    Set tags = parser.getRelTags();
-    Iterator iter = tags.iterator();
-    Metadata metadata = parse.getData().getParseMeta();
-    while (iter.hasNext()) {
-      metadata.add(REL_TAG, (String) iter.next());
-    }
-    return parseResult;
-  }
-
-  private static class Parser {
-
-    Set tags = null;
-    
-    Parser(Node node) {
-      tags = new TreeSet();
-      parse(node);
-    }
-  
-    Set getRelTags() {
-      return tags;
-    }
-    
-    void parse(Node node) {
-
-      if (node.getNodeType() == Node.ELEMENT_NODE) {
-        // Look for <a> tag
-        if ("a".equalsIgnoreCase(node.getNodeName())) {
-          NamedNodeMap attrs = node.getAttributes();
-          Node hrefNode = attrs.getNamedItem("href");
-          // Checks that it contains a href attribute
-          if (hrefNode != null) {
-            Node relNode = attrs.getNamedItem("rel");
-            // Checks that it contains a rel attribute too
-            if (relNode != null) {
-              // Finaly checks that rel=tag
-              if ("tag".equalsIgnoreCase(relNode.getNodeValue())) {
-                String tag = parseTag(hrefNode.getNodeValue());
-                if (!StringUtil.isEmpty(tag)) {
-                  tags.add(tag);
-                }
-              }
-            }
-          }
-        }
-      }
-      
-      // Recurse
-      NodeList children = node.getChildNodes();
-      for (int i=0; children != null && i<children.getLength(); i++) {
-        parse(children.item(i));
-      }
-    }
-    
-    private final static String parseTag(String url) {
-      String tag = null;
-      try {
-        URL u = new URL(url);
-        String path = u.getPath();
-        tag = URLDecoder.decode(path.substring(path.lastIndexOf('/') + 1), "UTF-8");
-      } catch (Exception e) {
-        // Malformed tag...
-        tag = null;
-      }
-      return tag;
-    }
-    
-  }
-
-
-  /* ----------------------------- *
-   * <implementation:Configurable> *
-   * ----------------------------- */
-  
-  public void setConf(Configuration conf) {
-    this.conf = conf;
-  }
-
-  public Configuration getConf() {
-    return this.conf;
-  }
-  
-  /* ------------------------------ *
-   * </implementation:Configurable> *
-   * ------------------------------ */
-  
+	public final static Log LOG = LogFactory.getLog(RelTagParser.class);
+
+	public final static String REL_TAG = "Rel-Tag";
+
+	private Configuration conf = null;
+
+	private static class Parser {
+
+		Set<String> tags = null;
+
+		Parser(Node node) {
+			tags = new TreeSet<String>();
+			parse(node);
+		}
+
+		Set<String> getRelTags() {
+			return tags;
+		}
+
+		void parse(Node node) {
+
+			if (node.getNodeType() == Node.ELEMENT_NODE) {
+				// Look for <a> tag
+				if ("a".equalsIgnoreCase(node.getNodeName())) {
+					NamedNodeMap attrs = node.getAttributes();
+					Node hrefNode = attrs.getNamedItem("href");
+					// Checks that it contains a href attribute
+					if (hrefNode != null) {
+						Node relNode = attrs.getNamedItem("rel");
+						// Checks that it contains a rel attribute too
+						if (relNode != null) {
+							// Finaly checks that rel=tag
+							if ("tag".equalsIgnoreCase(relNode.getNodeValue())) {
+								String tag = parseTag(hrefNode.getNodeValue());
+								if (!StringUtil.isEmpty(tag)) {
+									tags.add(tag);
+								}
+							}
+						}
+					}
+				}
+			}
+
+			// Recurse
+			NodeList children = node.getChildNodes();
+			for (int i = 0; children != null && i < children.getLength(); i++) {
+				parse(children.item(i));
+			}
+		}
+
+		private final static String parseTag(String url) {
+			String tag = null;
+			try {
+				URL u = new URL(url);
+				String path = u.getPath();
+				tag = URLDecoder.decode(
+						path.substring(path.lastIndexOf('/') + 1), "UTF-8");
+			} catch (Exception e) {
+				// Malformed tag...
+				tag = null;
+			}
+			return tag;
+		}
+
+	}
+
+	public void setConf(Configuration conf) {
+		this.conf = conf;
+	}
+
+	public Configuration getConf() {
+		return this.conf;
+	}
+
+	private static final Collection<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();
+
+	static {
+		FIELDS.add(WebPage.Field.BASE_URL);
+		FIELDS.add(WebPage.Field.METADATA);
+	}
+	
+	@Override
+	public Collection<Field> getFields() {
+		return FIELDS;
+	}
+
+	@Override
+	/**
+	 * Scan the HTML document looking at possible rel-tags
+	 */
+	public Parse filter(String url, WebPage page, Parse parse,
+			HTMLMetaTags metaTags, DocumentFragment doc) {
+		// Trying to find the document's rel-tags
+		Parser parser = new Parser(doc);
+		Set<String> tags = parser.getRelTags();
+		// can't store multiple values in page metadata -> separate by tabs
+		StringBuffer sb = new StringBuffer();
+		Iterator<String> iter = tags.iterator();
+		while (iter.hasNext()) {
+			sb.append(iter.next());
+			sb.append("\t");
+		}
+		ByteBuffer bb = ByteBuffer.wrap(sb.toString().getBytes());
+		page.putToMetadata(new Utf8(REL_TAG), bb);
+		return parse;
+	}
 }