You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by jn...@apache.org on 2014/05/15 10:10:09 UTC
svn commit: r1594812 [4/5] - in /nutch/branches/2.x: ./ ivy/ src/gora/
src/java/org/apache/nutch/api/ src/java/org/apache/nutch/crawl/
src/java/org/apache/nutch/fetcher/ src/java/org/apache/nutch/host/
src/java/org/apache/nutch/indexer/ src/java/org/ap...
Modified: nutch/branches/2.x/src/java/org/apache/nutch/util/EncodingDetector.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/util/EncodingDetector.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/util/EncodingDetector.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/util/EncodingDetector.java Thu May 15 08:10:07 2014
@@ -16,6 +16,15 @@
*/
package org.apache.nutch.util;
+import com.ibm.icu.text.CharsetDetector;
+import com.ibm.icu.text.CharsetMatch;
+import org.apache.avro.util.Utf8;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.net.protocols.Response;
+import org.apache.nutch.storage.WebPage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.ByteArrayInputStream;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
@@ -24,16 +33,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
-import org.apache.avro.util.Utf8;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.net.protocols.Response;
-import org.apache.nutch.storage.WebPage;
-
-import com.ibm.icu.text.CharsetDetector;
-import com.ibm.icu.text.CharsetMatch;
-
/**
* A simple class for detecting character encodings.
*
@@ -165,10 +164,10 @@ public class EncodingDetector {
public void autoDetectClues(WebPage page, boolean filter) {
autoDetectClues(page.getContent(), page.getContentType(),
- parseCharacterEncoding(page.getFromHeaders(CONTENT_TYPE_UTF8)), filter);
+ parseCharacterEncoding(page.getHeaders().get(CONTENT_TYPE_UTF8)), filter);
}
- private void autoDetectClues(ByteBuffer dataBuffer, Utf8 typeUtf8,
+ private void autoDetectClues(ByteBuffer dataBuffer, CharSequence typeUtf8,
String encoding, boolean filter) {
int length = dataBuffer.remaining();
String type = TableUtil.toString(typeUtf8);
@@ -224,7 +223,7 @@ public class EncodingDetector {
* @return Guessed encoding or defaultValue
*/
public String guessEncoding(WebPage page, String defaultValue) {
- Utf8 baseUrlUtf8 = page.getBaseUrl();
+ CharSequence baseUrlUtf8 = page.getBaseUrl();
String baseUrl = TableUtil.toString(baseUrlUtf8);
return guessEncoding(baseUrl, defaultValue);
}
@@ -347,9 +346,9 @@ public class EncodingDetector {
* This method was copied from org.apache.catalina.util.RequestUtil,
* which is licensed under the Apache License, Version 2.0 (the "License").
*
- * @param contentType a content type header
+ * @param contentTypeUtf8
*/
- public static String parseCharacterEncoding(Utf8 contentTypeUtf8) {
+ public static String parseCharacterEncoding(CharSequence contentTypeUtf8) {
if (contentTypeUtf8 == null)
return (null);
String contentType = contentTypeUtf8.toString();
Modified: nutch/branches/2.x/src/java/org/apache/nutch/util/NutchJob.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/util/NutchJob.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/util/NutchJob.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/util/NutchJob.java Thu May 15 08:10:07 2014
@@ -17,14 +17,14 @@
package org.apache.nutch.util;
-import java.io.IOException;
-
import org.apache.avro.util.Utf8;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
import org.apache.nutch.metadata.Nutch;
+import java.io.IOException;
+
/** A {@link Job} for Nutch jobs. */
public class NutchJob extends Job {
@@ -58,7 +58,7 @@ public class NutchJob extends Job {
return succeeded;
}
- public static boolean shouldProcess(Utf8 mark, Utf8 batchId) {
+ public static boolean shouldProcess(CharSequence mark, Utf8 batchId) {
if (mark == null) {
return false;
}
Modified: nutch/branches/2.x/src/java/org/apache/nutch/util/TableUtil.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/util/TableUtil.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/util/TableUtil.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/util/TableUtil.java Thu May 15 08:10:07 2014
@@ -16,14 +16,13 @@
******************************************************************************/
package org.apache.nutch.util;
+import org.apache.avro.util.Utf8;
+import org.apache.commons.lang.StringUtils;
+
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.ByteBuffer;
-import org.apache.avro.util.Utf8;
-import org.apache.commons.lang.StringUtils;
-import org.apache.nutch.util.StringUtil;
-
public class TableUtil {
public static final ByteBuffer YES_VAL = ByteBuffer.wrap(new byte[] { 'y' });
@@ -148,11 +147,12 @@ public class TableUtil {
* Convert given Utf8 instance to String and and cleans out
* any offending "�" from the String.
*
+ *
* @param utf8
* Utf8 object
* @return string-ifed Utf8 object or null if Utf8 instance is null
*/
- public static String toString(Utf8 utf8) {
+ public static String toString(CharSequence utf8) {
return (utf8 == null ? null : StringUtil.cleanField(utf8.toString()));
}
Modified: nutch/branches/2.x/src/java/org/apache/nutch/util/WebPageWritable.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/util/WebPageWritable.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/util/WebPageWritable.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/util/WebPageWritable.java Thu May 15 08:10:07 2014
@@ -16,15 +16,15 @@
******************************************************************************/
package org.apache.nutch.util;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
+import org.apache.gora.util.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.Writable;
import org.apache.nutch.storage.WebPage;
-import org.apache.gora.util.IOUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
public class WebPageWritable extends Configured
implements Writable {
@@ -32,7 +32,7 @@ implements Writable {
private WebPage webPage;
public WebPageWritable() {
- this(null, new WebPage());
+ this(null, WebPage.newBuilder().build());
}
public WebPageWritable(Configuration conf, WebPage webPage) {
Modified: nutch/branches/2.x/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java Thu May 15 08:10:07 2014
@@ -17,16 +17,7 @@
package org.creativecommons.nutch;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.StringTokenizer;
-
import org.apache.avro.util.Utf8;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.IndexingFilter;
@@ -35,6 +26,15 @@ import org.apache.nutch.metadata.Creativ
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.StringTokenizer;
/** Adds basic searchable fields to a document. */
public class CCIndexingFilter implements IndexingFilter {
@@ -100,7 +100,7 @@ public class CCIndexingFilter implements
public NutchDocument filter(NutchDocument doc, String url, WebPage page)
throws IndexingException {
- ByteBuffer blicense = page.getFromMetadata(new Utf8(
+ ByteBuffer blicense = page.getMetadata().get(new Utf8(
CreativeCommons.LICENSE_URL));
if (blicense != null) {
String licenseUrl = Bytes.toString(blicense);
@@ -117,7 +117,7 @@ public class CCIndexingFilter implements
}
// index the license location as cc:meta=xxx
- ByteBuffer blicenseloc = page.getFromMetadata(new Utf8(
+ ByteBuffer blicenseloc = page.getMetadata().get(new Utf8(
CreativeCommons.LICENSE_LOCATION));
if (blicenseloc != null) {
String licenseLocation = Bytes.toString(blicenseloc);
@@ -125,7 +125,7 @@ public class CCIndexingFilter implements
}
// index the work type cc:type=xxx
- ByteBuffer bworkType = page.getFromMetadata(new Utf8(
+ ByteBuffer bworkType = page.getMetadata().get(new Utf8(
CreativeCommons.WORK_TYPE));
if (bworkType != null) {
String workType = Bytes.toString(bworkType);
Modified: nutch/branches/2.x/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java (original)
+++ nutch/branches/2.x/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java Thu May 15 08:10:07 2014
@@ -17,6 +17,19 @@
package org.creativecommons.nutch;
+import org.apache.avro.util.Utf8;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.metadata.CreativeCommons;
+import org.apache.nutch.parse.*;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.storage.WebPage.Field;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.*;
+import org.xml.sax.InputSource;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URL;
@@ -25,29 +38,6 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
-import org.apache.avro.util.Utf8;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.metadata.CreativeCommons;
-import org.apache.nutch.parse.HTMLMetaTags;
-import org.apache.nutch.parse.ParseFilter;
-import org.apache.nutch.parse.Parse;
-import org.apache.nutch.parse.ParseException;
-import org.apache.nutch.parse.ParseStatusUtils;
-import org.apache.nutch.storage.WebPage;
-import org.apache.nutch.storage.WebPage.Field;
-import org.w3c.dom.Comment;
-import org.w3c.dom.Document;
-import org.w3c.dom.DocumentFragment;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
-
/** Adds metadata identifying the Creative Commons license used, if any. */
public class CCParseFilter implements ParseFilter {
public static final Logger LOG = LoggerFactory.getLogger(CCParseFilter.class);
@@ -95,9 +85,9 @@ public class CCParseFilter implements Pa
if (LOG.isDebugEnabled()) {
LOG.debug("CC: found " + licenseUrl + " in " + licenseLocation + " of " + base);
}
- page.putToMetadata(new Utf8(CreativeCommons.LICENSE_URL),
+ page.getMetadata().put(new Utf8(CreativeCommons.LICENSE_URL),
ByteBuffer.wrap(licenseUrl.getBytes()));
- page.putToMetadata(new Utf8(CreativeCommons.LICENSE_LOCATION),
+ page.getMetadata().put(new Utf8(CreativeCommons.LICENSE_LOCATION),
ByteBuffer.wrap(licenseLocation.getBytes()));
}
@@ -105,7 +95,7 @@ public class CCParseFilter implements Pa
if (LOG.isDebugEnabled()) {
LOG.debug("CC: found " + walker.workType + " in " + base);
}
- page.putToMetadata(new Utf8(CreativeCommons.WORK_TYPE),
+ page.getMetadata().put(new Utf8(CreativeCommons.WORK_TYPE),
ByteBuffer.wrap(walker.workType.getBytes()));
}
Modified: nutch/branches/2.x/src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java (original)
+++ nutch/branches/2.x/src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java Thu May 15 08:10:07 2014
@@ -17,18 +17,22 @@
package org.creativecommons.nutch;
-import org.apache.nutch.parse.ParseUtil;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.parse.ParseUtil;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.Bytes;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
-import java.io.*;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
import java.nio.ByteBuffer;
-import org.junit.Test;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
public class TestCCParseFilter {
@@ -64,7 +68,7 @@ public class TestCCParseFilter {
byte[] bytes = out.toByteArray();
Configuration conf = NutchConfiguration.create();
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setBaseUrl(new Utf8(url));
page.setContent(ByteBuffer.wrap(bytes));
MimeUtil mimeutil = new MimeUtil(conf);
@@ -73,11 +77,11 @@ public class TestCCParseFilter {
new ParseUtil(conf).parse(url, page);
- ByteBuffer bb = page.getFromMetadata(new Utf8("License-Url"));
+ ByteBuffer bb = page.getMetadata().get(new Utf8("License-Url"));
assertEquals(license, Bytes.toString(bb));
- bb = page.getFromMetadata(new Utf8("License-Location"));
+ bb = page.getMetadata().get(new Utf8("License-Location"));
assertEquals(location, Bytes.toString(bb));
- bb = page.getFromMetadata(new Utf8("Work-Type"));
+ bb = page.getMetadata().get(new Utf8("Work-Type"));
assertEquals(type, Bytes.toString(bb));
}
}
Modified: nutch/branches/2.x/src/plugin/index-anchor/src/java/org/apache/nutch/indexer/anchor/AnchorIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-anchor/src/java/org/apache/nutch/indexer/anchor/AnchorIndexingFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-anchor/src/java/org/apache/nutch/indexer/anchor/AnchorIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/index-anchor/src/java/org/apache/nutch/indexer/anchor/AnchorIndexingFilter.java Thu May 15 08:10:07 2014
@@ -16,10 +16,6 @@
*/
package org.apache.nutch.indexer.anchor;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map.Entry;
-
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.indexer.IndexingException;
@@ -30,6 +26,11 @@ import org.apache.nutch.util.TableUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.lang.CharSequence;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map.Entry;
+
/**
* Indexing filter that offers an option to either index all inbound anchor text for
* a document or deduplicate anchors. Deduplication does have it's con's,
@@ -82,7 +83,7 @@ public class AnchorIndexingFilter implem
throws IndexingException {
HashSet<String> set = null;
- for (Entry<Utf8, Utf8> e : page.getInlinks().entrySet()) {
+ for (Entry<CharSequence, CharSequence> e : page.getInlinks().entrySet()) {
String anchor = TableUtil.toString(e.getValue());
if(anchor.equals(""))
Modified: nutch/branches/2.x/src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java Thu May 15 08:10:07 2014
@@ -39,10 +39,10 @@ public class TestAnchorIndexingFilter {
AnchorIndexingFilter filter = new AnchorIndexingFilter();
filter.setConf(conf);
NutchDocument doc = new NutchDocument();
- WebPage page = new WebPage();
- page.putToInlinks(new Utf8("http://example1.com/"), new Utf8("cool site"));
- page.putToInlinks(new Utf8("http://example2.com/"), new Utf8("cool site"));
- page.putToInlinks(new Utf8("http://example3.com/"), new Utf8("fun site"));
+ WebPage page = WebPage.newBuilder().build();
+ page.getInlinks().put(new Utf8("http://example1.com/"), new Utf8("cool site"));
+ page.getInlinks().put(new Utf8("http://example2.com/"), new Utf8("cool site"));
+ page.getInlinks().put(new Utf8("http://example3.com/"), new Utf8("fun site"));
filter.filter(doc, "http://myurldoesnotmatter.com/", page);
assertTrue("test if there is an anchor at all", doc.getFieldNames().contains("anchor"));
Modified: nutch/branches/2.x/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java Thu May 15 08:10:07 2014
@@ -17,15 +17,6 @@
package org.apache.nutch.indexer.basic;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashSet;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.IndexingFilter;
@@ -35,6 +26,15 @@ import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.Bytes;
import org.apache.nutch.util.TableUtil;
import org.apache.solr.common.util.DateUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashSet;
/** Adds basic searchable fields to a document. The fields are:
* host - add host as un-stored, indexed and tokenized
@@ -74,9 +74,9 @@ public class BasicIndexingFilter impleme
throws IndexingException {
String reprUrl = null;
- if (page.isReadable(WebPage.Field.REPR_URL.getIndex())) {
+// if (page.isReadable(WebPage.Field.REPR_URL.getIndex())) {
reprUrl = TableUtil.toString(page.getReprUrl());
- }
+// }
String host = null;
try {
@@ -118,7 +118,7 @@ public class BasicIndexingFilter impleme
}
// add cached content/summary display policy, if available
ByteBuffer cachingRaw = page
- .getFromMetadata(Nutch.CACHING_FORBIDDEN_KEY_UTF8);
+ .getMetadata().get(Nutch.CACHING_FORBIDDEN_KEY_UTF8);
String caching = Bytes.toString(cachingRaw);
if (caching != null && !caching.equals(Nutch.CACHING_FORBIDDEN_NONE)) {
doc.add("cache", caching);
Modified: nutch/branches/2.x/src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java Thu May 15 08:10:07 2014
@@ -16,8 +16,6 @@
*/
package org.apache.nutch.indexer.basic;
-import java.nio.ByteBuffer;
-
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.indexer.NutchDocument;
@@ -25,6 +23,9 @@ import org.apache.nutch.metadata.Nutch;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
import static org.junit.Assert.*;
/**
@@ -47,13 +48,13 @@ public class TestBasicIndexingFilter {
filter.setConf(conf);
assertNotNull(filter);
NutchDocument doc = new NutchDocument();
- WebPage page = new WebPage();
- page.putToInlinks(new Utf8("http://nutch.apache.org/"), new Utf8("Welcome to Nutch"));
+ WebPage page = WebPage.newBuilder().build();
+ page.getInlinks().put(new Utf8("http://nutch.apache.org/"), new Utf8("Welcome to Nutch"));
page.setTitle(new Utf8("Welcome to Nutch"));
page.setReprUrl(new Utf8("http://www.urldoesnotmatter.org"));
byte[] bytes = new byte[10];
ByteBuffer bbuf = ByteBuffer.wrap(bytes);
- page.putToMetadata(Nutch.CACHING_FORBIDDEN_KEY_UTF8, bbuf);
+ page.getMetadata().put(Nutch.CACHING_FORBIDDEN_KEY_UTF8, bbuf);
page.setFetchTime(System.currentTimeMillis());
try {
filter.filter(doc, "http://www.apache.org/", page);
@@ -79,8 +80,8 @@ public class TestBasicIndexingFilter {
filter.setConf(conf);
assertNotNull(filter);
NutchDocument doc = new NutchDocument();
- WebPage page = new WebPage();
- page.putToInlinks(new Utf8("http://exceedmaximumtitleurl.org/"), new Utf8("exceeding title site"));
+ WebPage page = WebPage.newBuilder().build();
+ page.getInlinks().put(new Utf8("http://exceedmaximumtitleurl.org/"), new Utf8("exceeding title site"));
page.setTitle(new Utf8("This title exceeds maximum characters"));
try {
filter.filter(doc, "http://www.apache.org/", page);
Modified: nutch/branches/2.x/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java (original)
+++ nutch/branches/2.x/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/MetadataIndexer.java Thu May 15 08:10:07 2014
@@ -52,7 +52,7 @@ public class MetadataIndexer implements
// add the fields from parsemd
if (parseFieldnames != null) {
for (String metatag : parseFieldnames) {
- ByteBuffer bvalues = page.getFromMetadata(new Utf8(PARSE_META_PREFIX
+ ByteBuffer bvalues = page.getMetadata().get(new Utf8(PARSE_META_PREFIX
+ metatag));
if (bvalues != null) {
String value = new String(bvalues.array());
Modified: nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java Thu May 15 08:10:07 2014
@@ -1,19 +1,3 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
package org.apache.nutch.indexer.more;
import java.text.ParseException;
@@ -84,8 +68,8 @@ public class MoreIndexingFilter implemen
// last-modified, or, if that's not present, use fetch time.
private NutchDocument addTime(NutchDocument doc, WebPage page, String url) {
long time = -1;
- Utf8 lastModified = page
- .getFromHeaders(new Utf8(HttpHeaders.LAST_MODIFIED));
+ CharSequence lastModified = page
+ .getHeaders().get(new Utf8(HttpHeaders.LAST_MODIFIED));
// String lastModified = data.getMeta(Metadata.LAST_MODIFIED);
if (lastModified != null) { // try parse last-modified
time = getTime(lastModified.toString(), url); // use as time
@@ -139,8 +123,8 @@ public class MoreIndexingFilter implemen
// Add Content-Length
private NutchDocument addLength(NutchDocument doc, WebPage page, String url) {
- Utf8 contentLength = page.getFromHeaders(new Utf8(
- HttpHeaders.CONTENT_LENGTH));
+ CharSequence contentLength = page.getHeaders().get(new Utf8(
+ HttpHeaders.CONTENT_LENGTH));
if (contentLength != null) {
// NUTCH-1010 ContentLength not trimmed
String trimmed = contentLength.toString().trim();
@@ -173,9 +157,9 @@ public class MoreIndexingFilter implemen
*/
private NutchDocument addType(NutchDocument doc, WebPage page, String url) {
String mimeType = null;
- Utf8 contentType = page.getContentType();
+ CharSequence contentType = page.getContentType();
if (contentType == null)
- contentType = page.getFromHeaders(new Utf8(HttpHeaders.CONTENT_TYPE));
+ contentType = page.getHeaders().get(new Utf8(HttpHeaders.CONTENT_TYPE));
if (contentType == null) {
// Note by Jerome Charron on 20050415:
// Content Type not solved by a previous plugin
@@ -249,7 +233,7 @@ public class MoreIndexingFilter implemen
}
private NutchDocument resetTitle(NutchDocument doc, WebPage page, String url) {
- Utf8 contentDisposition = page.getFromHeaders(new Utf8(
+ CharSequence contentDisposition = page.getHeaders().get(new Utf8(
HttpHeaders.CONTENT_DISPOSITION));
if (contentDisposition == null)
return doc;
Modified: nutch/branches/2.x/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java Thu May 15 08:10:07 2014
@@ -16,11 +16,6 @@
*/
package org.apache.nutch.indexer.more;
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
-
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.indexer.IndexingException;
@@ -28,6 +23,11 @@ import org.apache.nutch.indexer.NutchDoc
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.EncodingDetector;
import org.apache.nutch.util.NutchConfiguration;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.*;
public class TestMoreIndexingFilter {
@@ -56,7 +56,7 @@ public class TestMoreIndexingFilter {
assertNotNull(filter);
NutchDocument doc = new NutchDocument();
try{
- filter.filter(doc, "http://nutch.apache.org/index.html", new WebPage());
+ filter.filter(doc, "http://nutch.apache.org/index.html", WebPage.newBuilder().build());
}
catch(Exception e){
e.printStackTrace();
@@ -78,11 +78,11 @@ public class TestMoreIndexingFilter {
private void assertContentType(Configuration conf, String source, String expected) throws IndexingException {
MoreIndexingFilter filter = new MoreIndexingFilter();
filter.setConf(conf);
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
String url = "http://www.example.com/";
page.setContent(ByteBuffer.wrap("text".getBytes()));
page.setTitle(new Utf8("title"));
- page.putToHeaders(EncodingDetector.CONTENT_TYPE_UTF8, new Utf8(source));
+ page.getHeaders().put(EncodingDetector.CONTENT_TYPE_UTF8, new Utf8(source));
NutchDocument doc = filter.filter(new NutchDocument(), url, page);
assertEquals("mime type not detected", expected, doc.getFieldValue("type"));
}
Modified: nutch/branches/2.x/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java (original)
+++ nutch/branches/2.x/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java Thu May 15 08:10:07 2014
@@ -17,33 +17,30 @@
package org.apache.nutch.analysis.lang;
// JDK imports
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Properties;
import org.apache.avro.util.Utf8;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.parse.HTMLMetaTags;
-import org.apache.nutch.parse.ParseFilter;
import org.apache.nutch.parse.Parse;
+import org.apache.nutch.parse.ParseFilter;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.Bytes;
import org.apache.nutch.util.NodeWalker;
import org.apache.tika.language.LanguageIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
+import java.lang.CharSequence;
+import java.nio.ByteBuffer;
+import java.util.*;
+
/**
* Adds metadata identifying language of document if found We could also run
* statistical analysis here but we'd miss all other formats
@@ -116,8 +113,8 @@ public class HTMLLanguageParser implemen
}
if (lang != null) {
- page.putToMetadata(new Utf8(Metadata.LANGUAGE), ByteBuffer.wrap(lang
- .getBytes()));
+ page.getMetadata().put(new Utf8(Metadata.LANGUAGE), ByteBuffer.wrap(lang
+ .getBytes()));
return parse;
}
@@ -138,7 +135,7 @@ public class HTMLLanguageParser implemen
return lang;
}
- Utf8 ulang = page.getFromHeaders(new Utf8(Response.CONTENT_LANGUAGE));
+ CharSequence ulang = page.getHeaders().get(new Utf8(Response.CONTENT_LANGUAGE));
if (ulang != null) {
lang = ulang.toString();
}
@@ -176,7 +173,7 @@ public class HTMLLanguageParser implemen
// Check in the metadata whether the language has already been stored there by
// Tika
private static ByteBuffer getLanguageFromMetadata(
- Map<Utf8, ByteBuffer> metadata) {
+ Map<CharSequence, ByteBuffer> metadata) {
if (metadata == null)
return null;
Modified: nutch/branches/2.x/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/LanguageIndexingFilter.java Thu May 15 08:10:07 2014
@@ -17,9 +17,6 @@
package org.apache.nutch.analysis.lang;
// Nutch imports
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.HashSet;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
@@ -31,6 +28,10 @@ import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.Bytes;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.HashSet;
+
/**
* An {@link org.apache.nutch.indexer.IndexingFilter} that adds a
* <code>lang</code> (language) field to the document.
@@ -61,7 +62,7 @@ public class LanguageIndexingFilter impl
throws IndexingException {
// check if LANGUAGE found, possibly put there by HTMLLanguageParser
- ByteBuffer blang = page.getFromMetadata(new Utf8(Metadata.LANGUAGE));
+ ByteBuffer blang = page.getMetadata().get(new Utf8(Metadata.LANGUAGE));
String lang = Bytes.toString(blang);
if (lang == null || lang.length() == 0) {
Modified: nutch/branches/2.x/src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java (original)
+++ nutch/branches/2.x/src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java Thu May 15 08:10:07 2014
@@ -17,12 +17,6 @@
package org.apache.nutch.analysis.lang;
// JUnit imports
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
import org.apache.avro.util.Utf8;
import org.apache.nutch.metadata.Metadata;
@@ -32,6 +26,14 @@ import org.apache.nutch.util.Bytes;
import org.apache.nutch.util.EncodingDetector;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.tika.language.LanguageIdentifier;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
public class TestHTMLLanguageParser {
@@ -59,7 +61,7 @@ public class TestHTMLLanguageParser {
for (int t = 0; t < docs.length; t++) {
WebPage page = getPage(docs[t]);
parser.parse(URL.toString(), page);
- ByteBuffer blang = page.getFromMetadata(new Utf8(Metadata.LANGUAGE));
+ ByteBuffer blang = page.getMetadata().get(new Utf8(Metadata.LANGUAGE));
String lang = Bytes.toString(blang);
assertEquals(metalanguages[t], lang);
}
@@ -145,12 +147,12 @@ public class TestHTMLLanguageParser {
}
private WebPage getPage(String text) {
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setBaseUrl(BASE);
page.setContent(ByteBuffer.wrap(text.getBytes()));
page.setContentType(new Utf8("text/html"));
page
- .putToHeaders(EncodingDetector.CONTENT_TYPE_UTF8, new Utf8("text/html"));
+ .getHeaders().put(EncodingDetector.CONTENT_TYPE_UTF8, new Utf8("text/html"));
return page;
}
}
Modified: nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java (original)
+++ nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java Thu May 15 08:10:07 2014
@@ -142,7 +142,7 @@ public abstract class HttpBase implement
int elapsedTime =(int) (System.currentTimeMillis() - startTime);
if(this.responseTime) {
- page.putToMetadata(RESPONSE_TIME, ByteBuffer.wrap(Bytes.toBytes(elapsedTime)));
+ page.getMetadata().put(RESPONSE_TIME, ByteBuffer.wrap(Bytes.toBytes(elapsedTime)));
}
int code = response.getCode();
@@ -370,7 +370,7 @@ public abstract class HttpBase implement
url = args[i];
}
- ProtocolOutput out = http.getProtocolOutput(url, new WebPage());
+ ProtocolOutput out = http.getProtocolOutput(url, WebPage.newBuilder().build());
Content content = out.getContent();
System.out.println("Status: " + out.getStatus());
Modified: nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java (original)
+++ nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java Thu May 15 08:10:07 2014
@@ -17,19 +17,17 @@
package org.apache.nutch.protocol.http.api;
-import java.net.URL;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import crawlercommons.robots.BaseRobotRules;
+import crawlercommons.robots.SimpleRobotRules;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.RobotRulesParser;
import org.apache.nutch.storage.WebPage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import crawlercommons.robots.BaseRobotRules;
-import crawlercommons.robots.SimpleRobotRules;
+import java.net.URL;
/**
* This class is used for parsing robots for urls belonging to HTTP protocol.
@@ -88,7 +86,7 @@ public class HttpRobotRulesParser extend
if (LOG.isTraceEnabled()) { LOG.trace("cache miss " + url); }
try {
Response response = ((HttpBase)http).getResponse(new URL(url, "/robots.txt"),
- new WebPage(), true);
+ WebPage.newBuilder().build(), true);
// try one level of redirection ?
if (response.getCode() == 301 || response.getCode() == 302) {
String redirection = response.getHeader("Location");
@@ -104,7 +102,7 @@ public class HttpRobotRulesParser extend
redir = new URL(redirection);
}
- response = ((HttpBase)http).getResponse(redir, new WebPage(), true);
+ response = ((HttpBase)http).getResponse(redir, WebPage.newBuilder().build(), true);
}
}
Modified: nutch/branches/2.x/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagIndexingFilter.java Thu May 15 08:10:07 2014
@@ -85,7 +85,7 @@ public class RelTagIndexingFilter implem
@Override
public NutchDocument filter(NutchDocument doc, String url, WebPage page) throws IndexingException {
// Check if some Rel-Tags found, possibly put there by RelTagParser
- ByteBuffer bb = page.getFromMetadata(new Utf8(RelTagParser.REL_TAG));
+ ByteBuffer bb = page.getMetadata().get(new Utf8(RelTagParser.REL_TAG));
if (bb != null) {
String[] tags = Bytes.toString(bb).split("\t");
Modified: nutch/branches/2.x/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java (original)
+++ nutch/branches/2.x/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java Thu May 15 08:10:07 2014
@@ -171,7 +171,7 @@ public class RelTagParser implements Par
sb.append("\t");
}
ByteBuffer bb = ByteBuffer.wrap(sb.toString().getBytes());
- page.putToMetadata(new Utf8(REL_TAG), bb);
+ page.getMetadata().put(new Utf8(REL_TAG), bb);
return parse;
}
}
Modified: nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java Thu May 15 08:10:07 2014
@@ -16,14 +16,15 @@
*/
package org.apache.nutch.microformats.reltag;
-import java.nio.ByteBuffer;
-
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
import static org.junit.Assert.*;
/**
@@ -42,10 +43,10 @@ import static org.junit.Assert.*;
filter.setConf(conf);
assertNotNull(filter);
NutchDocument doc = new NutchDocument();
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
byte[] bytes = new byte[10];
ByteBuffer bbuf = ByteBuffer.wrap(bytes);
- page.putToMetadata(new Utf8(RelTagParser.REL_TAG), bbuf);
+ page.getMetadata().put(new Utf8(RelTagParser.REL_TAG), bbuf);
try {
filter.filter(doc, "http://nutch.apache.org/", page);
} catch (Exception e) {
Modified: nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java (original)
+++ nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java Thu May 15 08:10:07 2014
@@ -16,12 +16,6 @@
*/
package org.apache.nutch.microformats.reltag;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.parse.Parse;
@@ -35,6 +29,14 @@ import org.junit.Test;
import static org.junit.Assert.*;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+
/**
* Junit test for {@link RelTagParser} based mainly John Xing's parser tests.
* We are not concerned with actual parse text within the sample file, instead
@@ -76,7 +78,7 @@ public class TestRelTagParser {
in.readFully(bytes);
in.close();
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setBaseUrl(new Utf8(urlString));
page.setContent(ByteBuffer.wrap(bytes));
MimeUtil mimeutil = new MimeUtil(conf);
@@ -84,7 +86,7 @@ public class TestRelTagParser {
page.setContentType(new Utf8(mtype));
parse = new ParseUtil(conf).parse(urlString, page);
//begin assertion for tests
- ByteBuffer bbuf = page.getFromMetadata(new Utf8("Rel-Tag"));
+ ByteBuffer bbuf = page.getMetadata().get(new Utf8("Rel-Tag"));
byte[] byteArray = new byte[bbuf.remaining()];
bbuf.get(byteArray);
String s = new String(byteArray);
Modified: nutch/branches/2.x/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java Thu May 15 08:10:07 2014
@@ -192,8 +192,8 @@ public class HtmlParser implements Parse
detector.addClue(sniffCharacterEncoding(contentInOctets), "sniffed");
String encoding = detector.guessEncoding(page, defaultCharEncoding);
- page.putToMetadata(new Utf8(Metadata.ORIGINAL_CHAR_ENCODING), ByteBuffer.wrap(Bytes.toBytes(encoding)));
- page.putToMetadata(new Utf8(Metadata.CHAR_ENCODING_FOR_CONVERSION), ByteBuffer.wrap(Bytes.toBytes(encoding)));
+ page.getMetadata().put(new Utf8(Metadata.ORIGINAL_CHAR_ENCODING), ByteBuffer.wrap(Bytes.toBytes(encoding)));
+ page.getMetadata().put(new Utf8(Metadata.CHAR_ENCODING_FOR_CONVERSION), ByteBuffer.wrap(Bytes.toBytes(encoding)));
input.setEncoding(encoding);
if (LOG.isTraceEnabled()) { LOG.trace("Parsing..."); }
@@ -240,19 +240,19 @@ public class HtmlParser implements Parse
}
}
- ParseStatus status = new ParseStatus();
- status.setMajorCode(ParseStatusCodes.SUCCESS);
+ ParseStatus status = ParseStatus.newBuilder().build();
+ status.setMajorCode((int)ParseStatusCodes.SUCCESS);
if (metaTags.getRefresh()) {
- status.setMinorCode(ParseStatusCodes.SUCCESS_REDIRECT);
- status.addToArgs(new Utf8(metaTags.getRefreshHref().toString()));
- status.addToArgs(new Utf8(Integer.toString(metaTags.getRefreshTime())));
+ status.setMinorCode((int)ParseStatusCodes.SUCCESS_REDIRECT);
+ status.getArgs().add(new Utf8(metaTags.getRefreshHref().toString()));
+ status.getArgs().add(new Utf8(Integer.toString(metaTags.getRefreshTime())));
}
Parse parse = new Parse(text, title, outlinks, status);
parse = htmlParseFilters.filter(url, page, parse, metaTags, root);
if (metaTags.getNoCache()) { // not okay to cache
- page.putToMetadata(new Utf8(Nutch.CACHING_FORBIDDEN_KEY),
+ page.getMetadata().put(new Utf8(Nutch.CACHING_FORBIDDEN_KEY),
ByteBuffer.wrap(Bytes.toBytes(cachingPolicy)));
}
@@ -351,7 +351,7 @@ public class HtmlParser implements Parse
Configuration conf = NutchConfiguration.create();
HtmlParser parser = new HtmlParser();
parser.setConf(conf);
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setBaseUrl(new Utf8(url));
page.setContent(ByteBuffer.wrap(bytes));
page.setContentType(new Utf8("text/html"));
Modified: nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestHtmlParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestHtmlParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestHtmlParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestHtmlParser.java Thu May 15 08:10:07 2014
@@ -135,8 +135,7 @@ public class TestHtmlParser {
String text = parse.getText();
String title = parse.getTitle();
//String keywords = parse.getMeta("keywords");
- String keywords = Bytes.toString(page
- .getFromMetadata(new Utf8("keywords")));
+ String keywords = Bytes.toString(page.getMetadata().get(new Utf8("keywords")));
LOG.info(name);
LOG.info("title:\t" + title);
LOG.info("keywords:\t" + keywords);
Modified: nutch/branches/2.x/src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java (original)
+++ nutch/branches/2.x/src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java Thu May 15 08:10:07 2014
@@ -16,12 +16,6 @@
*/
package org.apache.nutch.parse.js;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.parse.Outlink;
@@ -34,7 +28,14 @@ import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.*;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
/**
* JUnit test case for {@link JSParseFilter} which tests
@@ -74,7 +75,7 @@ public class TestJSParseFilter {
dip.readFully(bytes);
dip.close();
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setBaseUrl(new Utf8(urlString));
page.setContent(ByteBuffer.wrap(bytes));
MimeUtil mutil = new MimeUtil(conf);
Modified: nutch/branches/2.x/src/plugin/parse-metatags/src/java/org/apache/nutch/parse/MetaTagsParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-metatags/src/java/org/apache/nutch/parse/MetaTagsParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-metatags/src/java/org/apache/nutch/parse/MetaTagsParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-metatags/src/java/org/apache/nutch/parse/MetaTagsParser.java Thu May 15 08:10:07 2014
@@ -77,10 +77,9 @@ public class MetaTagsParser implements P
// check in the metadata first : the tika-parser
// might have stored the values there already
- Iterator<Entry<Utf8, ByteBuffer>> iterator = page.getMetadata().entrySet()
- .iterator();
+ Iterator<Entry<CharSequence, ByteBuffer>> iterator = page.getMetadata().entrySet().iterator();
while (iterator.hasNext()) {
- Entry<Utf8, ByteBuffer> entry = iterator.next();
+ Entry<CharSequence, ByteBuffer> entry = iterator.next();
String mdName = entry.getKey().toString();
String value = Bytes.toStringBinary(entry.getValue());
if (metatagset.contains("*") || metatagset.contains(mdName.toLowerCase())) {
@@ -94,7 +93,7 @@ public class MetaTagsParser implements P
Iterator<Entry<Utf8, ByteBuffer>> itm = metadata.entrySet().iterator();
while (iterator.hasNext()) {
Entry<Utf8, ByteBuffer> entry = itm.next();
- page.putToMetadata(entry.getKey(), entry.getValue());
+ page.getMetadata().put(entry.getKey(), entry.getValue());
}
Properties generalMetaTags = metaTags.getGeneralTags();
@@ -118,7 +117,7 @@ public class MetaTagsParser implements P
if (metatagset.contains("*") || metatagset.contains(name.toLowerCase())) {
// Add the recently parsed value of multiValued array to metadata
LOG.debug("Found meta tag : " + name + "\t" + sb.toString());
- page.putToMetadata(new Utf8(PARSE_META_PREFIX + name.toLowerCase()),
+ page.getMetadata().put(new Utf8(PARSE_META_PREFIX + name.toLowerCase()),
ByteBuffer.wrap(Bytes.toBytes(sb.toString())));
}
}
@@ -132,7 +131,7 @@ public class MetaTagsParser implements P
// specified *
if (metatagset.contains("*") || metatagset.contains(name.toLowerCase())) {
LOG.debug("Found meta tag : " + name + "\t" + value);
- page.putToMetadata(new Utf8(PARSE_META_PREFIX + name.toLowerCase()),
+ page.getMetadata().put(new Utf8(PARSE_META_PREFIX + name.toLowerCase()),
ByteBuffer.wrap(value.getBytes()));
}
}
Modified: nutch/branches/2.x/src/plugin/parse-metatags/src/test/org/apache/nutch/parse/TestMetaTagsParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-metatags/src/test/org/apache/nutch/parse/TestMetaTagsParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-metatags/src/test/org/apache/nutch/parse/TestMetaTagsParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-metatags/src/test/org/apache/nutch/parse/TestMetaTagsParser.java Thu May 15 08:10:07 2014
@@ -58,13 +58,14 @@ public class TestMetaTagsParser {
/**
*
+ *
* @param fileName
* This variable set test file.
* @param useUtil
* If value is True method use ParseUtil
* @return If successfully document parsed, it return metatags
*/
- public Map<Utf8, ByteBuffer> parseMetaTags(String fileName, boolean useUtil) {
+ public Map<CharSequence, ByteBuffer> parseMetaTags(String fileName, boolean useUtil) {
try {
Configuration conf = NutchConfiguration.create();
String urlString = "file:" + sampleDir + fileSeparator + fileName;
@@ -185,7 +186,7 @@ public class TestMetaTagsParser {
@Test
public void testMetaTagsParserWithConf() {
// check that we get the same values
- Map<Utf8, ByteBuffer> meta = parseMetaTags(sampleFile, true);
+ Map<CharSequence, ByteBuffer> meta = parseMetaTags(sampleFile, true);
assertEquals(description,
getMeta(meta, MetaTagsParser.PARSE_META_PREFIX + "description"));
@@ -200,7 +201,7 @@ public class TestMetaTagsParser {
@Test
public void testFilter() {
// check that we get the same values
- Map<Utf8, ByteBuffer> meta = parseMetaTags(sampleFile, false);
+ Map<CharSequence, ByteBuffer> meta = parseMetaTags(sampleFile, false);
assertEquals(description,
getMeta(meta, MetaTagsParser.PARSE_META_PREFIX + "description"));
@@ -208,7 +209,7 @@ public class TestMetaTagsParser {
getMeta(meta, MetaTagsParser.PARSE_META_PREFIX + "keywords"));
}
- private String getMeta(Map<Utf8, ByteBuffer> meta, String name) {
+ private String getMeta(Map<CharSequence, ByteBuffer> meta, String name) {
ByteBuffer raw = meta.get(new Utf8(name));
return Bytes.toString(raw);
}
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java Thu May 15 08:10:07 2014
@@ -16,36 +16,15 @@
*/
package org.apache.nutch.parse.tika;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-
import org.apache.avro.util.Utf8;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.util.Bytes;
import org.apache.html.dom.HTMLDocumentImpl;
import org.apache.nutch.metadata.Nutch;
-import org.apache.nutch.parse.HTMLMetaTags;
-import org.apache.nutch.parse.ParseFilters;
-import org.apache.nutch.parse.Outlink;
-import org.apache.nutch.parse.OutlinkExtractor;
-import org.apache.nutch.parse.Parse;
-import org.apache.nutch.parse.ParseStatusCodes;
-import org.apache.nutch.parse.ParseStatusUtils;
-import org.apache.nutch.parse.ParseUtil;
+import org.apache.nutch.parse.*;
import org.apache.nutch.storage.ParseStatus;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
+import org.apache.nutch.util.Bytes;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.TableUtil;
@@ -53,8 +32,22 @@ import org.apache.tika.metadata.Metadata
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.w3c.dom.DocumentFragment;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+
/**
* Wrapper for Tika parsers. Mimics the HTMLParser but using the XHTML
* representation returned by Tika as SAX events
@@ -169,7 +162,7 @@ public class TikaParser implements org.a
if (tikaMDName.equalsIgnoreCase(TikaCoreProperties.TITLE.toString()))
continue;
// TODO what if multivalued?
- page.putToMetadata(new Utf8(tikaMDName), ByteBuffer.wrap(Bytes.toBytes(tikamd
+ page.getMetadata().put(new Utf8(tikaMDName), ByteBuffer.wrap(Bytes.toBytes(tikamd
.get(tikaMDName))));
}
@@ -182,16 +175,16 @@ public class TikaParser implements org.a
ParseStatus status = ParseStatusUtils.STATUS_SUCCESS;
if (metaTags.getRefresh()) {
- status.setMinorCode(ParseStatusCodes.SUCCESS_REDIRECT);
- status.addToArgs(new Utf8(metaTags.getRefreshHref().toString()));
- status.addToArgs(new Utf8(Integer.toString(metaTags.getRefreshTime())));
+ status.setMinorCode((int)ParseStatusCodes.SUCCESS_REDIRECT);
+ status.getArgs().add(new Utf8(metaTags.getRefreshHref().toString()));
+ status.getArgs().add(new Utf8(Integer.toString(metaTags.getRefreshTime())));
}
Parse parse = new Parse(text, title, outlinks, status);
parse = htmlParseFilters.filter(url, page, parse, metaTags, root);
if (metaTags.getNoCache()) { // not okay to cache
- page.putToMetadata(new Utf8(Nutch.CACHING_FORBIDDEN_KEY), ByteBuffer.wrap(Bytes
+ page.getMetadata().put(new Utf8(Nutch.CACHING_FORBIDDEN_KEY), ByteBuffer.wrap(Bytes
.toBytes(cachingPolicy)));
}
@@ -241,7 +234,7 @@ public class TikaParser implements org.a
Configuration conf = NutchConfiguration.create();
// TikaParser parser = new TikaParser();
// parser.setConf(conf);
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setBaseUrl(new Utf8(url));
page.setContent(ByteBuffer.wrap(bytes));
MimeUtil mimeutil = new MimeUtil(conf);
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestImageMetadata.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestImageMetadata.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestImageMetadata.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestImageMetadata.java Thu May 15 08:10:07 2014
@@ -69,14 +69,14 @@ public class TestImageMetadata {
parse = new ParseUtil(conf).parse(urlString, page);
//assert width
- ByteBuffer bbufW = page.getFromMetadata(new Utf8("width"));
+ ByteBuffer bbufW = page.getMetadata().get(new Utf8("width"));
byte[] byteArrayW = new byte[bbufW.remaining()];
bbufW.get(byteArrayW);
String width = new String(byteArrayW);
assertEquals("121", width);
//assert height
- ByteBuffer bbufH = page.getFromMetadata(new Utf8("height"));
+ ByteBuffer bbufH = page.getMetadata().get(new Utf8("height"));
byte[] byteArrayH = new byte[bbufH.remaining()];
bbufH.get(byteArrayH);
String height = new String(byteArrayH);
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java Thu May 15 08:10:07 2014
@@ -17,16 +17,6 @@
package org.apache.nutch.parse.tika;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.*;
-
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.parse.Parse;
@@ -36,6 +26,16 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertTrue;
/**
* Unit tests for MSWordParser.
@@ -72,7 +72,7 @@ public class TestMSWordParser {
in.readFully(bytes);
in.close();
Parse parse;
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setBaseUrl(new Utf8("file:"+urlString));
page.setContent(ByteBuffer.wrap(bytes));
// set the content type?
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java Thu May 15 08:10:07 2014
@@ -17,16 +17,6 @@
package org.apache.nutch.parse.tika;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
-
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.parse.Parse;
@@ -36,6 +26,12 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
+import org.junit.Test;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertTrue;
/**
* Unit tests for OOParser.
@@ -95,7 +91,7 @@ public class TestOOParser {
in.readFully(bytes);
in.close();
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setBaseUrl(new Utf8(urlString));
page.setContent(ByteBuffer.wrap(bytes));
String mtype = mimeutil.getMimeType(file);
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java Thu May 15 08:10:07 2014
@@ -17,15 +17,6 @@
package org.apache.nutch.parse.tika;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
-
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.parse.Parse;
@@ -35,6 +26,15 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
+import org.junit.Test;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertTrue;
/**
* Unit tests for PdfParser.
@@ -69,7 +69,7 @@ public class TestPdfParser {
in.readFully(bytes);
in.close();
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setBaseUrl(new Utf8(urlString));
page.setContent(ByteBuffer.wrap(bytes));
String mtype = mimeutil.getMimeType(file);
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java Thu May 15 08:10:07 2014
@@ -17,26 +17,26 @@
package org.apache.nutch.parse.tika;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.nutch.protocol.ProtocolException;
-
-import org.apache.nutch.parse.Parse;
-import org.apache.nutch.parse.ParseUtil;
-import org.apache.nutch.parse.ParseException;
-import org.apache.nutch.parse.Outlink;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.parse.Outlink;
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.parse.ParseException;
+import org.apache.nutch.parse.ParseUtil;
+import org.apache.nutch.protocol.ProtocolException;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
-
import org.junit.Test;
-import static org.junit.Assert.*;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* Unit tests for the RSS Parser based on John Xing's TestPdfParser class.
@@ -83,7 +83,7 @@ public class TestRSSParser {
in.readFully(bytes);
in.close();
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setBaseUrl(new Utf8(urlString));
page.setContent(ByteBuffer.wrap(bytes));
String mtype = mimeutil.getMimeType(file);
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java Thu May 15 08:10:07 2014
@@ -17,14 +17,6 @@
package org.apache.nutch.parse.tika;
// JUnit imports
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
@@ -35,6 +27,15 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
+import org.junit.Test;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
/**
* Unit tests for TestRTFParser. (Adapted from John Xing msword unit tests).
@@ -67,7 +68,7 @@ public class TestRTFParser {
in.readFully(bytes);
in.close();
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setBaseUrl(new Utf8(urlString));
page.setContent(ByteBuffer.wrap(bytes));
String mtype = mimeutil.getMimeType(file);
Modified: nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java Thu May 15 08:10:07 2014
@@ -16,29 +16,20 @@
*/
package org.apache.nutch.protocol.file;
-import java.net.URL;
-import java.util.Collection;
-import java.util.HashSet;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import crawlercommons.robots.BaseRobotRules;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-
import org.apache.nutch.net.protocols.Response;
-import org.apache.nutch.protocol.Content;
-import org.apache.nutch.protocol.Protocol;
-import org.apache.nutch.protocol.ProtocolOutput;
-import org.apache.nutch.protocol.ProtocolStatusCodes;
-import org.apache.nutch.protocol.ProtocolStatusUtils;
-import org.apache.nutch.protocol.RobotRulesParser;
+import org.apache.nutch.protocol.*;
import org.apache.nutch.storage.ProtocolStatus;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.NutchConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import crawlercommons.robots.BaseRobotRules;
+import java.net.URL;
+import java.util.Collection;
+import java.util.HashSet;
/**
* This class is a protocol plugin used for file: scheme.
@@ -183,7 +174,7 @@ public class File implements Protocol {
if (maxContentLength != Integer.MIN_VALUE) // set maxContentLength
file.setMaxContentLength(maxContentLength);
- Content content = file.getProtocolOutput(urlString, new WebPage())
+ Content content = file.getProtocolOutput(urlString, WebPage.newBuilder().build())
.getContent();
System.out.println("Content-Type: " + content.getContentType());
Modified: nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java Thu May 15 08:10:07 2014
@@ -76,14 +76,14 @@ public class TestProtocolFile {
public void setContentType(String testTextFile) throws ProtocolNotFound {
String urlString = "file:" + sampleDir + fileSeparator + testTextFile;
assertNotNull(urlString);
- WebPage datum = new WebPage();
+ WebPage datum = WebPage.newBuilder().build();
Protocol protocol = new ProtocolFactory(conf).getProtocol(urlString);
ProtocolOutput output = protocol.getProtocolOutput(urlString,datum);
assertNotNull(output);
assertEquals("Status code: [" + output.getStatus().getCode()
+ "], not equal to: [" + ProtocolStatusCodes.SUCCESS + "]: args: ["
- + output.getStatus().getArgs() + "]", ProtocolStatusCodes.SUCCESS, output
+ + output.getStatus().getArgs() + "]", (Integer) ProtocolStatusCodes.SUCCESS, output
.getStatus().getCode());
assertNotNull(output.getContent());
assertNotNull(output.getContent().getContentType());
Modified: nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java Thu May 15 08:10:07 2014
@@ -17,25 +17,20 @@
package org.apache.nutch.protocol.ftp;
-import java.io.IOException;
-import java.net.URL;
-import java.util.Collection;
-import java.util.HashSet;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import crawlercommons.robots.BaseRobotRules;
import org.apache.commons.net.ftp.FTPFileEntryParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.net.protocols.Response;
-import org.apache.nutch.protocol.Content;
-import org.apache.nutch.protocol.Protocol;
-import org.apache.nutch.protocol.ProtocolOutput;
-import org.apache.nutch.protocol.ProtocolStatusCodes;
-import org.apache.nutch.protocol.ProtocolStatusUtils;
+import org.apache.nutch.protocol.*;
import org.apache.nutch.storage.ProtocolStatus;
import org.apache.nutch.storage.WebPage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import crawlercommons.robots.BaseRobotRules;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Collection;
+import java.util.HashSet;
/**
* This class is a protocol plugin used for ftp: scheme.
@@ -238,7 +233,7 @@ public class Ftp implements Protocol {
if (maxContentLength != Integer.MIN_VALUE) // set maxContentLength
ftp.setMaxContentLength(maxContentLength);
- Content content = ftp.getProtocolOutput(urlString, new WebPage())
+ Content content = ftp.getProtocolOutput(urlString, WebPage.newBuilder().build())
.getContent();
System.err.println("Content-Type: " + content.getContentType());
Modified: nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java Thu May 15 08:10:07 2014
@@ -18,14 +18,6 @@
package org.apache.nutch.protocol.ftp;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.URL;
-import java.nio.ByteBuffer;
-import java.util.LinkedList;
-import java.util.List;
-
import org.apache.avro.util.Utf8;
import org.apache.commons.net.ftp.FTP;
import org.apache.commons.net.ftp.FTPFile;
@@ -39,6 +31,14 @@ import org.apache.nutch.net.protocols.Re
import org.apache.nutch.protocol.Content;
import org.apache.nutch.storage.WebPage;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.LinkedList;
+import java.util.List;
+
/************************************
* FtpResponse.java mimics ftp replies as http response.
@@ -116,7 +116,7 @@ public class FtpResponse {
if (addr != null
&& conf.getBoolean("store.ip.address", false) == true) {
String ipString = addr.getHostAddress(); //get the ip address
- page.putToMetadata(new Utf8("_ip_"),
+ page.getMetadata().put(new Utf8("_ip_"),
ByteBuffer.wrap(ipString.getBytes()));
}
Modified: nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java Thu May 15 08:10:07 2014
@@ -17,9 +17,8 @@
package org.apache.nutch.protocol.ftp;
-import java.net.URL;
-
-import org.apache.commons.io.IOUtils;
+import crawlercommons.robots.BaseRobotRules;
+import crawlercommons.robots.SimpleRobotRules;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolOutput;
@@ -29,8 +28,7 @@ import org.apache.nutch.storage.WebPage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import crawlercommons.robots.BaseRobotRules;
-import crawlercommons.robots.SimpleRobotRules;
+import java.net.URL;
/**
* This class is used for parsing robots for urls belonging to FTP protocol.
@@ -75,7 +73,7 @@ public class FtpRobotRulesParser extends
try {
String robotsUrl = new URL(url, "/robots.txt").toString();
- ProtocolOutput output = ((Ftp)ftp).getProtocolOutput(robotsUrl, new WebPage());
+ ProtocolOutput output = ((Ftp)ftp).getProtocolOutput(robotsUrl, WebPage.newBuilder().build());
int statusCode = output.getStatus().getCode();
if (statusCode == ProtocolStatusCodes.SUCCESS) {
Modified: nutch/branches/2.x/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java Thu May 15 08:10:07 2014
@@ -17,17 +17,6 @@
package org.apache.nutch.protocol.http;
// JDK imports
-import java.io.BufferedInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PushbackInputStream;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.URL;
-import java.nio.ByteBuffer;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
@@ -40,6 +29,12 @@ import org.apache.nutch.protocol.http.ap
import org.apache.nutch.protocol.http.api.HttpException;
import org.apache.nutch.storage.WebPage;
+import java.io.*;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.URL;
+import java.nio.ByteBuffer;
+
/** An HTTP response. */
public class HttpResponse implements Response {
@@ -97,7 +92,7 @@ public class HttpResponse implements Res
if (sockAddr != null
&& conf.getBoolean("store.ip.address", false) == true) {
String ipString = sockAddr.getAddress().getHostAddress(); //get the ip address
- page.putToMetadata(new Utf8("_ip_"),
+ page.getMetadata().put(new Utf8("_ip_"),
ByteBuffer.wrap(ipString.getBytes()));
}
@@ -133,11 +128,11 @@ public class HttpResponse implements Res
reqStr.append("\r\n");
}
- if (page.isReadable(WebPage.Field.MODIFIED_TIME.getIndex())) {
+// if (page.isReadable(WebPage.Field.MODIFIED_TIME.getIndex())) {
reqStr.append("If-Modified-Since: " +
HttpDateFormat.toString(page.getModifiedTime()));
reqStr.append("\r\n");
- }
+// }
reqStr.append("\r\n");
byte[] reqBytes= reqStr.toString().getBytes();
@@ -177,7 +172,7 @@ public class HttpResponse implements Res
page.getHeaders().clear();
}
for (String key : headers.names()) {
- page.putToHeaders(new Utf8(key), new Utf8(headers.get(key)));
+ page.getHeaders().put(new Utf8(key), new Utf8(headers.get(key)));
}
} finally {
Modified: nutch/branches/2.x/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java Thu May 15 08:10:07 2014
@@ -176,7 +176,7 @@ public class HttpResponse implements Res
page.getHeaders().clear();
}
for (String key : headers.names()) {
- page.putToHeaders(new Utf8(key), new Utf8(headers.get(key)));
+ page.getHeaders().put(new Utf8(key), new Utf8(headers.get(key)));
}
// Logger trace message
Modified: nutch/branches/2.x/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java?rev=1594812&r1=1594811&r2=1594812&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java Thu May 15 08:10:07 2014
@@ -17,22 +17,22 @@
package org.apache.nutch.protocol.httpclient;
-import java.net.MalformedURLException;
-import java.net.URL;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.*;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.storage.WebPage;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.nio.SelectChannelConnector;
import org.mortbay.jetty.servlet.Context;
import org.mortbay.jetty.servlet.ServletHolder;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+import static org.junit.Assert.assertEquals;
+
/**
* Test cases for protocol-httpclient.
*
@@ -210,7 +210,7 @@ public class TestProtocolHttpClient {
private void fetchPage(String page, int expectedCode) throws Exception {
URL url = new URL("http", "127.0.0.1", port, page);
Response response = null;
- response = http.getResponse(url, new WebPage(), true);
+ response = http.getResponse(url, WebPage.newBuilder().build(), true);
int code = response.getCode();
assertEquals("HTTP Status Code for " + url, expectedCode, code);