You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by jn...@apache.org on 2010/06/30 12:36:29 UTC
svn commit: r959259 [11/12] - in /nutch/branches/nutchbase: ./ bin/ conf/
contrib/ docs/ ivy/ lib/ lib/jetty-ext/ src/engines/ src/gora/ src/java/
src/java/org/apache/nutch/analysis/ src/java/org/apache/nutch/clustering/
src/java/org/apache/nutch/crawl...
Added: nutch/branches/nutchbase/src/plugin/parse-tika/src/test/org/apache/nutch/tika/TestRTFParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/parse-tika/src/test/org/apache/nutch/tika/TestRTFParser.java?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/parse-tika/src/test/org/apache/nutch/tika/TestRTFParser.java (added)
+++ nutch/branches/nutchbase/src/plugin/parse-tika/src/test/org/apache/nutch/tika/TestRTFParser.java Wed Jun 30 10:36:20 2010
@@ -0,0 +1,100 @@
+package org.apache.nutch.tika;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// JUnit imports
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import junit.framework.TestCase;
+
+import org.apache.avro.util.Utf8;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.parse.ParseException;
+import org.apache.nutch.parse.ParseUtil;
+import org.apache.nutch.protocol.ProtocolException;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.util.MimeUtil;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.tika.mime.MimeType;
+
+/**
+ * Unit tests for TestRTFParser. (Adapted from John Xing msword unit tests).
+ *
+ * @author Andy Hedges
+ */
+public class TestRTFParser extends TestCase {
+
+ private String fileSeparator = System.getProperty("file.separator");
+ // This system property is defined in ./src/plugin/build-plugin.xml
+ private String sampleDir = System.getProperty("test.data", ".");
+ // Make sure sample files are copied to "test.data" as specified in
+ // ./src/plugin/parse-rtf/build.xml during plugin compilation.
+ // Check ./src/plugin/parse-rtf/sample/README.txt for what they are.
+ private String rtfFile = "test.rtf";
+
+ public TestRTFParser(String name) {
+ super(name);
+ }
+
+ protected void setUp() {
+ }
+
+ protected void tearDown() {
+ }
+
+ public void testIt() throws ProtocolException, ParseException, IOException {
+
+ String urlString;
+ Parse parse;
+ Configuration conf = NutchConfiguration.create();
+ MimeUtil mimeutil = new MimeUtil(conf);
+
+ urlString = "file:" + sampleDir + fileSeparator + rtfFile;
+
+ File file = new File(sampleDir + fileSeparator + rtfFile);
+ byte[] bytes = new byte[(int) file.length()];
+ DataInputStream in = new DataInputStream(new FileInputStream(file));
+ in.readFully(bytes);
+ in.close();
+
+ WebPage page = new WebPage();
+ page.setBaseUrl(new Utf8(urlString));
+ page.setContent(ByteBuffer.wrap(bytes));
+ MimeType mtype = mimeutil.getMimeType(file);
+ page.setContentType(new Utf8(mtype.getName()));
+
+ parse = new ParseUtil(conf).parse(urlString, page);
+
+ String text = parse.getText();
+ assertEquals("The quick brown fox jumps over the lazy dog", text.trim());
+
+ String title = parse.getTitle();
+ // HOW DO WE GET THE PARSE METADATA?
+ // Metadata meta = parse();
+
+ // METADATA extraction is not yet supported in Tika
+ // assertEquals("test rft document", title);
+ // assertEquals("tests", meta.get(DublinCore.SUBJECT));
+ }
+
+}
Added: nutch/branches/nutchbase/src/plugin/parse-zip/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/parse-zip/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/parse-zip/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/parse-zip/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Modified: nutch/branches/nutchbase/src/plugin/protocol-file/build.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-file/build.xml?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-file/build.xml (original)
+++ nutch/branches/nutchbase/src/plugin/protocol-file/build.xml Wed Jun 30 10:36:20 2010
@@ -17,6 +17,14 @@
-->
<project name="protocol-file" default="jar-core">
- <import file="../build-plugin.xml"/>
+ <import file="../build-plugin.xml" />
+
+ <!-- for junit test -->
+ <mkdir dir="${build.test}/data" />
+ <copy todir="${build.test}/data">
+ <fileset dir="sample">
+ <include name="*.*" />
+ </fileset>
+ </copy>
</project>
Added: nutch/branches/nutchbase/src/plugin/protocol-file/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-file/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-file/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/protocol-file/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/plugin/protocol-file/sample/testprotocolfile.txt
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-file/sample/testprotocolfile.txt?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-file/sample/testprotocolfile.txt (added)
+++ nutch/branches/nutchbase/src/plugin/protocol-file/sample/testprotocolfile.txt Wed Jun 30 10:36:20 2010
@@ -0,0 +1 @@
+Protocol File Test
Modified: nutch/branches/nutchbase/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java (original)
+++ nutch/branches/nutchbase/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java Wed Jun 30 10:36:20 2010
@@ -17,39 +17,45 @@
package org.apache.nutch.protocol.file;
+import java.net.URL;
+import java.util.Collection;
+import java.util.HashSet;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
-import org.apache.nutch.crawl.CrawlDatum;
-import org.apache.hadoop.io.Text;
-import org.apache.nutch.metadata.Metadata;
-import org.apache.nutch.net.protocols.HttpDateFormat;
-import org.apache.nutch.net.protocols.Response;
-
import org.apache.hadoop.conf.Configuration;
-
+import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.protocol.EmptyRobotRules;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolOutput;
-import org.apache.nutch.protocol.ProtocolStatus;
+import org.apache.nutch.protocol.ProtocolStatusCodes;
import org.apache.nutch.protocol.RobotRules;
+import org.apache.nutch.storage.ProtocolStatus;
+import org.apache.nutch.storage.ProtocolStatusUtils;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.NutchConfiguration;
-import java.net.URL;
-
/************************************
* File.java deals with file: scheme.
- *
- * Configurable parameters are defined under "FILE properties" section
- * in ./conf/nutch-default.xml or similar.
- *
+ *
+ * Configurable parameters are defined under "FILE properties" section in
+ * ./conf/nutch-default.xml or similar.
+ *
* @author John Xing
***********************************/
public class File implements Protocol {
public static final Log LOG = LogFactory.getLog(File.class);
+ private static final Collection<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();
+
+ static {
+ FIELDS.add(WebPage.Field.MODIFIED_TIME);
+ FIELDS.add(WebPage.Field.HEADERS);
+ }
+
static final int MAX_REDIRECTS = 5;
int maxContentLength;
@@ -61,69 +67,79 @@ public class File implements Protocol {
}
/** Set the point at which content is truncated. */
- public void setMaxContentLength(int length) {maxContentLength = length;}
+ public void setMaxContentLength(int length) {
+ maxContentLength = length;
+ }
- public ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum) {
+ public ProtocolOutput getProtocolOutput(String url, WebPage page) {
String urlString = url.toString();
try {
URL u = new URL(urlString);
-
+
int redirects = 0;
-
+
while (true) {
FileResponse response;
- response = new FileResponse(u, datum, this, getConf()); // make a request
-
+ response = new FileResponse(u, page, this, getConf()); // make
+ // a
+ // request
+
int code = response.getCode();
-
- if (code == 200) { // got a good response
- return new ProtocolOutput(response.toContent()); // return it
-
- } else if (code >= 300 && code < 400) { // handle redirect
+
+ if (code == 200) { // got a good response
+ return new ProtocolOutput(response.toContent()); // return
+ // it
+
+ } else if (code >= 300 && code < 400) { // handle redirect
if (redirects == MAX_REDIRECTS)
throw new FileException("Too many redirects: " + url);
u = new URL(response.getHeader("Location"));
- redirects++;
+ redirects++;
if (LOG.isTraceEnabled()) {
- LOG.trace("redirect to " + u);
+ LOG.trace("redirect to " + u);
}
-
- } else { // convert to exception
+
+ } else { // convert to exception
throw new FileError(code);
}
- }
+ }
} catch (Exception e) {
e.printStackTrace();
- return new ProtocolOutput(null, new ProtocolStatus(e));
+ ProtocolStatus ps = ProtocolStatusUtils.makeStatus(
+ ProtocolStatusCodes.EXCEPTION, e.toString());
+ return new ProtocolOutput(null, ps);
}
}
-// protected void finalize () {
-// // nothing here
-// }
+ @Override
+ public RobotRules getRobotRules(String url, WebPage page) {
+ return EmptyRobotRules.RULES;
+ }
+
+ @Override
+ public Collection<Field> getFields() {
+ return FIELDS;
+ }
/** For debugging. */
public static void main(String[] args) throws Exception {
int maxContentLength = Integer.MIN_VALUE;
- String logLevel = "info";
boolean dumpContent = false;
String urlString = null;
- String usage = "Usage: File [-logLevel level] [-maxContentLength L] [-dumpContent] url";
+ String usage = "Usage: File [-maxContentLength L] [-dumpContent] url";
if (args.length == 0) {
System.err.println(usage);
System.exit(-1);
}
-
+
for (int i = 0; i < args.length; i++) {
- if (args[i].equals("-logLevel")) {
- logLevel = args[++i];
- } else if (args[i].equals("-maxContentLength")) {
+ if (args[i].equals("-maxContentLength")) {
maxContentLength = Integer.parseInt(args[++i]);
} else if (args[i].equals("-dumpContent")) {
dumpContent = true;
- } else if (i != args.length-1) {
+ } else if (i != args.length - 1) {
System.err.println(usage);
System.exit(-1);
} else
@@ -137,15 +153,16 @@ public class File implements Protocol {
file.setMaxContentLength(maxContentLength);
// set log level
- //LOG.setLevel(Level.parse((new String(logLevel)).toUpperCase()));
+ // LOG.setLevel(Level.parse((new String(logLevel)).toUpperCase()));
- Content content = file.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();
+ Content content = file.getProtocolOutput(urlString, new WebPage())
+ .getContent();
- System.err.println("Content-Type: " + content.getContentType());
- System.err.println("Content-Length: " +
- content.getMetadata().get(Response.CONTENT_LENGTH));
- System.err.println("Last-Modified: " +
- content.getMetadata().get(Response.LAST_MODIFIED));
+ System.out.println("Content-Type: " + content.getContentType());
+ System.out.println("Content-Length: "
+ + content.getMetadata().get(Response.CONTENT_LENGTH));
+ System.out.println("Last-Modified: "
+ + content.getMetadata().get(Response.LAST_MODIFIED));
if (dumpContent) {
System.out.print(new String(content.getContent()));
}
@@ -161,8 +178,4 @@ public class File implements Protocol {
public Configuration getConf() {
return this.conf;
}
-
- public RobotRules getRobotRules(Text url, CrawlDatum datum) {
- return EmptyRobotRules.RULES;
- }
}
Modified: nutch/branches/nutchbase/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java (original)
+++ nutch/branches/nutchbase/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java Wed Jun 30 10:36:20 2010
@@ -18,25 +18,18 @@
package org.apache.nutch.protocol.file;
// JDK imports
-import java.net.URL;
-import java.util.Date;
-import java.util.TreeMap;
import java.io.IOException;
+import java.net.URL;
-// Nutch imports
-import org.apache.nutch.crawl.CrawlDatum;
-import org.apache.nutch.protocol.Content;
-import org.apache.nutch.util.MimeUtil;
+import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.net.protocols.HttpDateFormat;
import org.apache.nutch.net.protocols.Response;
-
-// Tika imports
+import org.apache.nutch.protocol.Content;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.util.MimeUtil;
import org.apache.tika.mime.MimeType;
-// Hadoop imports
-import org.apache.hadoop.conf.Configuration;
-
/************************************
* FileResponse.java mimics file replies as http response.
@@ -94,7 +87,7 @@ public class FileResponse {
headers, this.conf);
}
- public FileResponse(URL url, CrawlDatum datum, File file, Configuration conf)
+ public FileResponse(URL url, WebPage page, File file, Configuration conf)
throws FileException, IOException {
this.orig = url.toString();
@@ -148,7 +141,7 @@ public class FileResponse {
this.code = 300; // http redirect
return;
}
- if (f.lastModified() <= datum.getModifiedTime()) {
+ if (f.lastModified() <= page.getModifiedTime()) {
this.code = 304;
this.headers.set("Last-Modified", HttpDateFormat.toString(f.lastModified()));
return;
Modified: nutch/branches/nutchbase/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java (original)
+++ nutch/branches/nutchbase/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java Wed Jun 30 10:36:20 2010
@@ -18,18 +18,14 @@
package org.apache.nutch.protocol.file;
// Hadoop imports
-import org.apache.hadoop.io.Text;
+import junit.framework.TestCase;
-// Nutch imports
-import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.ProtocolOutput;
-import org.apache.nutch.protocol.ProtocolStatus;
+import org.apache.nutch.protocol.ProtocolStatusCodes;
+import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
-// Junit imports
-import junit.framework.TestCase;
-
/**
* @author mattmann
* @version $Revision$
@@ -45,14 +41,16 @@ public class TestProtocolFile extends Te
private static final String testTextFile = "testprotocolfile.txt";
- private static final CrawlDatum datum = new CrawlDatum();
-
private static final String expectedMimeType = "text/plain";
static {
fileProtocol.setConf(NutchConfiguration.create());
}
+ private String fileSeparator = System.getProperty("file.separator");
+ // This system property is defined in ./src/plugin/build-plugin.xml
+ private String sampleDir = System.getProperty("test.data", ".");
+
/**
* Tests the setting of the <code>Response.CONTENT_TYPE</code> metadata
* field.
@@ -61,14 +59,14 @@ public class TestProtocolFile extends Te
*
*/
public void testSetContentType() {
- Text fileUrl = new Text(this.getClass().getResource(testTextFile)
- .toString());
- assertNotNull(fileUrl);
- ProtocolOutput output = fileProtocol.getProtocolOutput(fileUrl, datum);
+ String fileURL = "file:"+sampleDir + fileSeparator + testTextFile;
+ assertNotNull(fileURL);
+ WebPage datum = new WebPage();
+ ProtocolOutput output = fileProtocol.getProtocolOutput(fileURL, datum);
assertNotNull(output);
assertEquals("Status code: [" + output.getStatus().getCode()
- + "], not equal to: [" + ProtocolStatus.SUCCESS + "]: args: ["
- + output.getStatus().getArgs() + "]", ProtocolStatus.SUCCESS, output
+ + "], not equal to: [" + ProtocolStatusCodes.SUCCESS + "]: args: ["
+ + output.getStatus().getArgs() + "]", ProtocolStatusCodes.SUCCESS, output
.getStatus().getCode());
assertNotNull(output.getContent());
assertNotNull(output.getContent().getContentType());
Added: nutch/branches/nutchbase/src/plugin/protocol-ftp/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-ftp/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-ftp/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/protocol-ftp/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,42 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ <dependency org="commons-net" name="commons-net" rev="1.2.2" conf="*->master"/>
+ </dependencies>
+
+</ivy-module>
Modified: nutch/branches/nutchbase/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java (original)
+++ nutch/branches/nutchbase/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java Wed Jun 30 10:36:20 2010
@@ -17,41 +17,48 @@
package org.apache.nutch.protocol.ftp;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Collection;
+import java.util.HashSet;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.commons.net.ftp.FTPFileEntryParser;
-
-import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
-import org.apache.nutch.net.protocols.HttpDateFormat;
+import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.net.protocols.Response;
-
-import org.apache.hadoop.conf.Configuration;
-
import org.apache.nutch.protocol.Content;
import org.apache.nutch.protocol.EmptyRobotRules;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolOutput;
-import org.apache.nutch.protocol.ProtocolStatus;
+import org.apache.nutch.protocol.ProtocolStatusCodes;
import org.apache.nutch.protocol.RobotRules;
-
-import java.net.URL;
-
-import java.io.IOException;
+import org.apache.nutch.storage.ProtocolStatus;
+import org.apache.nutch.storage.ProtocolStatusUtils;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.storage.WebPage.Field;
/************************************
* Ftp.java deals with ftp: scheme.
- *
- * Configurable parameters are defined under "FTP properties" section
- * in ./conf/nutch-default.xml or similar.
- *
+ *
+ * Configurable parameters are defined under "FTP properties" section in
+ * ./conf/nutch-default.xml or similar.
+ *
* @author John Xing
***********************************/
public class Ftp implements Protocol {
public static final Log LOG = LogFactory.getLog(Ftp.class);
+ private static final Collection<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();
+
+ static {
+ FIELDS.add(WebPage.Field.MODIFIED_TIME);
+ FIELDS.add(WebPage.Field.HEADERS);
+ }
+
static final int BUFFER_SIZE = 16384; // 16*1024 = 16384
static final int MAX_REDIRECTS = 5;
@@ -61,7 +68,7 @@ public class Ftp implements Protocol {
int maxContentLength;
String userName;
- String passWord;
+ String passWord;
// typical/default server timeout is 120*1000 millisec.
// better be conservative here
@@ -81,7 +88,6 @@ public class Ftp implements Protocol {
private Configuration conf;
-
// constructor
public Ftp() {
}
@@ -106,40 +112,41 @@ public class Ftp implements Protocol {
this.keepConnection = keepConnection;
}
- public ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum) {
- String urlString = url.toString();
+ public ProtocolOutput getProtocolOutput(String url, WebPage page) {
try {
- URL u = new URL(urlString);
-
+ URL u = new URL(url);
+
int redirects = 0;
-
+
while (true) {
FtpResponse response;
- response = new FtpResponse(u, datum, this, getConf()); // make a request
-
+ response = new FtpResponse(u, page, this, getConf()); // make a request
+
int code = response.getCode();
-
- if (code == 200) { // got a good response
- return new ProtocolOutput(response.toContent()); // return it
-
- } else if (code >= 300 && code < 400) { // handle redirect
+
+ if (code == 200) { // got a good response
+ return new ProtocolOutput(response.toContent()); // return it
+
+ } else if (code >= 300 && code < 400) { // handle redirect
if (redirects == MAX_REDIRECTS)
throw new FtpException("Too many redirects: " + url);
u = new URL(response.getHeader("Location"));
- redirects++;
+ redirects++;
if (LOG.isTraceEnabled()) {
- LOG.trace("redirect to " + u);
+ LOG.trace("redirect to " + u);
}
- } else { // convert to exception
+ } else { // convert to exception
throw new FtpError(code);
}
- }
+ }
} catch (Exception e) {
- return new ProtocolOutput(null, new ProtocolStatus(e));
+ ProtocolStatus ps = ProtocolStatusUtils.makeStatus(
+ ProtocolStatusCodes.EXCEPTION, e.toString());
+ return new ProtocolOutput(null, ps);
}
}
- protected void finalize () {
+ protected void finalize() {
try {
if (this.client != null && this.client.isConnected()) {
this.client.logout();
@@ -150,6 +157,25 @@ public class Ftp implements Protocol {
}
}
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ this.maxContentLength = conf.getInt("ftp.content.limit", 64 * 1024);
+ this.timeout = conf.getInt("ftp.timeout", 10000);
+ this.userName = conf.get("ftp.username", "anonymous");
+ this.passWord = conf.get("ftp.password", "anonymous@example.com");
+ this.serverTimeout = conf.getInt("ftp.server.timeout", 60 * 1000);
+ this.keepConnection = conf.getBoolean("ftp.keep.connection", false);
+ this.followTalk = conf.getBoolean("ftp.follow.talk", false);
+ }
+
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ public RobotRules getRobotRules(String url, WebPage page) {
+ return EmptyRobotRules.RULES;
+ }
+
/** For debugging. */
public static void main(String[] args) throws Exception {
int timeout = Integer.MIN_VALUE;
@@ -166,7 +192,7 @@ public class Ftp implements Protocol {
System.err.println(usage);
System.exit(-1);
}
-
+
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-logLevel")) {
logLevel = args[++i];
@@ -180,7 +206,7 @@ public class Ftp implements Protocol {
maxContentLength = Integer.parseInt(args[++i]);
} else if (args[i].equals("-dumpContent")) {
dumpContent = true;
- } else if (i != args.length-1) {
+ } else if (i != args.length - 1) {
System.err.println(usage);
System.exit(-1);
} else {
@@ -200,15 +226,16 @@ public class Ftp implements Protocol {
ftp.setMaxContentLength(maxContentLength);
// set log level
- //LOG.setLevel(Level.parse((new String(logLevel)).toUpperCase()));
+ // LOG.setLevel(Level.parse((new String(logLevel)).toUpperCase()));
- Content content = ftp.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();
+ Content content = ftp.getProtocolOutput(urlString, new WebPage())
+ .getContent();
System.err.println("Content-Type: " + content.getContentType());
- System.err.println("Content-Length: " +
- content.getMetadata().get(Response.CONTENT_LENGTH));
- System.err.println("Last-Modified: " +
- content.getMetadata().get(Response.LAST_MODIFIED));
+ System.err.println("Content-Length: "
+ + content.getMetadata().get(Response.CONTENT_LENGTH));
+ System.err.println("Last-Modified: "
+ + content.getMetadata().get(Response.LAST_MODIFIED));
if (dumpContent) {
System.out.print(new String(content.getContent()));
}
@@ -216,24 +243,8 @@ public class Ftp implements Protocol {
ftp = null;
}
-
- public void setConf(Configuration conf) {
- this.conf = conf;
- this.maxContentLength = conf.getInt("ftp.content.limit", 64 * 1024);
- this.timeout = conf.getInt("ftp.timeout", 10000);
- this.userName = conf.get("ftp.username", "anonymous");
- this.passWord = conf.get("ftp.password", "anonymous@example.com");
- this.serverTimeout = conf.getInt("ftp.server.timeout", 60 * 1000);
- this.keepConnection = conf.getBoolean("ftp.keep.connection", false);
- this.followTalk = conf.getBoolean("ftp.follow.talk", false);
- }
-
- public Configuration getConf() {
- return this.conf;
- }
-
- public RobotRules getRobotRules(Text url, CrawlDatum datum) {
- return EmptyRobotRules.RULES;
+ public Collection<WebPage.Field> getFields() {
+ return FIELDS;
}
}
Modified: nutch/branches/nutchbase/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java (original)
+++ nutch/branches/nutchbase/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java Wed Jun 30 10:36:20 2010
@@ -18,31 +18,26 @@
package org.apache.nutch.protocol.ftp;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.URL;
+import java.util.LinkedList;
+import java.util.List;
+
import org.apache.commons.net.ftp.FTP;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
-
import org.apache.commons.net.ftp.parser.DefaultFTPFileEntryParserFactory;
import org.apache.commons.net.ftp.parser.ParserInitializationException;
-
-import org.apache.nutch.crawl.CrawlDatum;
-import org.apache.nutch.protocol.Content;
+import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.net.protocols.HttpDateFormat;
import org.apache.nutch.net.protocols.Response;
+import org.apache.nutch.protocol.Content;
+import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.LogUtil;
-import org.apache.hadoop.conf.Configuration;
-
-import java.net.InetAddress;
-import java.net.URL;
-
-import java.util.List;
-import java.util.LinkedList;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
/************************************
* FtpResponse.java mimics ftp replies as http response.
@@ -85,7 +80,7 @@ public class FtpResponse {
headers, this.conf);
}
- public FtpResponse(URL url, CrawlDatum datum, Ftp ftp, Configuration conf)
+ public FtpResponse(URL url, WebPage page, Ftp ftp, Configuration conf)
throws FtpException, IOException {
this.orig = url.toString();
@@ -259,9 +254,9 @@ public class FtpResponse {
this.content = null;
if (path.endsWith("/")) {
- getDirAsHttpResponse(path, datum.getModifiedTime());
+ getDirAsHttpResponse(path, page.getModifiedTime());
} else {
- getFileAsHttpResponse(path, datum.getModifiedTime());
+ getFileAsHttpResponse(path, page.getModifiedTime());
}
// reset next renewalTime, take the lesser
Added: nutch/branches/nutchbase/src/plugin/protocol-http/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-http/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-http/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/protocol-http/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Modified: nutch/branches/nutchbase/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java (original)
+++ nutch/branches/nutchbase/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java Wed Jun 30 10:36:20 2010
@@ -28,26 +28,25 @@ import org.apache.hadoop.conf.Configurat
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.ProtocolException;
import org.apache.nutch.protocol.http.api.HttpBase;
+import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
-import org.apache.nutch.util.hbase.HbaseColumn;
-import org.apache.nutch.util.hbase.WebTableColumns;
-import org.apache.nutch.util.hbase.WebTableRow;
-
public class Http extends HttpBase {
public static final Log LOG = LogFactory.getLog(Http.class);
- private static final Collection<HbaseColumn> COLUMNS = new HashSet<HbaseColumn>();
-
+ private static final Collection<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();
+
static {
- COLUMNS.add(new HbaseColumn(WebTableColumns.MODIFIED_TIME));
+ FIELDS.add(WebPage.Field.MODIFIED_TIME);
+ FIELDS.add(WebPage.Field.HEADERS);
}
-
+
public Http() {
super(LOG);
}
+ @Override
public void setConf(Configuration conf) {
super.setConf(conf);
// Level logLevel = Level.WARNING;
@@ -64,13 +63,13 @@ public class Http extends HttpBase {
}
@Override
- protected Response getResponse(URL url, WebTableRow row, boolean redirect)
+ protected Response getResponse(URL url, WebPage page, boolean redirect)
throws ProtocolException, IOException {
- return new HttpResponse(this, url, row);
+ return new HttpResponse(this, url, page);
}
-
- public Collection<HbaseColumn> getColumns() {
- return COLUMNS;
+
+ public Collection<WebPage.Field> getFields() {
+ return FIELDS;
}
}
Modified: nutch/branches/nutchbase/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java (original)
+++ nutch/branches/nutchbase/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java Wed Jun 30 10:36:20 2010
@@ -28,6 +28,7 @@ import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.URL;
+import org.apache.avro.util.Utf8;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.metadata.SpellCheckedMetadata;
import org.apache.nutch.net.protocols.HttpDateFormat;
@@ -35,23 +36,21 @@ import org.apache.nutch.net.protocols.Re
import org.apache.nutch.protocol.ProtocolException;
import org.apache.nutch.protocol.http.api.HttpBase;
import org.apache.nutch.protocol.http.api.HttpException;
+import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.LogUtil;
-import org.apache.nutch.util.hbase.WebTableColumns;
-import org.apache.nutch.util.hbase.WebTableRow;
-
/** An HTTP response. */
public class HttpResponse implements Response {
-
- private HttpBase http;
- private URL url;
+
+ private final HttpBase http;
+ private final URL url;
private byte[] content;
private int code;
- private Metadata headers = new SpellCheckedMetadata();
+ private final Metadata headers = new SpellCheckedMetadata();
- public HttpResponse(HttpBase http, URL url, WebTableRow row)
- throws ProtocolException, IOException {
+ public HttpResponse(HttpBase http, URL url, WebPage page)
+ throws ProtocolException, IOException {
this.http = http;
this.url = url;
@@ -120,21 +119,21 @@ public class HttpResponse implements Res
reqStr.append("\r\n");
}
- reqStr.append("\r\n");
- if (row.hasColumn(WebTableColumns.MODIFIED_TIME, null)) {
+ if (page.isReadable(WebPage.Field.MODIFIED_TIME.getIndex())) {
reqStr.append("If-Modified-Since: " +
- HttpDateFormat.toString(row.getModifiedTime()));
+ HttpDateFormat.toString(page.getModifiedTime()));
reqStr.append("\r\n");
}
-
+ reqStr.append("\r\n");
+
byte[] reqBytes= reqStr.toString().getBytes();
req.write(reqBytes);
req.flush();
-
+
PushbackInputStream in = // process response
new PushbackInputStream(
- new BufferedInputStream(socket.getInputStream(), Http.BUFFER_SIZE),
+ new BufferedInputStream(socket.getInputStream(), Http.BUFFER_SIZE),
Http.BUFFER_SIZE) ;
StringBuffer line = new StringBuffer();
@@ -142,7 +141,7 @@ public class HttpResponse implements Res
boolean haveSeenNonContinueStatus= false;
while (!haveSeenNonContinueStatus) {
// parse status code line
- this.code = parseStatusLine(in, line);
+ this.code = parseStatusLine(in, line);
// parse headers
parseHeaders(in, line);
haveSeenNonContinueStatus= code != 100; // 100 is "Continue"
@@ -158,11 +157,13 @@ public class HttpResponse implements Res
Http.LOG.trace("fetched " + content.length + " bytes from " + url);
}
}
-
+
// add headers in metadata to row
- row.deleteHeaders();
+ if (page.getHeaders() != null) {
+ page.getHeaders().clear();
+ }
for (String key : headers.names()) {
- row.addHeader(key, headers.get(key));
+ page.putToHeaders(new Utf8(key), new Utf8(headers.get(key)));
}
} finally {
@@ -172,15 +173,15 @@ public class HttpResponse implements Res
}
-
+
/* ------------------------- *
* <implementation:Response> *
* ------------------------- */
-
+
public URL getUrl() {
return url;
}
-
+
public int getCode() {
return code;
}
@@ -188,7 +189,7 @@ public class HttpResponse implements Res
public String getHeader(String name) {
return headers.get(name);
}
-
+
public Metadata getHeaders() {
return headers;
}
@@ -200,9 +201,9 @@ public class HttpResponse implements Res
/* ------------------------- *
* <implementation:Response> *
* ------------------------- */
-
- private void readPlainContent(InputStream in)
+
+ private void readPlainContent(InputStream in)
throws HttpException, IOException {
int contentLength = Integer.MAX_VALUE; // get content length
@@ -241,14 +242,14 @@ public class HttpResponse implements Res
// handle lines with no plaintext result code, ie:
// "HTTP/1.1 200" vs "HTTP/1.1 200 OK"
- if (codeEnd == -1)
+ if (codeEnd == -1)
codeEnd= line.length();
int code;
try {
code= Integer.parseInt(line.substring(codeStart+1, codeEnd));
} catch (NumberFormatException e) {
- throw new HttpException("bad status line '" + line
+ throw new HttpException("bad status line '" + line
+ "': " + e.getMessage(), e);
}
@@ -291,8 +292,8 @@ public class HttpResponse implements Res
// handle HTTP responses with missing blank line after headers
int pos;
- if ( ((pos= line.indexOf("<!DOCTYPE")) != -1)
- || ((pos= line.indexOf("<HTML")) != -1)
+ if ( ((pos= line.indexOf("<!DOCTYPE")) != -1)
+ || ((pos= line.indexOf("<HTML")) != -1)
|| ((pos= line.indexOf("<html")) != -1) ) {
in.unread(line.substring(pos).getBytes("UTF-8"));
@@ -326,11 +327,11 @@ public class HttpResponse implements Res
if (peek(in) == '\n') {
in.read();
}
- case '\n':
+ case '\n':
if (line.length() > 0) {
// at EOL -- check for continued line if the current
// (possibly continued) line wasn't blank
- if (allowContinuedLine)
+ if (allowContinuedLine)
switch (peek(in)) {
case ' ' : case '\t': // line is continued
in.read();
Added: nutch/branches/nutchbase/src/plugin/protocol-httpclient/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/protocol-httpclient/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/protocol-httpclient/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/protocol-httpclient/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/plugin/scoring-link/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/scoring-link/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/scoring-link/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/scoring-link/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/plugin/scoring-opic/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/scoring-opic/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/scoring-opic/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/scoring-opic/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Modified: nutch/branches/nutchbase/src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java (original)
+++ nutch/branches/nutchbase/src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java Wed Jun 30 10:36:20 2010
@@ -19,24 +19,23 @@ package org.apache.nutch.scoring.opic;
import java.net.MalformedURLException;
import java.net.URL;
+import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.avro.util.Utf8;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.scoring.ScoreDatum;
import org.apache.nutch.scoring.ScoringFilter;
import org.apache.nutch.scoring.ScoringFilterException;
+import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.LogUtil;
-import org.apache.nutch.util.hbase.HbaseColumn;
-import org.apache.nutch.util.hbase.WebTableRow;
-import org.apache.nutch.util.hbase.WebTableColumns;
/**
* This plugin implements a variant of an Online Page Importance Computation
@@ -45,20 +44,20 @@ import org.apache.nutch.util.hbase.WebTa
* Abiteboul, Serge and Preda, Mihai and Cobena, Gregory (2003),
* Adaptive On-Line Page Importance Computation
* </a>.
- *
+ *
* @author Andrzej Bialecki
*/
public class OPICScoringFilter implements ScoringFilter {
private final static Log LOG = LogFactory.getLog(OPICScoringFilter.class);
-
- private final static byte[] CASH_KEY = Bytes.toBytes("_csh_");
-
- private final static Set<HbaseColumn> COLUMNS = new HashSet<HbaseColumn>();
-
+
+ private final static Utf8 CASH_KEY = new Utf8("_csh_");
+
+ private final static Set<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();
+
static {
- COLUMNS.add(new HbaseColumn(WebTableColumns.METADATA, CASH_KEY));
- COLUMNS.add(new HbaseColumn(WebTableColumns.SCORE, CASH_KEY));
+ FIELDS.add(WebPage.Field.METADATA);
+ FIELDS.add(WebPage.Field.SCORE);
}
private Configuration conf;
@@ -84,45 +83,53 @@ public class OPICScoringFilter implement
/** Set to the value defined in config, 1.0f by default. */
@Override
- public void injectedScore(String url, WebTableRow row)
+ public void injectedScore(String url, WebPage row)
throws ScoringFilterException {
row.setScore(scoreInjected);
- row.putMeta(CASH_KEY, Bytes.toBytes(scoreInjected));
+ row.putToMetadata(CASH_KEY, ByteBuffer.wrap(Bytes.toBytes(scoreInjected)));
}
/** Set to 0.0f (unknown value) - inlink contributions will bring it to
* a correct level. Newly discovered pages have at least one inlink. */
@Override
- public void initialScore(String url, WebTableRow row) throws ScoringFilterException {
+ public void initialScore(String url, WebPage row) throws ScoringFilterException {
row.setScore(0.0f);
- row.putMeta(CASH_KEY, Bytes.toBytes(0.0f));
+ row.putToMetadata(CASH_KEY, ByteBuffer.wrap(Bytes.toBytes(0.0f)));
}
- /** Use {@link CrawlDatum#getScore()}. */
+ /** Use {@link WebPage#getScore()}. */
@Override
- public float generatorSortValue(String url, WebTableRow row, float initSort) throws ScoringFilterException {
+ public float generatorSortValue(String url, WebPage row, float initSort) throws ScoringFilterException {
return row.getScore() * initSort;
}
/** Increase the score by a sum of inlinked scores. */
@Override
- public void updateScore(String url, WebTableRow row, List<ScoreDatum> inlinkedScoreData) {
+ public void updateScore(String url, WebPage row, List<ScoreDatum> inlinkedScoreData) {
float adjust = 0.0f;
for (ScoreDatum scoreDatum : inlinkedScoreData) {
adjust += scoreDatum.getScore();
}
float oldScore = row.getScore();
row.setScore(oldScore + adjust);
- float cash = Bytes.toFloat(row.getMeta(CASH_KEY));
- row.putMeta(CASH_KEY, Bytes.toBytes(cash + adjust));
+ ByteBuffer cashRaw = row.getFromMetadata(CASH_KEY);
+ float cash = 0.0f;
+ if (cashRaw != null) {
+ cash = Bytes.toFloat(cashRaw.array());
+ }
+ row.putToMetadata(CASH_KEY, ByteBuffer.wrap(Bytes.toBytes(cash + adjust)));
}
/** Get cash on hand, divide it by the number of outlinks and apply. */
@Override
public void distributeScoreToOutlinks(String fromUrl,
- WebTableRow row, Collection<ScoreDatum> scoreData,
+ WebPage row, Collection<ScoreDatum> scoreData,
int allCount) {
- float cash = Bytes.toFloat(row.getMeta(CASH_KEY));
+ ByteBuffer cashRaw = row.getFromMetadata(CASH_KEY);
+ if (cashRaw == null) {
+ return;
+ }
+ float cash = Bytes.toFloat(cashRaw.array());
if (cash == 0) {
return;
}
@@ -146,16 +153,16 @@ public class OPICScoringFilter implement
}
}
// reset cash to zero
- row.putMeta(CASH_KEY, Bytes.toBytes(0.0f));
+ row.putToMetadata(CASH_KEY, ByteBuffer.wrap(Bytes.toBytes(0.0f)));
}
/** Dampen the boost value by scorePower.*/
- public float indexerScore(String url, NutchDocument doc, WebTableRow row, float initScore) {
+ public float indexerScore(String url, NutchDocument doc, WebPage row, float initScore) {
return (float)Math.pow(row.getScore(), scorePower) * initScore;
}
@Override
- public Collection<HbaseColumn> getColumns() {
- return COLUMNS;
+ public Collection<WebPage.Field> getFields() {
+ return FIELDS;
}
}
Added: nutch/branches/nutchbase/src/plugin/subcollection/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/subcollection/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/subcollection/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/subcollection/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/plugin/tld/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/tld/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/tld/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/tld/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Modified: nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java (original)
+++ nutch/branches/nutchbase/src/plugin/tld/src/java/org/apache/nutch/indexer/tld/TLDIndexingFilter.java Wed Jun 30 10:36:20 2010
@@ -18,23 +18,24 @@
package org.apache.nutch.indexer.tld;
import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.nutch.crawl.CrawlDatum;
-import org.apache.nutch.crawl.Inlinks;
import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.IndexingFilter;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.indexer.lucene.LuceneWriter;
-import org.apache.nutch.parse.Parse;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.URLUtil;
import org.apache.nutch.util.domain.DomainSuffix;
/**
* Adds the Top level domain extensions to the index
+ *
* @author Enis Soztutar <enis.soz.nutch@gmail.com>
*/
public class TLDIndexingFilter implements IndexingFilter {
@@ -42,16 +43,16 @@ public class TLDIndexingFilter implement
private Configuration conf;
- public NutchDocument filter(NutchDocument doc, Parse parse, Text urlText, CrawlDatum datum, Inlinks inlinks)
- throws IndexingException {
-
+ private static final Collection<Field> fields = new ArrayList<Field>();
+
+ @Override
+ public NutchDocument filter(NutchDocument doc, String url, WebPage page)
+ throws IndexingException {
try {
- URL url = new URL(urlText.toString());
- DomainSuffix d = URLUtil.getDomainSuffix(url);
-
+ URL _url = new URL(url);
+ DomainSuffix d = URLUtil.getDomainSuffix(_url);
doc.add("tld", d.getDomain());
-
- }catch (Exception ex) {
+ } catch (Exception ex) {
LOG.warn(ex);
}
@@ -69,6 +70,11 @@ public class TLDIndexingFilter implement
public void addIndexBackendOptions(Configuration conf) {
// store, no index
LuceneWriter.addFieldOptions("tld", LuceneWriter.STORE.YES,
- LuceneWriter.INDEX.NO, conf);
+ LuceneWriter.INDEX.NO, conf);
+ }
+
+ @Override
+ public Collection<Field> getFields() {
+ return fields;
}
}
Added: nutch/branches/nutchbase/src/plugin/urlfilter-automaton/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/urlfilter-automaton/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/urlfilter-automaton/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/urlfilter-automaton/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/plugin/urlfilter-domain/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/urlfilter-domain/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/urlfilter-domain/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/urlfilter-domain/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/plugin/urlfilter-prefix/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/urlfilter-prefix/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/urlfilter-prefix/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/urlfilter-prefix/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/plugin/urlfilter-regex/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/urlfilter-regex/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/urlfilter-regex/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/urlfilter-regex/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/plugin/urlfilter-suffix/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/urlfilter-suffix/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/urlfilter-suffix/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/urlfilter-suffix/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/plugin/urlfilter-validator/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/urlfilter-validator/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/urlfilter-validator/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/urlfilter-validator/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/plugin/urlnormalizer-basic/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/urlnormalizer-basic/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/urlnormalizer-basic/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/urlnormalizer-basic/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/plugin/urlnormalizer-pass/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/urlnormalizer-pass/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/urlnormalizer-pass/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/urlnormalizer-pass/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/plugin/urlnormalizer-regex/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/plugin/urlnormalizer-regex/ivy.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/plugin/urlnormalizer-regex/ivy.xml (added)
+++ nutch/branches/nutchbase/src/plugin/urlnormalizer-regex/ivy.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="1.0">
+ <info organisation="org.apache.nutch" module="${ant.project.name}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+ <description>
+ Apache Nutch
+ </description>
+ </info>
+
+ <configurations>
+ <include file="../../../ivy/ivy-configurations.xml"/>
+ </configurations>
+
+ <publications>
+ <!--get the artifact from our module name-->
+ <artifact conf="master"/>
+ </publications>
+
+ <dependencies>
+ </dependencies>
+
+</ivy-module>
Added: nutch/branches/nutchbase/src/test/hbase-site.xml
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/test/hbase-site.xml?rev=959259&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/test/hbase-site.xml (added)
+++ nutch/branches/nutchbase/src/test/hbase-site.xml Wed Jun 30 10:36:20 2010
@@ -0,0 +1,137 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>1000</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.client.pause</name>
+ <value>5000</value>
+ <description>General client pause value. Used mostly as value to wait
+ before running a retry of a failed get, region lookup, etc.</description>
+ </property>
+ <property>
+ <name>hbase.master.meta.thread.rescanfrequency</name>
+ <value>10000</value>
+ <description>How long the HMaster sleeps (in milliseconds) between scans of
+ the root and meta tables.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as META scanner and log roller.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>5</value>
+ <description>Count of RPC Server instances spun up on RegionServers
+ Same property is used by the HMaster for count of master handlers.
+ Default is 10.
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.lease.period</name>
+ <value>6000</value>
+ <description>Length of time the master will wait before timing out a region
+ server lease. Since region servers report in every second (see above), this
+ value has been reduced so that the master will notice a dead region server
+ sooner. The default is 30 seconds.
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.info.port</name>
+ <value>-1</value>
+ <description>The port for the hbase master web UI
+ Set to -1 if you do not want the info server to run.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.info.port</name>
+ <value>-1</value>
+ <description>The port for the hbase regionserver web UI
+ Set to -1 if you do not want the info server to run.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.info.port.auto</name>
+ <value>true</value>
+ <description>Info server auto port bind. Enables automatic port
+ search if hbase.regionserver.info.port is already in use.
+ Enabled for testing to run multiple tests on one machine.
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.lease.thread.wakefrequency</name>
+ <value>3000</value>
+ <description>The interval between checks for expired region server leases.
+ This value has been reduced due to the other reduced values above so that
+ the master will notice a dead region server sooner. The default is 15 seconds.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.optionalcacheflushinterval</name>
+ <value>10000</value>
+ <description>
+ Amount of time to wait since the last time a region was flushed before
+ invoking an optional cache flush. Default 60,000.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.safemode</name>
+ <value>false</value>
+ <description>
+ Turn on/off safe mode in region server. Always on for production, always off
+ for tests.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.max.filesize</name>
+ <value>67108864</value>
+ <description>
+ Maximum desired file size for an HRegion. If filesize exceeds
+ value + (value / 2), the HRegion is split in two. Default: 256M.
+
+ Keep the maximum filesize small so we split more often in tests.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.log.dir</name>
+ <value>${user.dir}/../logs</value>
+ </property>
+ <property>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>21810</value>
+ <description>Property from ZooKeeper's config zoo.cfg.
+ The port at which the clients will connect.
+ </description>
+ </property>
+</configuration>
Modified: nutch/branches/nutchbase/src/test/log4j.properties
URL: http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/test/log4j.properties?rev=959259&r1=959258&r2=959259&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/test/log4j.properties (original)
+++ nutch/branches/nutchbase/src/test/log4j.properties Wed Jun 30 10:36:20 2010
@@ -1,7 +1,7 @@
# log4j configuration used during build and unit tests
log4j.rootLogger=info,stdout
-log4j.threshhold=ALL
+log4j.threshhold=INFO
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n