You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ab...@apache.org on 2006/01/31 17:13:17 UTC
svn commit: r373853 [6/6] - in /lucene/nutch/trunk/src:
java/org/apache/nutch/analysis/ java/org/apache/nutch/clustering/
java/org/apache/nutch/crawl/ java/org/apache/nutch/fetcher/
java/org/apache/nutch/fs/ java/org/apache/nutch/indexer/ java/org/apac...
Modified: lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java (original)
+++ lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java Tue Jan 31 08:08:58 2006
@@ -32,6 +32,7 @@
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.ContentProperties;
+import org.apache.nutch.protocol.http.api.HttpBase;
/**
@@ -47,24 +48,26 @@
private byte[] content;
+ private HttpBase http;
private int code;
private ContentProperties headers = new ContentProperties();
- public HttpResponse(URL url, CrawlDatum datum) throws IOException {
- this(url, datum, false);
+ public HttpResponse(HttpBase http, URL url, CrawlDatum datum) throws IOException {
+ this(http, url, datum, false);
}
- HttpResponse(URL url, CrawlDatum datum, boolean followRedirects) throws IOException {
+ HttpResponse(HttpBase http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {
+ this.http = http;
this.url = url;
this.base = url.toString();
this.orig = url.toString();
GetMethod get = new GetMethod(this.orig);
get.setFollowRedirects(followRedirects);
- get.setRequestHeader("User-Agent", Http.AGENT_STRING);
+ get.setRequestHeader("User-Agent", http.getUserAgent());
HttpMethodParams params = get.getParams();
// some servers cannot digest the new protocol
params.setVersion(HttpVersion.HTTP_1_0);
@@ -88,7 +91,7 @@
// for error.
try {
InputStream in = get.getResponseBodyAsStream();
- byte[] buffer = new byte[Http.BUFFER_SIZE];
+ byte[] buffer = new byte[http.BUFFER_SIZE];
int bufferFilled = 0;
int totalRead = 0;
ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -147,10 +150,10 @@
private int calculateTryToRead(int totalRead) {
int tryToRead = Http.BUFFER_SIZE;
- if (Http.MAX_CONTENT <= 0) {
- return Http.BUFFER_SIZE;
- } else if (Http.MAX_CONTENT - totalRead < Http.BUFFER_SIZE) {
- tryToRead = Http.MAX_CONTENT - totalRead;
+ if (http.getMaxContent() <= 0) {
+ return http.BUFFER_SIZE;
+ } else if (http.getMaxContent() - totalRead < http.BUFFER_SIZE) {
+ tryToRead = http.getMaxContent() - totalRead;
}
return tryToRead;
}
Modified: lucene/nutch/trunk/src/plugin/query-basic/src/java/org/apache/nutch/searcher/basic/BasicQueryFilter.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/query-basic/src/java/org/apache/nutch/searcher/basic/BasicQueryFilter.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/query-basic/src/java/org/apache/nutch/searcher/basic/BasicQueryFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/query-basic/src/java/org/apache/nutch/searcher/basic/BasicQueryFilter.java Tue Jan 31 08:08:58 2006
@@ -35,46 +35,43 @@
* expanded to search the url, anchor and content document fields.*/
public class BasicQueryFilter implements QueryFilter {
- private static float URL_BOOST =
- NutchConf.get().getFloat("query.url.boost", 4.0f);
+ private float URL_BOOST;
- private static float ANCHOR_BOOST =
- NutchConf.get().getFloat("query.anchor.boost", 2.0f);
+ private float ANCHOR_BOOST ;
- private static float TITLE_BOOST =
- NutchConf.get().getFloat("query.title.boost", 1.5f);
+ private float TITLE_BOOST;
- private static float HOST_BOOST =
- NutchConf.get().getFloat("query.host.boost", 2.0f);
+ private float HOST_BOOST;
private static int SLOP = Integer.MAX_VALUE;
- private static float PHRASE_BOOST =
- NutchConf.get().getFloat("query.phrase.boost", 1.0f);
+ private float PHRASE_BOOST;
private static final String[] FIELDS =
{ "url", "anchor", "content", "title", "host" };
- private static final float[] FIELD_BOOSTS =
+ private final float[] FIELD_BOOSTS =
{ URL_BOOST, ANCHOR_BOOST, 1.0f, TITLE_BOOST, HOST_BOOST };
/**
* Set the boost factor for url matches, relative to content and anchor
* matches
*/
- public static void setUrlBoost(float boost) { URL_BOOST = boost; }
+ public void setUrlBoost(float boost) { URL_BOOST = boost; }
/** Set the boost factor for title/anchor matches, relative to url and
* content matches. */
- public static void setAnchorBoost(float boost) { ANCHOR_BOOST = boost; }
+ public void setAnchorBoost(float boost) { ANCHOR_BOOST = boost; }
/** Set the boost factor for sloppy phrase matches relative to unordered term
* matches. */
- public static void setPhraseBoost(float boost) { PHRASE_BOOST = boost; }
+ public void setPhraseBoost(float boost) { PHRASE_BOOST = boost; }
/** Set the maximum number of terms permitted between matching terms in a
* sloppy phrase match. */
- public static void setSlop(int slop) { SLOP = slop; }
+ public void setSlop(int slop) { SLOP = slop; }
+
+ private NutchConf nutchConf;
public BooleanQuery filter(Query input, BooleanQuery output) {
addTerms(input, output);
@@ -82,7 +79,7 @@
return output;
}
- private static void addTerms(Query input, BooleanQuery output) {
+ private void addTerms(Query input, BooleanQuery output) {
Clause[] clauses = input.getClauses();
for (int i = 0; i < clauses.length; i++) {
Clause c = clauses[i];
@@ -95,11 +92,11 @@
Clause o = c;
if (c.isPhrase()) { // optimize phrase clauses
- String[] opt = CommonGrams.optimizePhrase(c.getPhrase(), FIELDS[f]);
+ String[] opt = new CommonGrams(getConf()).optimizePhrase(c.getPhrase(), FIELDS[f]);
if (opt.length==1) {
- o = new Clause(new Term(opt[0]), c.isRequired(), c.isProhibited());
+ o = new Clause(new Term(opt[0]), c.isRequired(), c.isProhibited(), getConf());
} else {
- o = new Clause(new Phrase(opt), c.isRequired(), c.isProhibited());
+ o = new Clause(new Phrase(opt), c.isRequired(), c.isProhibited(), getConf());
}
}
@@ -112,7 +109,7 @@
}
}
- private static void addSloppyPhrases(Query input, BooleanQuery output) {
+ private void addSloppyPhrases(Query input, BooleanQuery output) {
Clause[] clauses = input.getClauses();
for (int f = 0; f < FIELDS.length; f++) {
@@ -145,7 +142,7 @@
}
- private static org.apache.lucene.search.Query
+ private org.apache.lucene.search.Query
termQuery(String field, Term term, float boost) {
TermQuery result = new TermQuery(luceneTerm(field, term));
result.setBoost(boost);
@@ -153,7 +150,7 @@
}
/** Utility to construct a Lucene exact phrase query for a Nutch phrase. */
- private static org.apache.lucene.search.Query
+ private org.apache.lucene.search.Query
exactPhrase(Phrase nutchPhrase,
String field, float boost) {
Term[] terms = nutchPhrase.getTerms();
@@ -169,5 +166,18 @@
private static org.apache.lucene.index.Term luceneTerm(String field,
Term term) {
return new org.apache.lucene.index.Term(field, term.toString());
+ }
+
+ public void setConf(NutchConf conf) {
+ this.nutchConf = conf;
+ this.URL_BOOST = conf.getFloat("query.url.boost", 4.0f);
+ this.ANCHOR_BOOST = conf.getFloat("query.anchor.boost", 2.0f);
+ this.TITLE_BOOST = conf.getFloat("query.title.boost", 1.5f);
+ this.HOST_BOOST = conf.getFloat("query.host.boost", 2.0f);
+ this.PHRASE_BOOST = conf.getFloat("query.phrase.boost", 1.0f);
+ }
+
+ public NutchConf getConf() {
+ return this.nutchConf;
}
}
Modified: lucene/nutch/trunk/src/plugin/query-more/src/java/org/apache/nutch/searcher/more/DateQueryFilter.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/query-more/src/java/org/apache/nutch/searcher/more/DateQueryFilter.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/query-more/src/java/org/apache/nutch/searcher/more/DateQueryFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/query-more/src/java/org/apache/nutch/searcher/more/DateQueryFilter.java Tue Jan 31 08:08:58 2006
@@ -22,6 +22,7 @@
import org.apache.nutch.searcher.QueryException;
import org.apache.nutch.util.LogFormatter;
+import org.apache.nutch.util.NutchConf;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.RangeQuery;
@@ -47,6 +48,8 @@
// query syntax is defined as date:yyyymmdd-yyyymmdd
private static final Pattern pattern = Pattern.compile("^(\\d{8})-(\\d{8})$");
+
+ private NutchConf nutchConf;
public BooleanQuery filter(Query input, BooleanQuery output)
throws QueryException {
@@ -84,4 +87,11 @@
return output;
}
+ public void setConf(NutchConf conf) {
+ this.nutchConf = conf;
+ }
+
+ public NutchConf getConf() {
+ return this.nutchConf;
+ }
}
Modified: lucene/nutch/trunk/src/plugin/query-more/src/java/org/apache/nutch/searcher/more/TypeQueryFilter.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/query-more/src/java/org/apache/nutch/searcher/more/TypeQueryFilter.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/query-more/src/java/org/apache/nutch/searcher/more/TypeQueryFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/query-more/src/java/org/apache/nutch/searcher/more/TypeQueryFilter.java Tue Jan 31 08:08:58 2006
@@ -17,6 +17,7 @@
package org.apache.nutch.searcher.more;
import org.apache.nutch.searcher.RawFieldQueryFilter;
+import org.apache.nutch.util.NutchConf;
/**
* Handles "type:" query clauses, causing them to search the field
@@ -26,7 +27,17 @@
*/
public class TypeQueryFilter extends RawFieldQueryFilter {
+ private NutchConf nutchConf;
+
public TypeQueryFilter() {
super("type");
+ }
+
+ public void setConf(NutchConf conf) {
+ this.nutchConf = conf;
+ }
+
+ public NutchConf getConf() {
+ return this.nutchConf;
}
}
Modified: lucene/nutch/trunk/src/plugin/query-site/src/java/org/apache/nutch/searcher/site/SiteQueryFilter.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/query-site/src/java/org/apache/nutch/searcher/site/SiteQueryFilter.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/query-site/src/java/org/apache/nutch/searcher/site/SiteQueryFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/query-site/src/java/org/apache/nutch/searcher/site/SiteQueryFilter.java Tue Jan 31 08:08:58 2006
@@ -17,11 +17,22 @@
package org.apache.nutch.searcher.site;
import org.apache.nutch.searcher.RawFieldQueryFilter;
+import org.apache.nutch.util.NutchConf;
/** Handles "site:" query clauses, causing them to search the field indexed by
* SiteIndexingFilter. */
public class SiteQueryFilter extends RawFieldQueryFilter {
+ private NutchConf nutchConf;
+
public SiteQueryFilter() {
super("site");
+ }
+
+ public void setConf(NutchConf conf) {
+ this.nutchConf = conf;
+ }
+
+ public NutchConf getConf() {
+ return this.nutchConf;
}
}
Modified: lucene/nutch/trunk/src/plugin/query-url/src/java/org/apache/nutch/searcher/url/URLQueryFilter.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/query-url/src/java/org/apache/nutch/searcher/url/URLQueryFilter.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/query-url/src/java/org/apache/nutch/searcher/url/URLQueryFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/query-url/src/java/org/apache/nutch/searcher/url/URLQueryFilter.java Tue Jan 31 08:08:58 2006
@@ -17,11 +17,24 @@
package org.apache.nutch.searcher.url;
import org.apache.nutch.searcher.FieldQueryFilter;
+import org.apache.nutch.util.NutchConf;
-/** Handles "url:" query clauses, causing them to search the field indexed by
- * BasicIndexingFilter. */
+/**
+ * Handles "url:" query clauses, causing them to search the field indexed by
+ * BasicIndexingFilter.
+ */
public class URLQueryFilter extends FieldQueryFilter {
+ private NutchConf nutchConf;
+
public URLQueryFilter() {
super("url");
+ }
+
+ public void setConf(NutchConf conf) {
+ this.nutchConf = conf;
+ }
+
+ public NutchConf getConf() {
+ return this.nutchConf;
}
}
Modified: lucene/nutch/trunk/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/net/PrefixURLFilter.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/net/PrefixURLFilter.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/net/PrefixURLFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/net/PrefixURLFilter.java Tue Jan 31 08:08:58 2006
@@ -52,40 +52,13 @@
// read in attribute "file" of this plugin.
private static String attributeFile = null;
- static {
- String pluginName = "urlfilter-prefix";
- Extension[] extensions = PluginRepository.getInstance()
- .getExtensionPoint(URLFilter.class.getName()).getExtensions();
- for (int i=0; i < extensions.length; i++) {
- Extension extension = extensions[i];
- if (extension.getDescriptor().getPluginId().equals(pluginName)) {
- attributeFile = extension.getAttribute("file");
- break;
- }
- }
- if (attributeFile != null && attributeFile.trim().equals(""))
- attributeFile = null;
- if (attributeFile != null) {
- LOG.info("Attribute \"file\" is defined for plugin "+pluginName+" as "+attributeFile);
- } else {
- //LOG.warning("Attribute \"file\" is not defined in plugin.xml for plugin "+pluginName);
- }
- }
private TrieStringMatcher trie;
- public PrefixURLFilter() throws IOException {
- String file = NutchConf.get().get("urlfilter.prefix.file");
- // attribute "file" takes precedence if defined
- if (attributeFile != null)
- file = attributeFile;
- Reader reader = NutchConf.get().getConfResourceAsReader(file);
+ private NutchConf nutchConf;
- if (reader == null) {
- trie = new PrefixStringMatcher(new String[0]);
- } else {
- trie = readConfigurationFile(reader);
- }
+ public PrefixURLFilter() throws IOException {
+
}
public PrefixURLFilter(String filename) throws IOException {
@@ -99,7 +72,7 @@
return url;
}
- private static TrieStringMatcher readConfigurationFile(Reader reader)
+ private TrieStringMatcher readConfigurationFile(Reader reader)
throws IOException {
BufferedReader in=new BufferedReader(reader);
@@ -139,6 +112,52 @@
System.out.println(out);
}
}
+ }
+
+ public void setConf(NutchConf conf) {
+ this.nutchConf = conf;
+
+ String pluginName = "urlfilter-prefix";
+ Extension[] extensions = conf.getPluginRepository().getExtensionPoint(
+ URLFilter.class.getName()).getExtensions();
+ for (int i = 0; i < extensions.length; i++) {
+ Extension extension = extensions[i];
+ if (extension.getDescriptor().getPluginId().equals(pluginName)) {
+ attributeFile = extension.getAttribute("file");
+ break;
+ }
+ }
+ if (attributeFile != null && attributeFile.trim().equals(""))
+ attributeFile = null;
+ if (attributeFile != null) {
+ LOG.info("Attribute \"file\" is defined for plugin " + pluginName
+ + " as " + attributeFile);
+ } else {
+ // LOG.warning("Attribute \"file\" is not defined in plugin.xml for
+ // plugin "+pluginName);
+ }
+
+ String file = conf.get("urlfilter.prefix.file");
+ // attribute "file" takes precedence if defined
+ if (attributeFile != null)
+ file = attributeFile;
+ Reader reader = conf.getConfResourceAsReader(file);
+
+ if (reader == null) {
+ trie = new PrefixStringMatcher(new String[0]);
+ } else {
+ try {
+ trie = readConfigurationFile(reader);
+ } catch (IOException e) {
+ LOG.severe(e.getMessage());
+ // TODO mb@media-style.com: throw Exception? Because broken api.
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ }
+ }
+
+ public NutchConf getConf() {
+ return this.nutchConf;
}
}
Modified: lucene/nutch/trunk/src/plugin/urlfilter-regex/src/java/org/apache/nutch/net/RegexURLFilter.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/plugin/urlfilter-regex/src/java/org/apache/nutch/net/RegexURLFilter.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/urlfilter-regex/src/java/org/apache/nutch/net/RegexURLFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/urlfilter-regex/src/java/org/apache/nutch/net/RegexURLFilter.java Tue Jan 31 08:08:58 2006
@@ -21,6 +21,7 @@
import org.apache.nutch.plugin.Extension;
import org.apache.nutch.plugin.PluginRepository;
+import org.apache.oro.text.regex.MalformedPatternException;
import java.io.Reader;
import java.io.FileReader;
@@ -54,25 +55,6 @@
// read in attribute "file" of this plugin.
private static String attributeFile = null;
- static {
- String pluginName = "urlfilter-regex";
- Extension[] extensions = PluginRepository.getInstance()
- .getExtensionPoint(URLFilter.class.getName()).getExtensions();
- for (int i=0; i < extensions.length; i++) {
- Extension extension = extensions[i];
- if (extension.getDescriptor().getPluginId().equals(pluginName)) {
- attributeFile = extension.getAttribute("file");
- break;
- }
- }
- if (attributeFile != null && attributeFile.trim().equals(""))
- attributeFile = null;
- if (attributeFile != null) {
- LOG.info("Attribute \"file\" is defined for plugin "+pluginName+" as "+attributeFile);
- } else {
- //LOG.warning("Attribute \"file\" is not defined in plugin.xml for plugin "+pluginName);
- }
- }
private static class Rule {
public Pattern pattern;
@@ -82,18 +64,9 @@
private List rules;
- public RegexURLFilter() throws IOException, PatternSyntaxException {
- String file = NutchConf.get().get("urlfilter.regex.file");
- // attribute "file" takes precedence if defined
- if (attributeFile != null)
- file = attributeFile;
- Reader reader = NutchConf.get().getConfResourceAsReader(file);
+ private NutchConf nutchConf;
- if (reader == null) {
- LOG.severe("Can't find resource: " + file);
- } else {
- rules=readConfigurationFile(reader);
- }
+ public RegexURLFilter() {
}
public RegexURLFilter(String filename)
@@ -177,6 +150,49 @@
System.out.println(line);
}
}
+ }
+
+ public void setConf(NutchConf conf) {
+ this.nutchConf = conf;
+ String pluginName = "urlfilter-regex";
+ Extension[] extensions = conf.getPluginRepository().getExtensionPoint(
+ URLFilter.class.getName()).getExtensions();
+ for (int i = 0; i < extensions.length; i++) {
+ Extension extension = extensions[i];
+ if (extension.getDescriptor().getPluginId().equals(pluginName)) {
+ attributeFile = extension.getAttribute("file");
+ break;
+ }
+ }
+ if (attributeFile != null && attributeFile.trim().equals(""))
+ attributeFile = null;
+ if (attributeFile != null) {
+ LOG.info("Attribute \"file\" is defined for plugin " + pluginName
+ + " as " + attributeFile);
+ } else {
+ //LOG.warning("Attribute \"file\" is not defined in plugin.xml for plugin "+pluginName);
+ }
+ String file = conf.get("urlfilter.regex.file");
+ // attribute "file" takes precedence if defined
+ if (attributeFile != null)
+ file = attributeFile;
+ Reader reader = conf.getConfResourceAsReader(file);
+
+ if (reader == null) {
+ LOG.severe("Can't find resource: " + file);
+ } else {
+ try {
+ rules = readConfigurationFile(reader);
+ } catch (IOException e) {
+ LOG.severe(e.getMessage());
+ //TODO mb@media-style.com: throw Exception? Because broken api.
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ }
+ }
+
+ public NutchConf getConf() {
+ return this.nutchConf;
}
}
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/analysis/TestQueryParser.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/analysis/TestQueryParser.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/analysis/TestQueryParser.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/analysis/TestQueryParser.java Tue Jan 31 08:08:58 2006
@@ -17,6 +17,7 @@
package org.apache.nutch.analysis;
import org.apache.nutch.searcher.Query;
+import org.apache.nutch.util.NutchConf;
import junit.framework.TestCase;
@@ -26,9 +27,10 @@
*/
public class TestQueryParser extends TestCase {
+ private static NutchConf nutchConf = new NutchConf();
public void assertQueryEquals(String query, String result) throws Exception {
try {
- Query q = NutchAnalysis.parseQuery(query);
+ Query q = NutchAnalysis.parseQuery(query, nutchConf);
String s = q.toString();
if (!s.equals(result)) {
fail("Query /" + query + "/ yielded /" + s + "/, expecting /" + result
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/fs/TestNutchFileSystem.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/fs/TestNutchFileSystem.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/fs/TestNutchFileSystem.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/fs/TestNutchFileSystem.java Tue Jan 31 08:08:58 2006
@@ -30,8 +30,8 @@
public class TestNutchFileSystem extends TestCase {
private static final Logger LOG = InputFormatBase.LOG;
- private static int BUFFER_SIZE =
- NutchConf.get().getInt("io.file.buffer.size", 4096);
+ private static NutchConf nutchConf = new NutchConf();
+ private static int BUFFER_SIZE = nutchConf.getInt("io.file.buffer.size", 4096);
private static final long MEGA = 1024 * 1024;
private static final int SEEKS_PER_FILE = 4;
@@ -49,7 +49,7 @@
public static void testFs(long megaBytes, int numFiles, long seed)
throws Exception {
- NutchFileSystem fs = NutchFileSystem.get();
+ NutchFileSystem fs = NutchFileSystem.get(nutchConf);
if (seed == 0)
seed = new Random().nextLong();
@@ -107,7 +107,7 @@
{
try {
- fs = NutchFileSystem.get();
+ fs = NutchFileSystem.get(nutchConf);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -164,7 +164,7 @@
fs.delete(DATA_DIR);
fs.delete(WRITE_DIR);
- JobConf job = new JobConf(NutchConf.get());
+ JobConf job = new JobConf(nutchConf);
job.setBoolean("fs.test.fastCheck", fastCheck);
job.setInputDir(CONTROL_DIR);
@@ -191,7 +191,7 @@
{
try {
- fs = NutchFileSystem.get();
+ fs = NutchFileSystem.get(nutchConf);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -255,7 +255,7 @@
fs.delete(READ_DIR);
- JobConf job = new JobConf(NutchConf.get());
+ JobConf job = new JobConf(nutchConf);
job.setBoolean("fs.test.fastCheck", fastCheck);
@@ -283,7 +283,7 @@
{
try {
- fs = NutchFileSystem.get();
+ fs = NutchFileSystem.get(nutchConf);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -347,7 +347,7 @@
fs.delete(READ_DIR);
- JobConf job = new JobConf(NutchConf.get());
+ JobConf job = new JobConf(nutchConf);
job.setBoolean("fs.test.fastCheck", fastCheck);
job.setInputDir(CONTROL_DIR);
@@ -401,7 +401,7 @@
LOG.info("files = " + files);
LOG.info("megaBytes = " + megaBytes);
- NutchFileSystem fs = NutchFileSystem.get();
+ NutchFileSystem fs = NutchFileSystem.get(nutchConf);
if (!noWrite) {
createControlFile(fs, megaBytes*MEGA, files, seed);
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/io/TestArrayFile.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/io/TestArrayFile.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/io/TestArrayFile.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/io/TestArrayFile.java Tue Jan 31 08:08:58 2006
@@ -35,16 +35,18 @@
}
public void testArrayFile() throws Exception {
- NutchFileSystem nfs = new LocalFileSystem();
+ NutchConf nutchConf = new NutchConf();
+ NutchFileSystem nfs = new LocalFileSystem(nutchConf);
RandomDatum[] data = generate(10000);
writeTest(nfs, data, FILE);
- readTest(nfs, data, FILE);
+ readTest(nfs, data, FILE, nutchConf);
}
public void testEmptyFile() throws Exception {
- NutchFileSystem nfs = new LocalFileSystem();
+ NutchConf nutchConf = new NutchConf();
+ NutchFileSystem nfs = new LocalFileSystem(nutchConf);
writeTest(nfs, new RandomDatum[0], FILE);
- ArrayFile.Reader reader = new ArrayFile.Reader(nfs, FILE);
+ ArrayFile.Reader reader = new ArrayFile.Reader(nfs, FILE, nutchConf);
assertNull(reader.get(0, new RandomDatum()));
reader.close();
}
@@ -71,11 +73,11 @@
writer.close();
}
- private static void readTest(NutchFileSystem nfs, RandomDatum[] data, String file)
+ private static void readTest(NutchFileSystem nfs, RandomDatum[] data, String file, NutchConf nutchConf)
throws IOException {
RandomDatum v = new RandomDatum();
LOG.fine("reading " + data.length + " records");
- ArrayFile.Reader reader = new ArrayFile.Reader(nfs, file);
+ ArrayFile.Reader reader = new ArrayFile.Reader(nfs, file, nutchConf);
for (int i = 0; i < data.length; i++) { // try forwards
reader.get(i, v);
if (!v.equals(data[i])) {
@@ -106,8 +108,9 @@
System.exit(-1);
}
+ NutchConf nutchConf = new NutchConf();
int i = 0;
- NutchFileSystem nfs = NutchFileSystem.parseArgs(args, i);
+ NutchFileSystem nfs = NutchFileSystem.parseArgs(args, i, nutchConf);
try {
for (; i < args.length; i++) { // parse command line
if (args[i] == null) {
@@ -138,7 +141,7 @@
}
if (check) {
- readTest(nfs, data, file);
+ readTest(nfs, data, file, nutchConf);
}
} finally {
nfs.close();
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/io/TestSequenceFile.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/io/TestSequenceFile.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/io/TestSequenceFile.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/io/TestSequenceFile.java Tue Jan 31 08:08:58 2006
@@ -28,6 +28,8 @@
public class TestSequenceFile extends TestCase {
private static Logger LOG = SequenceFile.LOG;
+ private static NutchConf nutchConf = new NutchConf();
+
public TestSequenceFile(String name) { super(name); }
/** Unit tests for SequenceFile. */
@@ -39,7 +41,7 @@
int seed = new Random().nextInt();
- NutchFileSystem nfs = new LocalFileSystem();
+ NutchFileSystem nfs = new LocalFileSystem(new NutchConf());
try {
//LOG.setLevel(Level.FINE);
writeTest(nfs, count, seed, file, false);
@@ -85,7 +87,7 @@
RandomDatum k = new RandomDatum();
RandomDatum v = new RandomDatum();
LOG.fine("reading " + count + " records");
- SequenceFile.Reader reader = new SequenceFile.Reader(nfs, file);
+ SequenceFile.Reader reader = new SequenceFile.Reader(nfs, file, nutchConf);
RandomDatum.Generator generator = new RandomDatum.Generator(seed);
for (int i = 0; i < count; i++) {
generator.next();
@@ -129,7 +131,7 @@
RandomDatum k = new RandomDatum();
RandomDatum v = new RandomDatum();
Iterator iterator = map.entrySet().iterator();
- SequenceFile.Reader reader = new SequenceFile.Reader(nfs, file + ".sorted");
+ SequenceFile.Reader reader = new SequenceFile.Reader(nfs, file + ".sorted", nutchConf);
for (int i = 0; i < count; i++) {
Map.Entry entry = (Map.Entry)iterator.next();
RandomDatum key = (RandomDatum)entry.getKey();
@@ -195,8 +197,8 @@
int megabytes, int factor) {
SequenceFile.Sorter sorter =
fast
- ? new SequenceFile.Sorter(nfs, new RandomDatum.Comparator(),RandomDatum.class)
- : new SequenceFile.Sorter(nfs, RandomDatum.class, RandomDatum.class);
+ ? new SequenceFile.Sorter(nfs, new RandomDatum.Comparator(),RandomDatum.class, nutchConf)
+ : new SequenceFile.Sorter(nfs, RandomDatum.class, RandomDatum.class, nutchConf);
sorter.setMemory(megabytes * 1024*1024);
sorter.setFactor(factor);
return sorter;
@@ -221,7 +223,7 @@
System.exit(-1);
}
int i = 0;
- NutchFileSystem nfs = NutchFileSystem.parseArgs(args, i);
+ NutchFileSystem nfs = NutchFileSystem.parseArgs(args, i, nutchConf);
try {
for (; i < args.length; i++) { // parse command line
if (args[i] == null) {
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/io/TestSetFile.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/io/TestSetFile.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/io/TestSetFile.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/io/TestSetFile.java Tue Jan 31 08:08:58 2006
@@ -30,10 +30,12 @@
private static String FILE =
System.getProperty("test.build.data",".") + "/test.set";
+ private static NutchConf nutchConf = new NutchConf();
+
public TestSetFile(String name) { super(name); }
public void testSetFile() throws Exception {
- NutchFileSystem nfs = new LocalFileSystem();
+ NutchFileSystem nfs = new LocalFileSystem(nutchConf);
try {
RandomDatum[] data = generate(10000);
writeTest(nfs, data, FILE);
@@ -70,7 +72,7 @@
throws IOException {
RandomDatum v = new RandomDatum();
LOG.fine("reading " + data.length + " records");
- SetFile.Reader reader = new SetFile.Reader(nfs, file);
+ SetFile.Reader reader = new SetFile.Reader(nfs, file, nutchConf);
for (int i = 0; i < data.length; i++) {
if (!reader.seek(data[i]))
throw new RuntimeException("wrong value at " + i);
@@ -94,7 +96,7 @@
}
int i = 0;
- NutchFileSystem nfs = NutchFileSystem.parseArgs(args, i);
+ NutchFileSystem nfs = NutchFileSystem.parseArgs(args, i, nutchConf);
try {
for (; i < args.length; i++) { // parse command line
if (args[i] == null) {
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/io/TestWritable.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/io/TestWritable.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/io/TestWritable.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/io/TestWritable.java Tue Jan 31 08:08:58 2006
@@ -20,6 +20,8 @@
import java.util.Random;
import junit.framework.TestCase;
import org.apache.nutch.io.*;
+import org.apache.nutch.parse.ParseData;
+import org.apache.nutch.util.NutchConf;
/** Unit tests for Writable. */
public class TestWritable extends TestCase {
@@ -68,6 +70,10 @@
dib.reset(dob.getData(), dob.getLength());
Writable after = (Writable)before.getClass().newInstance();
+ if(after instanceof ParseData) {
+ ParseData parseData = (ParseData) after;
+ parseData.setConf(new NutchConf());
+ }
after.readFields(dib);
assertEquals(before, after);
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/ipc/TestIPC.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/ipc/TestIPC.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/ipc/TestIPC.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/ipc/TestIPC.java Tue Jan 31 08:08:58 2006
@@ -29,12 +29,15 @@
import java.util.logging.Level;
import org.apache.nutch.util.LogFormatter;
+import org.apache.nutch.util.NutchConf;
/** Unit tests for IPC. */
public class TestIPC extends TestCase {
public static final Logger LOG =
LogFormatter.getLogger("org.apache.nutch.ipc.TestIPC");
+ private static NutchConf nutchConf = new NutchConf();
+
// quiet during testing, since output ends up on console
static {
LOG.setLevel(Level.WARNING);
@@ -52,7 +55,7 @@
private boolean sleep;
public TestServer(int port, int handlerCount, boolean sleep) {
- super(port, LongWritable.class, handlerCount);
+ super(port, LongWritable.class, handlerCount, nutchConf);
this.setTimeout(1000);
this.sleep = sleep;
}
@@ -145,7 +148,7 @@
Client[] clients = new Client[clientCount];
for (int i = 0; i < clientCount; i++) {
- clients[i] = new Client(LongWritable.class);
+ clients[i] = new Client(LongWritable.class, nutchConf);
}
SerialCaller[] callers = new SerialCaller[callerCount];
@@ -184,7 +187,7 @@
Client[] clients = new Client[clientCount];
for (int i = 0; i < clientCount; i++) {
- clients[i] = new Client(LongWritable.class);
+ clients[i] = new Client(LongWritable.class, nutchConf);
}
ParallelCaller[] callers = new ParallelCaller[callerCount];
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/ipc/TestRPC.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/ipc/TestRPC.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/ipc/TestRPC.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/ipc/TestRPC.java Tue Jan 31 08:08:58 2006
@@ -27,6 +27,7 @@
import java.util.Arrays;
import org.apache.nutch.util.LogFormatter;
+import org.apache.nutch.util.NutchConf;
/** Unit tests for RPC. */
public class TestRPC extends TestCase {
@@ -34,6 +35,8 @@
public static final Logger LOG =
LogFormatter.getLogger("org.apache.nutch.ipc.TestRPC");
+
+ private static NutchConf nutchConf = new NutchConf();
// quiet during testing, since output ends up on console
static {
@@ -80,12 +83,12 @@
}
public void testCalls() throws Exception {
- Server server = RPC.getServer(new TestImpl(), PORT);
+ Server server = RPC.getServer(new TestImpl(), PORT, nutchConf);
server.start();
InetSocketAddress addr = new InetSocketAddress(PORT);
TestProtocol proxy =
- (TestProtocol)RPC.getProxy(TestProtocol.class, addr);
+ (TestProtocol)RPC.getProxy(TestProtocol.class, addr, nutchConf);
proxy.ping();
@@ -114,12 +117,12 @@
Method echo =
TestProtocol.class.getMethod("echo", new Class[] { String.class });
String[] strings = (String[])RPC.call(echo, new String[][]{{"a"},{"b"}},
- new InetSocketAddress[] {addr, addr});
+ new InetSocketAddress[] {addr, addr}, nutchConf);
assertTrue(Arrays.equals(strings, new String[]{"a","b"}));
Method ping = TestProtocol.class.getMethod("ping", new Class[] {});
Object[] voids = (Object[])RPC.call(ping, new Object[][]{{},{}},
- new InetSocketAddress[] {addr, addr});
+ new InetSocketAddress[] {addr, addr}, nutchConf);
assertEquals(voids, null);
server.stop();
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/mapred/TestSequenceFileInputFormat.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/mapred/TestSequenceFileInputFormat.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/mapred/TestSequenceFileInputFormat.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/mapred/TestSequenceFileInputFormat.java Tue Jan 31 08:08:58 2006
@@ -29,10 +29,11 @@
private static final Logger LOG = InputFormatBase.LOG;
private static int MAX_LENGTH = 10000;
+ private static NutchConf nutchConf = new NutchConf();
public void testFormat() throws Exception {
- JobConf job = new JobConf(NutchConf.get());
- NutchFileSystem fs = NutchFileSystem.getNamed("local");
+ JobConf job = new JobConf(nutchConf);
+ NutchFileSystem fs = NutchFileSystem.getNamed("local", nutchConf);
File dir = new File(System.getProperty("test.build.data",".") + "/mapred");
File file = new File(dir, "test.seq");
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/mapred/TestTextInputFormat.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/mapred/TestTextInputFormat.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/mapred/TestTextInputFormat.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/mapred/TestTextInputFormat.java Tue Jan 31 08:08:58 2006
@@ -29,10 +29,11 @@
private static final Logger LOG = InputFormatBase.LOG;
private static int MAX_LENGTH = 10000;
-
+ private static NutchConf nutchConf = new NutchConf();
+
public void testFormat() throws Exception {
- JobConf job = new JobConf(NutchConf.get());
- NutchFileSystem fs = NutchFileSystem.getNamed("local");
+ JobConf job = new JobConf(nutchConf);
+ NutchFileSystem fs = NutchFileSystem.getNamed("local", nutchConf);
File dir = new File(System.getProperty("test.build.data",".") + "/mapred");
File file = new File(dir, "test.txt");
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/ndfs/TestNDFS.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/ndfs/TestNDFS.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/ndfs/TestNDFS.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/ndfs/TestNDFS.java Tue Jan 31 08:08:58 2006
@@ -72,8 +72,9 @@
private static final Logger LOG =
LogFormatter.getLogger("org.apache.nutch.ndfs.TestNDFS");
+ private static NutchConf nutchConf = new NutchConf();
private static int BUFFER_SIZE =
- NutchConf.get().getInt("io.file.buffer.size", 4096);
+ nutchConf.getInt("io.file.buffer.size", 4096);
private static int testCycleNumber = 0;
@@ -124,7 +125,7 @@
protected void setUp() throws Exception {
super.setUp();
- NutchConf.get().setBoolean("test.ndfs.same.host.targets.allowed", true);
+ nutchConf.setBoolean("test.ndfs.same.host.targets.allowed", true);
}
/**
@@ -134,7 +135,7 @@
protected void prepareTempFileSpace() {
if (baseDir.exists()) {
try { // start from a blank slate
- FileUtil.fullyDelete(baseDir);
+ FileUtil.fullyDelete(baseDir, nutchConf);
} catch (Exception ignored) {
}
}
@@ -192,17 +193,17 @@
//
// set given config param to override other config settings
- NutchConf.get().setInt("test.ndfs.block_size", blockSize);
+ nutchConf.setInt("test.ndfs.block_size", blockSize);
// verify that config changed
- assertTrue(blockSize == NutchConf.get().getInt("test.ndfs.block_size", 2)); // 2 is an intentional obviously-wrong block size
+ assertTrue(blockSize == nutchConf.getInt("test.ndfs.block_size", 2)); // 2 is an intentional obviously-wrong block size
// downsize for testing (just to save resources)
- NutchConf.get().setInt("ndfs.namenode.handler.count", 3);
+ nutchConf.setInt("ndfs.namenode.handler.count", 3);
if (false) { // use MersenneTwister, if present
- NutchConf.get().set("nutch.random.class",
+ nutchConf.set("nutch.random.class",
"org.apache.nutch.util.MersenneTwister");
}
- NutchConf.get().setLong("ndfs.blockreport.intervalMsec", 50*1000L);
- NutchConf.get().setLong("ndfs.datanode.startupMsec", 15*1000L);
+ nutchConf.setLong("ndfs.blockreport.intervalMsec", 50*1000L);
+ nutchConf.setLong("ndfs.datanode.startupMsec", 15*1000L);
String nameFSDir = baseDirSpecified + "/name";
msg("----Start Test Cycle=" + currentTestCycleNumber +
@@ -216,20 +217,19 @@
int nameNodePort = 9000 + testCycleNumber++; // ToDo: settable base port
String nameNodeSocketAddr = "localhost:" + nameNodePort;
- NameNode nameNodeDaemon = new NameNode(new File(nameFSDir), nameNodePort);
+ NameNode nameNodeDaemon = new NameNode(new File(nameFSDir), nameNodePort, nutchConf);
NDFSClient ndfsClient = null;
try {
//
// start some DataNodes
//
ArrayList listOfDataNodeDaemons = new ArrayList();
- NutchConf conf = NutchConf.get();
- conf.set("fs.default.name", nameNodeSocketAddr);
+ nutchConf.set("fs.default.name", nameNodeSocketAddr);
for (int i = 0; i < initialDNcount; i++) {
// uniquely config real fs path for data storage for this datanode
String dataDir = baseDirSpecified + "/datanode" + i;
- conf.set("ndfs.data.dir", dataDir);
- DataNode dn = DataNode.makeInstanceForDir(dataDir, conf);
+ nutchConf.set("ndfs.data.dir", dataDir);
+ DataNode dn = DataNode.makeInstanceForDir(dataDir, nutchConf);
if (dn != null) {
listOfDataNodeDaemons.add(dn);
(new Thread(dn, "DataNode" + i + ": " + dataDir)).start();
@@ -244,7 +244,7 @@
awaitQuiescence();
// act as if namenode is a remote process
- ndfsClient = new NDFSClient(new InetSocketAddress("localhost", nameNodePort));
+ ndfsClient = new NDFSClient(new InetSocketAddress("localhost", nameNodePort), nutchConf);
//
// write nBytes of data using randomDataGenerator to numFiles
@@ -431,12 +431,12 @@
* testing (a default is set here if property is not set.)
*/
private Random makeRandomDataGenerator() {
- long seed = NutchConf.get().getLong("test.ndfs.random.seed", 0xB437EF);
+ long seed = nutchConf.getLong("test.ndfs.random.seed", 0xB437EF);
try {
if (randomDataGeneratorCtor == null) {
// lazy init
String rndDataGenClassname =
- NutchConf.get().get("nutch.random.class", "java.util.Random");
+ nutchConf.get("nutch.random.class", "java.util.Random");
Class clazz = Class.forName(rndDataGenClassname);
randomDataGeneratorCtor = clazz.getConstructor(new Class[]{Long.TYPE});
}
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/net/TestBasicUrlNormalizer.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/net/TestBasicUrlNormalizer.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/net/TestBasicUrlNormalizer.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/net/TestBasicUrlNormalizer.java Tue Jan 31 08:08:58 2006
@@ -17,6 +17,9 @@
package org.apache.nutch.net;
import java.net.URL;
+
+import org.apache.nutch.util.NutchConf;
+
import junit.framework.TestCase;
/** Unit tests for BasicUrlNormalizer. */
@@ -81,7 +84,7 @@
}
private void normalizeTest(String weird, String normal) throws Exception {
- assertEquals(normal, UrlNormalizerFactory.getNormalizer().normalize(weird));
+ assertEquals(normal, new UrlNormalizerFactory(new NutchConf()).getNormalizer().normalize(weird));
}
public static void main(String[] args) throws Exception {
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java Tue Jan 31 08:08:58 2006
@@ -18,6 +18,7 @@
import org.apache.nutch.parse.Outlink;
import org.apache.nutch.parse.OutlinkExtractor;
+import org.apache.nutch.util.NutchConf;
import junit.framework.TestCase;
@@ -30,14 +31,15 @@
*/
public class TestOutlinkExtractor extends TestCase {
+ private static NutchConf nutchConf = new NutchConf();
public void testGetNoOutlinks() {
Outlink[] outlinks = null;
- outlinks = OutlinkExtractor.getOutlinks(null);
+ outlinks = OutlinkExtractor.getOutlinks(null, nutchConf);
assertNotNull(outlinks);
assertEquals(0, outlinks.length);
- outlinks = OutlinkExtractor.getOutlinks("");
+ outlinks = OutlinkExtractor.getOutlinks("", nutchConf);
assertNotNull(outlinks);
assertEquals(0, outlinks.length);
}
@@ -46,7 +48,7 @@
Outlink[] outlinks = OutlinkExtractor.getOutlinks(
"Test with http://www.nutch.org/index.html is it found? " +
"What about www.google.com at http://www.google.de " +
- "A longer URL could be http://www.sybit.com/solutions/portals.html");
+ "A longer URL could be http://www.sybit.com/solutions/portals.html", nutchConf);
assertTrue("Url not found!", outlinks.length == 3);
assertEquals("Wrong URL", "http://www.nutch.org/index.html", outlinks[0].getToUrl());
@@ -58,7 +60,7 @@
Outlink[] outlinks = OutlinkExtractor.getOutlinks(
"Test with http://www.nutch.org/index.html is it found? " +
"What about www.google.com at http://www.google.de " +
- "A longer URL could be http://www.sybit.com/solutions/portals.html", "http://www.sybit.de");
+ "A longer URL could be http://www.sybit.com/solutions/portals.html", "http://www.sybit.de", nutchConf);
assertTrue("Url not found!", outlinks.length == 3);
assertEquals("Wrong URL", "http://www.nutch.org/index.html", outlinks[0].getToUrl());
@@ -68,7 +70,7 @@
public void testGetOutlinksFtp() {
Outlink[] outlinks = OutlinkExtractor.getOutlinks(
"Test with ftp://www.nutch.org is it found? " +
- "What about www.google.com at ftp://www.google.de");
+ "What about www.google.com at ftp://www.google.de", nutchConf);
assertTrue("Url not found!", outlinks.length >1);
assertEquals("Wrong URL", "ftp://www.nutch.org/", outlinks[0].getToUrl());
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/parse/TestParseData.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/parse/TestParseData.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/parse/TestParseData.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/parse/TestParseData.java Tue Jan 31 08:08:58 2006
@@ -18,11 +18,16 @@
import org.apache.nutch.io.*;
import org.apache.nutch.protocol.ContentProperties;
+import org.apache.nutch.util.NutchConf;
+
import junit.framework.TestCase;
/** Unit tests for ParseData. */
public class TestParseData extends TestCase {
+
+ private NutchConf nutchConf = new NutchConf();
+
public TestParseData(String name) { super(name); }
public void testParseData() throws Exception {
@@ -30,8 +35,8 @@
String title = "The Foo Page";
Outlink[] outlinks = new Outlink[] {
- new Outlink("http://foo.com/", "Foo"),
- new Outlink("http://bar.com/", "Bar")
+ new Outlink("http://foo.com/", "Foo", nutchConf),
+ new Outlink("http://bar.com/", "Bar", nutchConf)
};
ContentProperties metaData = new ContentProperties();
@@ -39,6 +44,7 @@
metaData.put("Charset", "UTF-8");
ParseData r = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks, metaData);
+ r.setConf(nutchConf);
TestWritable.testWritable(r);
}
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/parse/TestParserFactory.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/parse/TestParserFactory.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/parse/TestParserFactory.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/parse/TestParserFactory.java Tue Jan 31 08:08:58 2006
@@ -27,6 +27,7 @@
import java.io.File;
import java.net.MalformedURLException;
+import java.util.Arrays;
/**
@@ -37,69 +38,74 @@
*/
public class TestParserFactory extends TestCase {
-
+ private NutchConf nutchConf;
+ private ParserFactory parserFactory;
+
public TestParserFactory(String name) { super(name); }
/** Inits the Test Case with the test parse-plugin file */
protected void setUp() throws Exception {
- NutchConf.get().set("parse.plugin.file",
+ nutchConf = new NutchConf();
+ nutchConf.set("plugin.includes", ".*");
+ nutchConf.set("parse.plugin.file",
"org/apache/nutch/parse/parse-plugin-test.xml");
+ parserFactory = new ParserFactory(nutchConf);
}
/** Unit test for <code>getParser(String, String)</code> method. */
public void testGetParser() throws Exception {
- Parser parser = ParserFactory.getParser("text/html", "http://foo.com/");
+ Parser parser = parserFactory.getParser("text/html", "http://foo.com/");
assertNotNull(parser);
- parser = ParserFactory.getParser("foo/bar", "http://foo.com/");
+ parser = parserFactory.getParser("foo/bar", "http://foo.com/");
assertNotNull(parser);
}
/** Unit test for <code>getExtensions(String)</code> method. */
public void testGetExtensions() throws Exception {
- Extension ext = (Extension)ParserFactory.getExtensions("text/html").get(0);
+ Extension ext = (Extension)parserFactory.getExtensions("text/html").get(0);
assertEquals("parse-html", ext.getDescriptor().getPluginId());
- ext = (Extension) ParserFactory.getExtensions("text/html; charset=ISO-8859-1").get(0);
+ ext = (Extension) parserFactory.getExtensions("text/html; charset=ISO-8859-1").get(0);
assertEquals("parse-html", ext.getDescriptor().getPluginId());
- ext = (Extension)ParserFactory.getExtensions("foo/bar").get(0);
+ ext = (Extension)parserFactory.getExtensions("foo/bar").get(0);
assertEquals("parse-text", ext.getDescriptor().getPluginId());
}
/** Unit test to check <code>getParsers</code> method */
public void testGetParsers() throws Exception {
- Parser [] parsers = ParserFactory.getParsers("text/html", "http://foo.com");
+ Parser [] parsers = parserFactory.getParsers("text/html", "http://foo.com");
assertNotNull(parsers);
assertEquals(1, parsers.length);
assertEquals("org.apache.nutch.parse.html.HtmlParser",
parsers[0].getClass().getName());
- parsers = ParserFactory.getParsers("text/html; charset=ISO-8859-1", "http://foo.com");
+ parsers = parserFactory.getParsers("text/html; charset=ISO-8859-1", "http://foo.com");
assertNotNull(parsers);
assertEquals(1, parsers.length);
assertEquals("org.apache.nutch.parse.html.HtmlParser",
parsers[0].getClass().getName());
- parsers = ParserFactory.getParsers("application/x-javascript",
+ parsers = parserFactory.getParsers("application/x-javascript",
"http://foo.com");
assertNotNull(parsers);
assertEquals(1, parsers.length);
assertEquals("org.apache.nutch.parse.js.JSParseFilter",
parsers[0].getClass().getName());
- parsers = ParserFactory.getParsers("text/plain", "http://foo.com");
+ parsers = parserFactory.getParsers("text/plain", "http://foo.com");
assertNotNull(parsers);
assertEquals(1, parsers.length);
assertEquals("org.apache.nutch.parse.text.TextParser",
parsers[0].getClass().getName());
- Parser parser1 = ParserFactory.getParsers("text/plain", "http://foo.com")[0];
- Parser parser2 = ParserFactory.getParsers("*", "http://foo.com")[0];
+ Parser parser1 = parserFactory.getParsers("text/plain", "http://foo.com")[0];
+ Parser parser2 = parserFactory.getParsers("*", "http://foo.com")[0];
assertEquals("Different instances!", parser1.hashCode(), parser2.hashCode());
//test and make sure that the rss parser is loaded even though its plugin.xml
//doesn't claim to support text/rss, only application/rss+xml
- parsers = ParserFactory.getParsers("text/rss","http://foo.com");
+ parsers = parserFactory.getParsers("text/rss","http://foo.com");
assertNotNull(parsers);
assertEquals(1,parsers.length);
assertEquals("org.apache.nutch.parse.rss.RSSParser",parsers[0].getClass().getName());
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java Tue Jan 31 08:08:58 2006
@@ -38,10 +38,17 @@
private int fPluginCount;
private LinkedList fFolders = new LinkedList();
+ private NutchConf nutchConf ;
+ private PluginRepository repository;
protected void setUp() throws Exception {
+ this.nutchConf = new NutchConf();
+ nutchConf.set("plugin.includes", ".*");
+// String string = this.nutchConf.get("plugin.includes", "");
+// nutchConf.set("plugin.includes", string + "|Dummy*");
fPluginCount = 5;
createDummyPlugins(fPluginCount);
+ this.repository = nutchConf.getPluginRepository();
}
/*
@@ -72,7 +79,7 @@
/**
*/
public void testLoadPlugins() {
- PluginDescriptor[] descriptors = PluginRepository.getInstance()
+ PluginDescriptor[] descriptors = repository
.getPluginDescriptors();
int k = descriptors.length;
assertTrue(fPluginCount <= k);
@@ -91,10 +98,10 @@
*/
public void testGetExtensionAndAttributes() {
String xpId = " sdsdsd";
- ExtensionPoint extensionPoint = PluginRepository.getInstance()
+ ExtensionPoint extensionPoint =repository
.getExtensionPoint(xpId);
assertEquals(extensionPoint, null);
- Extension[] extension1 = PluginRepository.getInstance()
+ Extension[] extension1 = repository
.getExtensionPoint(getGetExtensionId()).getExtensions();
assertEquals(extension1.length, fPluginCount);
for (int i = 0; i < extension1.length; i++) {
@@ -108,7 +115,7 @@
* @throws PluginRuntimeException
*/
public void testGetExtensionInstances() throws PluginRuntimeException {
- Extension[] extensions = PluginRepository.getInstance()
+ Extension[] extensions = repository
.getExtensionPoint(getGetExtensionId()).getExtensions();
assertEquals(extensions.length, fPluginCount);
for (int i = 0; i < extensions.length; i++) {
@@ -127,7 +134,7 @@
*
*/
public void testGetClassLoader() {
- PluginDescriptor[] descriptors = PluginRepository.getInstance()
+ PluginDescriptor[] descriptors = repository
.getPluginDescriptors();
for (int i = 0; i < descriptors.length; i++) {
PluginDescriptor descriptor = descriptors[i];
@@ -139,7 +146,7 @@
* @throws IOException
*/
public void testGetResources() throws IOException {
- PluginDescriptor[] descriptors = PluginRepository.getInstance()
+ PluginDescriptor[] descriptors = repository
.getPluginDescriptors();
for (int i = 0; i < descriptors.length; i++) {
PluginDescriptor descriptor = descriptors[i];
@@ -159,12 +166,12 @@
* @return a PluginFolderPath
*/
private String getPluginFolder() {
- String[] strings = NutchConf.get().getStrings("plugin.folders");
+ String[] strings = nutchConf.getStrings("plugin.folders");
if (strings == null || strings.length == 0)
fail("no plugin directory setuped..");
String name = strings[0];
- return PluginManifestParser.getPluginFolder(name).toString();
+ return new PluginManifestParser(nutchConf, this.repository).getPluginFolder(name).toString();
}
/**
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/protocol/TestContent.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/protocol/TestContent.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/protocol/TestContent.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/protocol/TestContent.java Tue Jan 31 08:08:58 2006
@@ -17,11 +17,16 @@
package org.apache.nutch.protocol;
import org.apache.nutch.io.*;
+import org.apache.nutch.util.NutchConf;
+
import junit.framework.TestCase;
/** Unit tests for Content. */
public class TestContent extends TestCase {
+
+ private static NutchConf nutchConf = new NutchConf();
+
public TestContent(String name) { super(name); }
public void testContent() throws Exception {
@@ -35,7 +40,7 @@
metaData.put("Content-Type", "text/html");
Content r = new Content(url, url, page.getBytes("UTF8"), "text/html",
- metaData);
+ metaData, nutchConf);
TestWritable.testWritable(r);
assertEquals("text/html", r.getMetadata().get("Content-Type"));
@@ -50,49 +55,49 @@
c = new Content("http://www.foo.com/",
"http://www.foo.com/",
"".getBytes("UTF8"),
- "text/html; charset=UTF-8", p);
+ "text/html; charset=UTF-8", p, nutchConf);
assertEquals("text/html", c.getContentType());
c = new Content("http://www.foo.com/foo.html",
"http://www.foo.com/",
"".getBytes("UTF8"),
- "", p);
+ "", p, nutchConf);
assertEquals("text/html", c.getContentType());
c = new Content("http://www.foo.com/foo.html",
"http://www.foo.com/",
"".getBytes("UTF8"),
- null, p);
+ null, p, nutchConf);
assertEquals("text/html", c.getContentType());
c = new Content("http://www.foo.com/",
"http://www.foo.com/",
"<html></html>".getBytes("UTF8"),
- "", p);
+ "", p, nutchConf);
assertEquals("text/html", c.getContentType());
c = new Content("http://www.foo.com/foo.html",
"http://www.foo.com/",
"<html></html>".getBytes("UTF8"),
- "text/plain", p);
+ "text/plain", p, nutchConf);
assertEquals("text/html", c.getContentType());
c = new Content("http://www.foo.com/foo.png",
"http://www.foo.com/",
"<html></html>".getBytes("UTF8"),
- "text/plain", p);
+ "text/plain", p, nutchConf);
assertEquals("text/html", c.getContentType());
c = new Content("http://www.foo.com/",
"http://www.foo.com/",
"".getBytes("UTF8"),
- "", p);
+ "", p, nutchConf);
assertEquals("", c.getContentType());
c = new Content("http://www.foo.com/",
"http://www.foo.com/",
"".getBytes("UTF8"),
- null, p);
+ null, p, nutchConf);
assertNull(c.getContentType());
}
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/protocol/TestContentProperties.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/protocol/TestContentProperties.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/protocol/TestContentProperties.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/protocol/TestContentProperties.java Tue Jan 31 08:08:58 2006
@@ -17,6 +17,7 @@
package org.apache.nutch.protocol;
import org.apache.nutch.io.TestWritable;
+import org.apache.nutch.util.NutchConf;
import junit.framework.TestCase;
@@ -53,7 +54,7 @@
}
TestWritable.testWritable(properties);
Content content = new Content("url", "url", new byte[0], "text/html",
- new ContentProperties());
+ new ContentProperties(), new NutchConf());
ContentProperties metadata = content.getMetadata();
for (int i = 0; i < 100; i++) {
metadata.setProperty("aKey", "" + i);
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/searcher/TestQuery.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/test/org/apache/nutch/searcher/TestQuery.java?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/searcher/TestQuery.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/searcher/TestQuery.java Tue Jan 31 08:08:58 2006
@@ -20,36 +20,40 @@
import junit.framework.TestCase;
import java.util.Arrays;
import org.apache.nutch.analysis.NutchAnalysis;
+import org.apache.nutch.util.NutchConf;
public class TestQuery extends TestCase {
+
+ private static NutchConf nutchConf = new NutchConf();
+
public TestQuery(String name) { super(name); }
public void testRequiredTerm() throws Exception {
- Query query = new Query();
+ Query query = new Query(nutchConf);
query.addRequiredTerm("bobo");
testQuery(query, "bobo");
}
public void testProhibitedTerm() throws Exception {
- Query query = new Query();
+ Query query = new Query(nutchConf);
query.addProhibitedTerm("bobo");
testQuery(query, "-bobo");
}
public void testRequiredPhrase() throws Exception {
- Query query = new Query();
+ Query query = new Query(nutchConf);
query.addRequiredPhrase(new String[] {"bobo", "bogo"});
testQuery(query, "\"bobo bogo\"");
}
public void testProhibitedPhrase() throws Exception {
- Query query = new Query();
+ Query query = new Query(nutchConf);
query.addProhibitedPhrase(new String[] {"bobo", "bogo"});
testQuery(query, "-\"bobo bogo\"");
}
public void testComplex() throws Exception {
- Query query = new Query();
+ Query query = new Query(nutchConf);
query.addRequiredTerm("bobo");
query.addProhibitedTerm("bono");
query.addRequiredPhrase(new String[] {"bobo", "bogo"});
@@ -69,7 +73,7 @@
public static void testQueryParser(Query query, String string)
throws Exception {
- Query after = NutchAnalysis.parseQuery(string);
+ Query after = NutchAnalysis.parseQuery(string, nutchConf);
assertEquals(after, query);
assertEquals(after.toString(), string);
}
@@ -82,7 +86,7 @@
ByteArrayInputStream iBuf = new ByteArrayInputStream(oBuf.toByteArray());
DataInputStream in = new DataInputStream(iBuf);
- Query after = Query.read(in);
+ Query after = Query.read(in, nutchConf);
assertEquals(after, query);
}
@@ -95,7 +99,7 @@
public static void testQueryTerms(String query, String[] terms)
throws Exception {
- assertTrue(Arrays.equals(NutchAnalysis.parseQuery(query).getTerms(),
+ assertTrue(Arrays.equals(NutchAnalysis.parseQuery(query, nutchConf).getTerms(),
terms));
}
Modified: lucene/nutch/trunk/src/web/jsp/anchors.jsp
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/web/jsp/anchors.jsp?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/web/jsp/anchors.jsp (original)
+++ lucene/nutch/trunk/src/web/jsp/anchors.jsp Tue Jan 31 08:08:58 2006
@@ -9,8 +9,14 @@
import="org.apache.nutch.html.Entities"
import="org.apache.nutch.searcher.*"
+ import="org.apache.nutch.util.NutchConf"
%><%
- NutchBean bean = NutchBean.get(application);
+ NutchConf nutchConf = (NutchConf) application.getAttribute(NutchConf.class.getName());
+ if (nutchConf == null) {
+ nutchConf = new NutchConf();
+ application.setAttribute(NutchConf.class.getName(), nutchConf);
+ }
+ NutchBean bean = NutchBean.get(application, nutchConf);
// set the character encoding to use when interpreting request values
request.setCharacterEncoding("UTF-8");
bean.LOG.info("anchors request from " + request.getRemoteAddr());
Modified: lucene/nutch/trunk/src/web/jsp/cached.jsp
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/web/jsp/cached.jsp?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/web/jsp/cached.jsp (original)
+++ lucene/nutch/trunk/src/web/jsp/cached.jsp Tue Jan 31 08:08:58 2006
@@ -8,8 +8,14 @@
import="org.apache.nutch.searcher.*"
import="org.apache.nutch.parse.ParseData"
import="org.apache.nutch.protocol.ContentProperties"
+ import="org.apache.nutch.util.NutchConf"
%><%
- NutchBean bean = NutchBean.get(application);
+ NutchConf nutchConf = (NutchConf) application.getAttribute(NutchConf.class.getName());
+ if (nutchConf == null) {
+ nutchConf = new NutchConf();
+ application.setAttribute(NutchConf.class.getName(), nutchConf);
+ }
+ NutchBean bean = NutchBean.get(application, nutchConf);
bean.LOG.info("cache request from " + request.getRemoteAddr());
Hit hit = new Hit(Integer.parseInt(request.getParameter("idx")),
Integer.parseInt(request.getParameter("id")));
Modified: lucene/nutch/trunk/src/web/jsp/explain.jsp
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/web/jsp/explain.jsp?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/web/jsp/explain.jsp (original)
+++ lucene/nutch/trunk/src/web/jsp/explain.jsp Tue Jan 31 08:08:58 2006
@@ -7,15 +7,21 @@
import="java.io.*"
import="java.util.*"
import="org.apache.nutch.searcher.*"
+ import="org.apache.nutch.util.NutchConf"
%><%
- NutchBean bean = NutchBean.get(application);
+ NutchConf nutchConf = (NutchConf) application.getAttribute(NutchConf.class.getName());
+ if (nutchConf == null) {
+ nutchConf = new NutchConf();
+ application.setAttribute(NutchConf.class.getName(), nutchConf);
+ }
+ NutchBean bean = NutchBean.get(application, nutchConf);
// set the character encoding to use when interpreting request values
request.setCharacterEncoding("UTF-8");
bean.LOG.info("explain request from " + request.getRemoteAddr());
Hit hit = new Hit(Integer.parseInt(request.getParameter("idx")),
Integer.parseInt(request.getParameter("id")));
HitDetails details = bean.getDetails(hit);
- Query query = Query.parse(request.getParameter("query"));
+ Query query = Query.parse(request.getParameter("query"), nutchConf);
String language =
ResourceBundle.getBundle("org.nutch.jsp.explain", request.getLocale())
.getLocale().getLanguage();
Modified: lucene/nutch/trunk/src/web/jsp/refine-query-init.jsp
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/web/jsp/refine-query-init.jsp?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/web/jsp/refine-query-init.jsp (original)
+++ lucene/nutch/trunk/src/web/jsp/refine-query-init.jsp Tue Jan 31 08:08:58 2006
@@ -1,4 +1,4 @@
-<%!
+<%
// 20041129, Mike Pan and John Xing
// Initiates Ontology ontology and loads in all owl files.
@@ -6,16 +6,20 @@
// siliently ignored.
// Please check ./refine-query.jsp, which provides query-refinement hypertext.
-private static org.apache.nutch.ontology.Ontology ontology;
+org.apache.nutch.ontology.Ontology ontology = null;
// note: should we ignore plugin exceptions, or rethrow it below?
// Rethrowing it effectively prevents the servlet class from
// being loaded into the JVM. Need improvement in future.
-static {
try {
- String urls = org.apache.nutch.util.NutchConf.get().get("extension.ontology.urls");
- ontology = org.apache.nutch.ontology.OntologyFactory.getOntology();
+ org.apache.nutch.util.NutchConf nutchConf = (org.apache.nutch.util.NutchConf) application.getAttribute(org.apache.nutch.util.NutchConf.class.getName());
+ if (nutchConf == null) {
+ nutchConf = new org.apache.nutch.util.NutchConf();
+ application.setAttribute(org.apache.nutch.util.NutchConf.class.getName(), nutchConf);
+ }
+ String urls = nutchConf.get("extension.ontology.urls");
+ ontology = new org.apache.nutch.ontology.OntologyFactory(nutchConf).getOntology();
if (urls==null || urls.trim().equals("")) {
// ignored siliently
} else {
@@ -24,6 +28,5 @@
} catch (Exception e) {
// ignored siliently
}
-};
%>
Modified: lucene/nutch/trunk/src/web/jsp/search.jsp
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/web/jsp/search.jsp?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/web/jsp/search.jsp (original)
+++ lucene/nutch/trunk/src/web/jsp/search.jsp Tue Jan 31 08:08:58 2006
@@ -14,44 +14,43 @@
import="org.apache.nutch.clustering.*"
import="org.apache.nutch.util.NutchConf"
-%><%!
- // NOTE: Dawid Weiss
- // these are servlet's class variables. Merely to speed up access
- // to them (instead of referencing static objects).
-
+%><%
+ NutchConf nutchConf = (NutchConf) application.getAttribute(NutchConf.class.getName());
+ if (nutchConf == null) {
+ nutchConf = new NutchConf();
+ application.setAttribute(NutchConf.class.getName(), nutchConf);
+ }
+
/**
* Number of hits to retrieve and cluster if clustering extension is available
* and clustering is on. By default, 100. Configurable via nutch-conf.xml.
*/
- private final static int HITS_TO_CLUSTER =
- NutchConf.get().getInt("extension.clustering.hits-to-cluster", 100);
+ int HITS_TO_CLUSTER =
+ nutchConf.getInt("extension.clustering.hits-to-cluster", 100);
/**
* An instance of the clustering extension, if available.
*/
- private static OnlineClusterer clusterer;
-
- static {
- try {
- clusterer = OnlineClustererFactory.getOnlineClusterer();
- } catch (PluginRuntimeException e) {
- // NOTE: Dawid Weiss
- // should we ignore plugin exceptions, or rethrow it? Rethrowing
- // it effectively prevents the servlet class from being loaded into
- // the JVM
- }
+ OnlineClusterer clusterer = null;
+ try {
+ clusterer = new OnlineClustererFactory(nutchConf).getOnlineClusterer();
+ } catch (PluginRuntimeException e) {
+ // NOTE: Dawid Weiss
+ // should we ignore plugin exceptions, or rethrow it? Rethrowing
+ // it effectively prevents the servlet class from being loaded into
+ // the JVM
}
%>
<%--
// Uncomment this to enable query refinement.
-// Do the same to "refine-query.jsp" below.
+// Do the same to "refine-query.jsp" below.,
<%@ include file="./refine-query-init.jsp" %>
--%>
<%
- NutchBean bean = NutchBean.get(application);
+ NutchBean bean = NutchBean.get(application, nutchConf);
// set the character encoding to use when interpreting request values
request.setCharacterEncoding("UTF-8");
@@ -94,7 +93,7 @@
int hitsToCluster = HITS_TO_CLUSTER; // number of hits to cluster
- Query query = Query.parse(queryString);
+ Query query = Query.parse(queryString, nutchConf);
bean.LOG.info("query: " + queryString);
String language =
Modified: lucene/nutch/trunk/src/web/jsp/text.jsp
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/web/jsp/text.jsp?rev=373853&r1=373852&r2=373853&view=diff
==============================================================================
--- lucene/nutch/trunk/src/web/jsp/text.jsp (original)
+++ lucene/nutch/trunk/src/web/jsp/text.jsp Tue Jan 31 08:08:58 2006
@@ -7,12 +7,17 @@
import="org.apache.nutch.searcher.*"
import="org.apache.nutch.parse.ParseText"
+ import="org.apache.nutch.util.NutchConf"
%><%
// show the content of a hit as plain text
-
- NutchBean bean = NutchBean.get(application);
+ NutchConf nutchConf = (NutchConf) application.getAttribute(NutchConf.class.getName());
+ if (nutchConf == null) {
+ nutchConf = new NutchConf();
+ application.setAttribute(NutchConf.class.getName(), nutchConf);
+ }
+ NutchBean bean = NutchBean.get(application, nutchConf);
bean.LOG.info("text request from " + request.getRemoteAddr());