You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by si...@apache.org on 2005/04/07 21:53:14 UTC
svn commit: r160446 - in incubator/nutch/trunk: CHANGES.txt
src/java/org/apache/nutch/searcher/DistributedSearch.java
src/web/jsp/search.jsp
Author: siren
Date: Thu Apr 7 12:53:14 2005
New Revision: 160446
URL: http://svn.apache.org/viewcvs?view=rev&rev=160446
Log:
Added some features to DistributedSearch: new segments can be added
to searchservers without restarting the frontend, defective search
servers are not queried until tey come back online, watchdog keeps
an eye for your searchservers and writes simple statistics.
Modified:
incubator/nutch/trunk/CHANGES.txt
incubator/nutch/trunk/src/java/org/apache/nutch/searcher/DistributedSearch.java
incubator/nutch/trunk/src/web/jsp/search.jsp
Modified: incubator/nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewcvs/incubator/nutch/trunk/CHANGES.txt?view=diff&r1=160445&r2=160446
==============================================================================
--- incubator/nutch/trunk/CHANGES.txt (original)
+++ incubator/nutch/trunk/CHANGES.txt Thu Apr 7 12:53:14 2005
@@ -51,6 +51,11 @@
the root. Also fixed links to the about and help pages. Bug #32.
(Jerome Charron via cutting, 20050404)
+10. Added some features to DistributedSearch: new segments can be added
+ to searchservers without restarting the frontend, defective search
+ servers are not queried until tey come back online, watchdog keeps
+ an eye for your searchservers and writes simple statistics.
+ (Sami Siren, 20050407)
Release 0.6
Modified: incubator/nutch/trunk/src/java/org/apache/nutch/searcher/DistributedSearch.java
URL: http://svn.apache.org/viewcvs/incubator/nutch/trunk/src/java/org/apache/nutch/searcher/DistributedSearch.java?view=diff&r1=160445&r2=160446
==============================================================================
--- incubator/nutch/trunk/src/java/org/apache/nutch/searcher/DistributedSearch.java (original)
+++ incubator/nutch/trunk/src/java/org/apache/nutch/searcher/DistributedSearch.java Thu Apr 7 12:53:14 2005
@@ -25,8 +25,6 @@
import org.apache.nutch.parse.ParseText;
import org.apache.nutch.util.LogFormatter;
import org.apache.nutch.io.*;
-import org.apache.nutch.ipc.*;
-
/** Implements the search API over IPC connnections. */
public class DistributedSearch {
@@ -283,10 +281,21 @@
/** The search client. */
public static class Client extends org.apache.nutch.ipc.Client
- implements Searcher, HitDetailer, HitSummarizer, HitContent {
+ implements Searcher, HitDetailer, HitSummarizer, HitContent, Runnable {
- private InetSocketAddress[] addresses;
+ private InetSocketAddress[] addresses=new InetSocketAddress[0];
+ private InetSocketAddress[] defaultaddresses;
private HashMap segmentToAddress = new HashMap();
+
+ /**
+ * Flag for watchdog, true=keep running, false=stop
+ */
+ private boolean shouldrun=true;
+
+ /**
+ * Backgroudthread that polls search servers.
+ */
+ private Thread watchdog;
/** Construct a client talking to servers listed in the named file.
* Each line in the file lists a server hostname and port, separated by
@@ -320,29 +329,47 @@
/** Construct a client talking to the named servers. */
public Client(InetSocketAddress[] addresses) throws IOException {
super(Result.class);
+ this.defaultaddresses = addresses;
+ watchdog=new Thread(this);
+ watchdog.start();
+ }
+
+ /** Updates segments
+ *
+ * @throws IOException
+ */
+ public void updateSegments() throws IOException {
+
+ int statServers=0;
+ int statSegments=0;
+ Vector aliveaddresses=new Vector();
- this.addresses = addresses;
-
// build segmentToAddress map
Param param = new Param(OP_SEGMENTS, NullWritable.get());
- Writable[] params = new Writable[addresses.length];
+ Writable[] params = new Writable[defaultaddresses.length];
for (int i = 0; i < params.length; i++) {
params[i] = param; // build param for parallel call
}
- Writable[] results = call(params, addresses); // make parallel call
+ Writable[] results = call(params, defaultaddresses); // make parallel call
for (int i = 0; i < results.length; i++) { // process results of call
Result result = (Result)results[i];
if (result == null) {
- LOG.warning("Client: no segments from: " + addresses[i]);
+ LOG.warning("Client: no segments from: " + defaultaddresses[i]);
continue;
}
String[] segments = ((ArrayWritable)result.value).toStrings();
for (int j = 0; j < segments.length; j++) {
- LOG.info("Client: segment "+segments[j]+" at "+addresses[i]);
- segmentToAddress.put(segments[j], addresses[i]);
+ LOG.info("Client: segment "+segments[j]+" at "+defaultaddresses[i]);
+ segmentToAddress.put(segments[j], defaultaddresses[i]);
+ aliveaddresses.add(defaultaddresses[i]);
}
+ statServers++;
+ statSegments+=segments.length;
}
+
+ addresses=(InetSocketAddress[])aliveaddresses.toArray(new InetSocketAddress[statServers]);
+ LOG.info("STATS: " + statServers + " servers / " + statSegments + " segments online.");
}
/** Return the names of segments searched. */
@@ -504,7 +531,31 @@
}
-
+ public void run() {
+ while (shouldrun=true){
+ try{
+ LOG.info("Querying segments from search servers");
+ updateSegments();
+ } catch (IOException ioe) {
+ LOG.warning("No search servers available!");
+ addresses=new InetSocketAddress[0];
+ }
+ try{
+ Thread.sleep(10000);
+ } catch (InterruptedException ie){
+ LOG.info("Thread sleep interrupted.");
+ }
+ }
+ }
+
+ /**
+ * Stops the watchdog thread.
+ */
+ public void stop() {
+ super.stop();
+ LOG.info("stopping watchdog.");
+ shouldrun=false;
+ watchdog.interrupt();
+ }
}
-
}
Modified: incubator/nutch/trunk/src/web/jsp/search.jsp
URL: http://svn.apache.org/viewcvs/incubator/nutch/trunk/src/web/jsp/search.jsp?view=diff&r1=160445&r2=160446
==============================================================================
--- incubator/nutch/trunk/src/web/jsp/search.jsp (original)
+++ incubator/nutch/trunk/src/web/jsp/search.jsp Thu Apr 7 12:53:14 2005
@@ -150,7 +150,12 @@
// NOTE by Dawid Weiss:
// The 'clustering' window actually moves with the start
// position.... this is good, bad?... ugly?....
- Hits hits = bean.search(query, start + hitsToRetrieve, hitsPerSite);
+ Hits hits;
+ try{
+ hits = bean.search(query, start + hitsToRetrieve, hitsPerSite);
+ } catch (IOException e){
+ hits = new Hits(0,new Hit[0]);
+ }
int end = (int)Math.min(hits.getLength(), start + hitsPerPage);
int length = end-start;
int realEnd = (int)Math.min(hits.getLength(), start + hitsToRetrieve);