You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@labs.apache.org by th...@apache.org on 2008/02/25 12:56:36 UTC
svn commit: r630819 - in /labs/droids/trunk/src/core/java/org/apache/droids:
DefaultCrawler.java DefaultWorker.java api/Droid.java api/Worker.java
Author: thorsten
Date: Mon Feb 25 03:56:09 2008
New Revision: 630819
URL: http://svn.apache.org/viewvc?rev=630819&view=rev
Log:
Switching to thread ids generated by the thread.class - compare http://java.sun.com/j2se/1.5.0/docs/api/java/lang/Thread.html#getId() - to get rid of the home made id generation.
Modified:
labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java
labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java
labs/droids/trunk/src/core/java/org/apache/droids/api/Droid.java
labs/droids/trunk/src/core/java/org/apache/droids/api/Worker.java
Modified: labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java?rev=630819&r1=630818&r2=630819&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java Mon Feb 25 03:56:09 2008
@@ -36,13 +36,7 @@
protected int freeSlots;
- private ConcurrentHashMap<Integer, Worker> runningWorker;
-
- private int x = 0;
-
- private synchronized void increment() {
- x++;
- }
+ private ConcurrentHashMap<Long, Worker> runningWorker;
public void run() {
runningThreads = 0;
@@ -51,25 +45,25 @@
freeSlots = maxThreads - runningThreads;
pool = new ThreadPoolExecutor(freeSlots, freeSlots, 50000L,
TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>());
- runningWorker = new ConcurrentHashMap<Integer, Worker>();
+ runningWorker = new ConcurrentHashMap<Long, Worker>();
initQueue();
- startWorkers(0);
+ startWorkers();
Core.threadMessage("Finshed invocation, waiting for workers to finish.");
}
- private synchronized int startWorkers(int x) {
+ private synchronized long startWorkers() {
+ long id=0;
try {
- Worker worker = getWorker(x);
- runningWorker.put(x, worker);
+ Worker worker = getWorker();
+ id = worker.getId();
+ runningWorker.put(id, worker);
pool.execute(worker);
- increment();
- // try {
- Core.threadMessage("suspending");
+ Core.threadMessage("suspending "+id);
Thread.sleep(4000);
} catch (Exception e) {
Core.threadMessage(e.getMessage());
}
- return x;
+ return id;
}
public synchronized void initQueue() {
@@ -77,8 +71,8 @@
queue.init((Task[]) new Task[] { initialLink });
}
- public synchronized Worker getWorker(int x) {
- DefaultWorker worker = new DefaultWorker(x);
+ public synchronized Worker getWorker() {
+ DefaultWorker worker = new DefaultWorker();
worker.setQueue(getQueue());
worker.setDroid(this);
return worker;
@@ -159,7 +153,7 @@
this.url = url;
}
- public synchronized void finishedWorker(int id) {
+ public synchronized void finishedWorker(long id) {
Worker worker = runningWorker.get(id);
if (null != worker) {
int y = worker.getDepth() + 1;
@@ -173,7 +167,7 @@
for (int i = 0; i < (getMaxThreads() - pool.getActiveCount()); i++) {
if (!queue.hasNext())
break;
- startWorkers(y + i);
+ startWorkers();
}
}
}
Modified: labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java?rev=630819&r1=630818&r2=630819&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java Mon Feb 25 03:56:09 2008
@@ -20,7 +20,7 @@
import org.apache.droids.helper.factories.URLFiltersFactory;
import org.apache.droids.parse.Outlink;
-public class DefaultWorker implements Worker {
+public class DefaultWorker extends Thread implements Worker {
protected final Log log = LogFactory.getLog(this.getClass().getCanonicalName());
@@ -62,11 +62,12 @@
// parse contains the outlinks and can be used later
Parse parse = getParse();
handle(parse);
- droid.finishedWorker(id);
+ //Core.threadMessage("Trying to shut down "+getId());
+ droid.finishedWorker(getId());
} else {
Core.threadMessage("stopping processing since"
+ " bots are not allowed for this url.");
- droid.finishedWorker(id);
+ droid.finishedWorker(getId());
}
} catch (Exception e) {
e.printStackTrace();
@@ -128,10 +129,6 @@
parserFactory = droid.getCore().getParserFactory();
filtersFactory = droid.getCore().getFiltersFactory();
handlerFactory = droid.getCore().getHandlerFactory();
- }
-
- public void setId(int x) {
- id = x;
}
public int getDepth() {
Modified: labs/droids/trunk/src/core/java/org/apache/droids/api/Droid.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/api/Droid.java?rev=630819&r1=630818&r2=630819&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/api/Droid.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/api/Droid.java Mon Feb 25 03:56:09 2008
@@ -6,6 +6,6 @@
public abstract void run();
public abstract void initQueue();
public abstract void setQueue(Queue queue);
- public abstract void finishedWorker(int id);
+ public abstract void finishedWorker(long id);
public abstract Core getCore();
}
Modified: labs/droids/trunk/src/core/java/org/apache/droids/api/Worker.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/api/Worker.java?rev=630819&r1=630818&r2=630819&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/api/Worker.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/api/Worker.java Mon Feb 25 03:56:09 2008
@@ -4,7 +4,7 @@
public abstract void run();
public abstract void setQueue(Queue queue);
public abstract void setDroid(Droid droid);
- public abstract void setId(int x);
public abstract void setDepth(int x);
public abstract int getDepth();
+ public long getId() ;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@labs.apache.org
For additional commands, e-mail: commits-help@labs.apache.org