You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@labs.apache.org by th...@apache.org on 2008/05/15 14:22:57 UTC
svn commit: r656623 - in /labs/droids/trunk/src/core/java/org/apache/droids:
DefaultCrawler.java DefaultWorker.java
Author: thorsten
Date: Thu May 15 05:22:56 2008
New Revision: 656623
URL: http://svn.apache.org/viewvc?rev=656623&view=rev
Log:
Using getter and setter method everywhere instead of directly using the variables. This way it is save to override all methods of the default droid
Modified:
labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java
labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java
Modified: labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java?rev=656623&r1=656622&r2=656623&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java Thu May 15 05:22:56 2008
@@ -52,13 +52,13 @@
* in the Droid and their workers)
*/
public void run() {
- runningThreads = 0;
- taskDate = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date(System
- .currentTimeMillis()));
- freeSlots = maxThreads - runningThreads;
- pool = new ThreadPoolExecutor(freeSlots, freeSlots, 50000L,
- TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>());
- runningWorker = new ConcurrentHashMap<Long, Worker>();
+ setRunningThreads (0);
+ setTaskDate( new SimpleDateFormat("yyyyMMddHHmmss").format(new Date(System
+ .currentTimeMillis())));
+ setFreeSlots (maxThreads - runningThreads);
+ setPool( new ThreadPoolExecutor(freeSlots, freeSlots, 50000L,
+ TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>()));
+ setRunningWorker(new ConcurrentHashMap<Long, Worker>());
initQueue();
startWorkers();
Core.threadMessage("Finshed invocation, waiting for workers to finish.");
@@ -69,8 +69,8 @@
try {
Worker worker = getWorker();
id = worker.getId();
- runningWorker.put(id, worker);
- pool.execute(worker);
+ getRunningWorker().put(id, worker);
+ getPool().execute(worker);
Core.threadMessage("suspending "+id);
Thread.sleep(4000);
} catch (Exception e) {
@@ -84,7 +84,7 @@
*/
public synchronized void initQueue() {
QueueLink initialLink = new QueueLink(url, taskDate, 0);
- queue.init((Task[]) new Task[] { initialLink });
+ getQueue().init((Task[]) new Task[] { initialLink });
}
/**
@@ -151,19 +151,19 @@
protected synchronized void shutdownAndAwaitTermination() {
Core.threadMessage("SHUTTING DOWN");
- pool.shutdown(); // Disable new tasks from being submitted
+ getPool().shutdown(); // Disable new tasks from being submitted
try {
// Wait a while for existing tasks to terminate
- if (!pool.awaitTermination(1, TimeUnit.SECONDS)) {
+ if (!getPool().awaitTermination(1, TimeUnit.SECONDS)) {
Core.threadMessage("SHUT DOWN NOW");
- pool.shutdownNow(); // Cancel currently executing tasks
+ getPool().shutdownNow(); // Cancel currently executing tasks
// Wait a while for tasks to respond to being cancelled
- if (!pool.awaitTermination(1, TimeUnit.SECONDS))
+ if (!getPool().awaitTermination(1, TimeUnit.SECONDS))
Core.threadMessage("Pool did not terminate");
}
} catch (InterruptedException ie) {
// (Re-)Cancel if current thread also interrupted
- pool.shutdownNow();
+ getPool().shutdownNow();
// Preserve interrupt status
Thread.currentThread().interrupt();
}
@@ -180,18 +180,18 @@
}
public synchronized void finishedWorker(long id) {
- Worker worker = runningWorker.get(id);
+ Worker worker = getRunningWorker().get(id);
if (null != worker) {
int y = worker.getDepth() + 1;
- pool.remove(worker);
- runningWorker.remove(id);
+ getPool().remove(worker);
+ getRunningWorker().remove(id);
Core.threadMessage("Worker \"" + id + "\" has finished.");
- if (runningWorker.size() == 0 & !queue.hasNext()) {
+ if (getRunningWorker().size() == 0 & !getQueue().hasNext()) {
shutdownAndAwaitTermination();
Core.threadMessage("All threads has finished.");
- } else if (queue.hasNext()) {
- for (int i = 0; i < (getMaxThreads() - pool.getActiveCount()); i++) {
- if (!queue.hasNext())
+ } else if (getQueue().hasNext()) {
+ for (int i = 0; i < (getMaxThreads() - getPool().getActiveCount()); i++) {
+ if (!getQueue().hasNext())
break;
startWorkers();
}
@@ -207,4 +207,40 @@
this.core = core;
}
+ public ThreadPoolExecutor getPool() {
+ return pool;
+ }
+
+ public void setPool(ThreadPoolExecutor pool) {
+ this.pool = pool;
+ }
+
+ public ConcurrentHashMap<Long, Worker> getRunningWorker() {
+ return runningWorker;
+ }
+
+ public void setRunningWorker(ConcurrentHashMap<Long, Worker> runningWorker) {
+ this.runningWorker = runningWorker;
+ }
+
+ public void setRunningThreads(int runningThreads) {
+ this.runningThreads = runningThreads;
+ }
+
+ public String getTaskDate() {
+ return taskDate;
+ }
+
+ public void setTaskDate(String taskDate) {
+ this.taskDate = taskDate;
+ }
+
+ public int getFreeSlots() {
+ return freeSlots;
+ }
+
+ public void setFreeSlots(int freeSlots) {
+ this.freeSlots = freeSlots;
+ }
+
}
Modified: labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java?rev=656623&r1=656622&r2=656623&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java Thu May 15 05:22:56 2008
@@ -63,11 +63,11 @@
Parse parse = getParse();
handle(parse);
//Core.threadMessage("Trying to shut down "+getId());
- droid.finishedWorker(getId());
+ getDroid().finishedWorker(getId());
} else {
Core.threadMessage("stopping processing since"
+ " bots are not allowed for this url.");
- droid.finishedWorker(getId());
+ getDroid().finishedWorker(getId());
}
} catch (Exception e) {
e.printStackTrace();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@labs.apache.org
For additional commands, e-mail: commits-help@labs.apache.org