You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2015/03/31 17:16:13 UTC
svn commit: r1670364 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src:
main/java/org/apache/uima/ducc/container/jd/fsm/wi/
test/java/org/apache/uima/ducc/container/jd/test/
Author: degenaro
Date: Tue Mar 31 15:16:13 2015
New Revision: 1670364
URL: http://svn.apache.org/r1670364
Log:
UIMA-4318 DUCC Web Server (WS) reduce JD storage use
WiTracker unassign when JP vanishes
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/TestWiTracker.java (with props)
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionAckTimeout.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEndTimeout.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessFailure.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessPreempt.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessVolunteered.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionAckTimeout.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionAckTimeout.java?rev=1670364&r1=1670363&r2=1670364&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionAckTimeout.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionAckTimeout.java Tue Mar 31 15:16:13 2015
@@ -46,10 +46,11 @@ public class ActionAckTimeout extends Ac
return ActionAckTimeout.class.getName();
}
- private void preemptWorkItem(IActionData actionData, CasManager cm, IMetaCas metaCas) {
- String location = "preemptWorkItem";
+ private void recallWorkItem(IActionData actionData, CasManager cm, IMetaCas metaCas, IWorkItem wi) {
+ String location = "recallWorkItem";
MessageBuffer mb = LoggerHelper.getMessageBuffer(actionData);
logger.info(location, ILogger.null_id, mb.toString());
+ WiTracker.getInstance().unassign(wi);
TimeoutManager.getInstance().cancelTimer(actionData);
cm.putMetaCas(metaCas, RetryReason.TimeoutRetry);
cm.getCasManagerStats().incEndRetry();
@@ -70,8 +71,7 @@ public class ActionAckTimeout extends Ac
IRemoteWorkerProcess rwp = jdh.getRemoteWorkerProcess(wi);
if(rwp != null) {
if(metaCas != null) {
- WiTracker.getInstance().unassign(wi);
- preemptWorkItem(actionData, cm, metaCas);
+ recallWorkItem(actionData, cm, metaCas, wi);
IWorkItemStateKeeper wisk = jd.getWorkItemStateKeeper();
MetaCasHelper metaCasHelper = new MetaCasHelper(metaCas);
IProcessStatistics pStats = jdh.getProcessStatistics(rwp);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEndTimeout.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEndTimeout.java?rev=1670364&r1=1670363&r2=1670364&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEndTimeout.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEndTimeout.java Tue Mar 31 15:16:13 2015
@@ -46,10 +46,11 @@ public class ActionEndTimeout extends Ac
return ActionEndTimeout.class.getName();
}
- private void preemptWorkItem(IActionData actionData, CasManager cm, IMetaCas metaCas) {
- String location = "preemptWorkItem";
+ private void recallWorkItem(IActionData actionData, CasManager cm, IMetaCas metaCas, IWorkItem wi) {
+ String location = "recallWorkItem";
MessageBuffer mb = LoggerHelper.getMessageBuffer(actionData);
logger.info(location, ILogger.null_id, mb.toString());
+ WiTracker.getInstance().unassign(wi);
TimeoutManager.getInstance().cancelTimer(actionData);
cm.putMetaCas(metaCas, RetryReason.TimeoutRetry);
cm.getCasManagerStats().incEndRetry();
@@ -70,8 +71,7 @@ public class ActionEndTimeout extends Ac
IRemoteWorkerProcess rwp = jdh.getRemoteWorkerProcess(wi);
if(rwp != null) {
if(metaCas != null) {
- WiTracker.getInstance().unassign(wi);
- preemptWorkItem(actionData, cm, metaCas);
+ recallWorkItem(actionData, cm, metaCas, wi);
IWorkItemStateKeeper wisk = jd.getWorkItemStateKeeper();
MetaCasHelper metaCasHelper = new MetaCasHelper(metaCas);
IProcessStatistics pStats = jdh.getProcessStatistics(rwp);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessFailure.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessFailure.java?rev=1670364&r1=1670363&r2=1670364&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessFailure.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessFailure.java Tue Mar 31 15:16:13 2015
@@ -37,6 +37,7 @@ import org.apache.uima.ducc.container.jd
import org.apache.uima.ducc.container.jd.timeout.TimeoutManager;
import org.apache.uima.ducc.container.jd.wi.IProcessStatistics;
import org.apache.uima.ducc.container.jd.wi.IWorkItem;
+import org.apache.uima.ducc.container.jd.wi.WiTracker;
import org.apache.uima.ducc.container.net.iface.IMetaCas;
public class ActionProcessFailure extends Action implements IAction {
@@ -57,6 +58,7 @@ public class ActionProcessFailure extend
MessageBuffer mb = LoggerHelper.getMessageBuffer(actionData);
logger.info(location, ILogger.null_id, mb.toString());
//
+ WiTracker.getInstance().unassign(wi);
TimeoutManager.getInstance().cancelTimer(actionData);
cm.putMetaCas(metaCas, RetryReason.ProcessDown);
cm.getCasManagerStats().incEndRetry();
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessPreempt.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessPreempt.java?rev=1670364&r1=1670363&r2=1670364&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessPreempt.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessPreempt.java Tue Mar 31 15:16:13 2015
@@ -34,6 +34,7 @@ import org.apache.uima.ducc.container.jd
import org.apache.uima.ducc.container.jd.timeout.TimeoutManager;
import org.apache.uima.ducc.container.jd.wi.IProcessStatistics;
import org.apache.uima.ducc.container.jd.wi.IWorkItem;
+import org.apache.uima.ducc.container.jd.wi.WiTracker;
import org.apache.uima.ducc.container.net.iface.IMetaCas;
public class ActionProcessPreempt extends Action implements IAction {
@@ -45,10 +46,11 @@ public class ActionProcessPreempt extend
return ActionProcessPreempt.class.getName();
}
- private void preemptWorkItem(IActionData actionData, CasManager cm, IMetaCas metaCas) {
- String location = "preemptWorkItem";
+ private void recallWorkItem(IActionData actionData, CasManager cm, IMetaCas metaCas, IWorkItem wi) {
+ String location = "recallWorkItem";
MessageBuffer mb = LoggerHelper.getMessageBuffer(actionData);
logger.info(location, ILogger.null_id, mb.toString());
+ WiTracker.getInstance().unassign(wi);
TimeoutManager.getInstance().cancelTimer(actionData);
cm.putMetaCas(metaCas, RetryReason.ProcessPreempt);
cm.getCasManagerStats().incEndRetry();
@@ -69,7 +71,7 @@ public class ActionProcessPreempt extend
IRemoteWorkerProcess rwp = jdh.getRemoteWorkerProcess(wi);
if(rwp != null) {
if(metaCas != null) {
- preemptWorkItem(actionData, cm, metaCas);
+ recallWorkItem(actionData, cm, metaCas, wi);
IWorkItemStateKeeper wisk = jd.getWorkItemStateKeeper();
MetaCasHelper metaCasHelper = new MetaCasHelper(metaCas);
IProcessStatistics pStats = jdh.getProcessStatistics(rwp);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessVolunteered.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessVolunteered.java?rev=1670364&r1=1670363&r2=1670364&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessVolunteered.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionProcessVolunteered.java Tue Mar 31 15:16:13 2015
@@ -34,6 +34,7 @@ import org.apache.uima.ducc.container.jd
import org.apache.uima.ducc.container.jd.timeout.TimeoutManager;
import org.apache.uima.ducc.container.jd.wi.IProcessStatistics;
import org.apache.uima.ducc.container.jd.wi.IWorkItem;
+import org.apache.uima.ducc.container.jd.wi.WiTracker;
import org.apache.uima.ducc.container.net.iface.IMetaCas;
public class ActionProcessVolunteered extends Action implements IAction {
@@ -45,10 +46,11 @@ public class ActionProcessVolunteered ex
return ActionProcessVolunteered.class.getName();
}
- private void preemptWorkItem(IActionData actionData, CasManager cm, IMetaCas metaCas) {
- String location = "preemptWorkItem";
+ private void recallWorkItem(IActionData actionData, CasManager cm, IMetaCas metaCas, IWorkItem wi) {
+ String location = "recallWorkItem";
MessageBuffer mb = LoggerHelper.getMessageBuffer(actionData);
logger.info(location, ILogger.null_id, mb.toString());
+ WiTracker.getInstance().unassign(wi);
TimeoutManager.getInstance().cancelTimer(actionData);
cm.putMetaCas(metaCas, RetryReason.ProcessVolunteered);
cm.getCasManagerStats().incEndRetry();
@@ -69,7 +71,7 @@ public class ActionProcessVolunteered ex
IRemoteWorkerProcess rwp = jdh.getRemoteWorkerProcess(wi);
if(rwp != null) {
if(metaCas != null) {
- preemptWorkItem(actionData, cm, metaCas);
+ recallWorkItem(actionData, cm, metaCas, wi);
IWorkItemStateKeeper wisk = jd.getWorkItemStateKeeper();
MetaCasHelper metaCasHelper = new MetaCasHelper(metaCas);
IProcessStatistics pStats = jdh.getProcessStatistics(rwp);
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/TestWiTracker.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/TestWiTracker.java?rev=1670364&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/TestWiTracker.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/TestWiTracker.java Tue Mar 31 15:16:13 2015
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.container.jd.test;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.Random;
+
+import org.apache.uima.ducc.container.common.fsm.iface.IFsm;
+import org.apache.uima.ducc.container.jd.mh.RemoteWorkerThread;
+import org.apache.uima.ducc.container.jd.mh.iface.remote.IRemoteWorkerThread;
+import org.apache.uima.ducc.container.jd.wi.IWorkItem;
+import org.apache.uima.ducc.container.jd.wi.WiTracker;
+import org.apache.uima.ducc.container.jd.wi.WorkItem;
+import org.apache.uima.ducc.container.net.iface.IMetaCas;
+import org.apache.uima.ducc.container.net.impl.MetaCas;
+import org.junit.Test;
+
+public class TestWiTracker {
+
+ private static Random random = new Random();
+
+ private IFsm getFsm() {
+ IFsm fsm = null;
+ return fsm;
+ }
+
+ private IMetaCas getMetaCas(int seqNo) {
+ IMetaCas metaCas = null;
+ String text = ""+seqNo;
+ Object cas = null;
+ metaCas = new MetaCas(seqNo, text, cas);
+ return metaCas;
+ }
+
+ private IWorkItem getWi(int seqNo) {
+ IWorkItem wi = null;
+ IMetaCas metaCas = getMetaCas(seqNo);
+ IFsm fsm = getFsm();
+ wi = new WorkItem(metaCas, fsm);
+ return wi;
+ }
+
+ private IRemoteWorkerThread getRemoteWorkerThread() {
+ IRemoteWorkerThread rwt = null;
+ int node = random.nextInt(900)+100;
+ String nodeName = "node:"+node;
+ String nodeAddress = "address:"+node;
+ int pid = random.nextInt(30000)+10000;
+ String pidName = "pid:"+pid;
+ int tid = random.nextInt(90)+10;
+ rwt = new RemoteWorkerThread(nodeName, nodeAddress, pidName, pid, tid);
+ return rwt;
+ }
+
+ @Test
+ public void test() {
+ WiTracker tracker = WiTracker.getInstance();
+ int seqNo = 0;
+ //
+ seqNo = 1;
+ IWorkItem wi01A = getWi(seqNo);
+ IRemoteWorkerThread rwt01A = getRemoteWorkerThread();
+ tracker.assign(wi01A,rwt01A);
+ //
+ assertTrue(tracker.getSize() == 1);
+ //
+ IWorkItem wi01B = getWi(seqNo);
+ IRemoteWorkerThread rwt01B = getRemoteWorkerThread();
+ tracker.assign(wi01B,rwt01B);
+ //
+ assertTrue(tracker.getSize() == 1);
+ //
+ seqNo = 2;
+ IWorkItem wi02A = getWi(seqNo);
+ IRemoteWorkerThread rwt02A = getRemoteWorkerThread();
+ tracker.assign(wi02A,rwt02A);
+ //
+ assertTrue(tracker.getSize() == 2);
+ //
+ IWorkItem wi02B = getWi(seqNo);
+ IRemoteWorkerThread rwt02B = getRemoteWorkerThread();
+ tracker.assign(wi02B,rwt02B);
+ //
+ assertTrue(tracker.getSize() == 2);
+ }
+
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/TestWiTracker.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/TestWiTracker.java
------------------------------------------------------------------------------
svn:mime-type = text/plain