You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/12/07 21:41:23 UTC

svn commit: r1718459 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ src/main/resources/ uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ uima-ducc-rm/src/main/java/org/apache/uim...

Author: challngr
Date: Mon Dec  7 20:41:22 2015
New Revision: 1718459

URL: http://svn.apache.org/viewvc?rev=1718459&view=rev
Log:
UIMA-4577. Multiple minor tweaks. Fix RM vary/on-off with node death issue.  Change ducc.properties
           for db from ducc.state.database.url to simple ducc.database.url.  Fix automatic config
           of DB URL in cassandra.yaml.

Removed:
    uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ReadCkpt.java
Modified:
    uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
    uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml
    uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
    uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccMonitor.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbVerify.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Mon Dec  7 20:41:22 2015
@@ -205,7 +205,7 @@ class DuccUtil(DuccBase):
         if ( self.db_bypass == True ):
             return True
 
-        dbnode = self.ducc_properties.get('ducc.state.database.url')
+        dbnode = self.ducc_properties.get('ducc.database.host')
         if ( dbnode == None ):
             print 'No database location defined.'
             return False
@@ -235,7 +235,7 @@ class DuccUtil(DuccBase):
             return True
 
         print 'Starting database'
-        dbnode = self.ducc_properties.get('ducc.state.database.url')
+        dbnode = self.ducc_properties.get('ducc.database.host')
         dbu.update_cassandra_config(self.DUCC_HOME, dbnode)
 
         max_attempts = 5

Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/cassandra.yaml Mon Dec  7 20:41:22 2015
@@ -274,7 +274,7 @@ seed_provider:
       parameters:
           # seeds is actually a comma-delimited list of addresses.
           # Ex: "<ip1>,<ip2>,<ip3>"
-          - seeds: "bluej538"
+          - seeds: "DUCCHOST"
 
 # For workloads with more data than can fit in memory, Cassandra's
 # bottleneck will be reads that need to fetch data from
@@ -387,7 +387,7 @@ ssl_storage_port: 7001
 # you can specify which should be chosen using listen_interface_prefer_ipv6. If false the first ipv4
 # address will be used. If true the first ipv6 address will be used. Defaults to false preferring
 # ipv4. If there is only one address it will be selected regardless of ipv4/ipv6.
-listen_address: bluej538
+listen_address: DUCCHOST
 # listen_interface: eth0
 # listen_interface_prefer_ipv6: false
 
@@ -445,7 +445,7 @@ start_rpc: true
 # you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4
 # address will be used. If true the first ipv6 address will be used. Defaults to false preferring
 # ipv4. If there is only one address it will be selected regardless of ipv4/ipv6.
-rpc_address: bluej538
+rpc_address: DUCCHOST
 # rpc_interface: eth1
 # rpc_interface_prefer_ipv6: false
 

Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties Mon Dec  7 20:41:22 2015
@@ -510,13 +510,11 @@ ducc.flow-controller.specifier=org.apach
 
 
 # database
-# Directory where the databases and associated stuff is kept, relative to DUCC_HOME.  
-ducc.database.location        = database
 
 ducc.database.host            = --disabled--
 
 # Name of the database with active state. Svc registry, etc.
-ducc.state.database.url       = ${ducc.head}
+ducc.database.url             = ${ducc.head}
 
 # Database JMX host. Further Cassandra configuration (not recommended) is necessary to support other than "localhost".
 ducc.database.jmx.host = localhost

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccMonitor.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccMonitor.java?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccMonitor.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccMonitor.java Mon Dec  7 20:41:22 2015
@@ -205,6 +205,7 @@ public abstract class DuccMonitor {
 		}
 	}
 
+	@SuppressWarnings("unused")
 	private void error(String message) {
 		if (flag_error.get()) {
 			messageProcessor.status(timestamp(message));

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java Mon Dec  7 20:41:22 2015
@@ -98,7 +98,7 @@ public class DbCreate
                 session = cluster.connect();
                 session.execute("CREATE USER IF NOT EXISTS " + adminid + " with password '" + adminpw + "' SUPERUSER");
                 cluster.close();
-                doLog(methodName, "Created user " + adminid);                    
+                doLog(methodName, "Created database super user " + adminid);                    
                     
                 auth = new PlainTextAuthProvider(adminid, adminpw);
                 cluster = Cluster.builder()
@@ -109,7 +109,7 @@ public class DbCreate
                     
                 String uglypw = UUID.randomUUID().toString();
                 session.execute("ALTER USER cassandra  with password '" + uglypw + "' NOSUPERUSER");
-                doLog(methodName, "Changed default super user's password and revoked its superuser authority.");
+                doLog(methodName, "Changed default database super user's password and revoked its superuser authority.");
                 doLog(methodName, "From this point, this DB can only be accessed in super user mode by user 'ducc'");
                 break;
             } catch ( NoHostAvailableException e ) {
@@ -205,7 +205,7 @@ public class DbCreate
         session.execute("REVOKE SELECT ON KEYSPACE system FROM guest");
         session.execute("REVOKE SELECT ON KEYSPACE system_auth FROM guest");
         session.execute("REVOKE SELECT ON KEYSPACE system_traces FROM guest");
-        doLog(methodName, "Created user 'guest' with SELECT priveleges on DUCC tables.");
+        doLog(methodName, "Created database user 'guest' with SELECT priveleges on DUCC tables.");
                     
         session.execute("USE " + DUCC_KEYSPACE);
 

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java Mon Dec  7 20:41:22 2015
@@ -27,7 +27,6 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
-import java.util.UUID;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -136,7 +135,7 @@ public class DbLoader
         serviceRegistry        = from + serviceRegistry;
         checkpointFile         = from + checkpointFile;
 
-        System.setProperty("ducc.state.database.url", state_url);
+        System.setProperty(DbManager.URL_PROPERTY, state_url);
     }
 
     void closeStream(InputStream in)
@@ -530,13 +529,15 @@ public class DbLoader
     }
 
 
-    void run()
+    @SuppressWarnings("unused")
+	void run()
     	throws Exception
     {
     	String methodName = "run";
         long now = System.currentTimeMillis();
+        boolean run_test = false;
 
-        if ( false ) {
+        if ( run_test ) {
             try {
                 dbManager = new DbManager(state_url, logger);
                 dbManager.init();

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java Mon Dec  7 20:41:22 2015
@@ -42,7 +42,7 @@ import com.datastax.driver.core.policies
  */
 public class DbManager
 {
-
+    static final String URL_PROPERTY = "ducc.database.url";
     String dburl;
     DuccLogger logger;
 

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbVerify.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbVerify.java?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbVerify.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbVerify.java Mon Dec  7 20:41:22 2015
@@ -44,6 +44,9 @@ public class DbVerify
     DbManager dbManager = null;
     long total_bytes = 0;
 
+    boolean verify_res = false;
+    boolean verify_svc = false;
+    boolean verify_job = false;
     public DbVerify()
         throws Exception
     {
@@ -135,9 +138,9 @@ public class DbVerify
 
             verifyServices();
 
-            if ( false ) verify("ducc.res_history");
-            if ( false ) verify("ducc.svc_history");                
-            if ( false ) verify("ducc.job_history");
+            if ( verify_res ) verify("ducc.res_history");
+            if ( verify_svc ) verify("ducc.svc_history");                
+            if ( verify_job ) verify("ducc.job_history");
             
         } finally {
             dbManager.shutdown();

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java Mon Dec  7 20:41:22 2015
@@ -110,7 +110,7 @@ public class HistoryManagerDb
         throws Exception
     {
         this.logger = logger;
-        String historyUrl = System.getProperty("ducc.state.database.url");
+        String historyUrl = System.getProperty(DbManager.URL_PROPERTY);
         return init(historyUrl, null);
     }
 
@@ -119,7 +119,7 @@ public class HistoryManagerDb
     	throws Exception
     {
     	this.logger = logger;
-        String stateUrl = System.getProperty("ducc.state.database.url");
+        String stateUrl = System.getProperty(DbManager.URL_PROPERTY);
         return init(stateUrl, dbManager);
     }
 
@@ -220,7 +220,8 @@ public class HistoryManagerDb
     /**
      * Part of history management, recover ths indicated job from history.
      */
-    <T> T restoreWork(Class<T> cl, String tablename, long friendly_id)
+    @SuppressWarnings("unchecked")
+	<T> T restoreWork(Class<T> cl, String tablename, long friendly_id)
         throws Exception
     {
     	String methodName = "restoreWork";
@@ -249,7 +250,8 @@ public class HistoryManagerDb
      *
      * Reminder to self, we need to pass Clas<T> cl so compiler can infer T.
      */
-    public <T> ArrayList<T> restoreSeveralThings(Class<T> cl, String tablename, long max)
+    @SuppressWarnings("unchecked")
+	public <T> ArrayList<T> restoreSeveralThings(Class<T> cl, String tablename, long max)
         throws Exception
     {
     	String methodName = "restoreSeveralThings";
@@ -470,7 +472,8 @@ public class HistoryManagerDb
                 workbytes = bbmap.array();
                 bais = new ByteArrayInputStream(workbytes);
                 ois = new ObjectInputStream(bais);
-                Map<DuccId, DuccId> processToJob = (Map<DuccId, DuccId>) ois.readObject();
+                @SuppressWarnings("unchecked")
+				Map<DuccId, DuccId> processToJob = (Map<DuccId, DuccId>) ois.readObject();
                 ois.close();
 
                 // hack because java serializion is stupid and won't call the no-args constructor - need
@@ -488,7 +491,7 @@ public class HistoryManagerDb
                     logger.info(methodName, id, "Restored", w.getClass());
                 }
                 
-                ret = new Pair(work, processToJob);
+                ret = new Pair<DuccWorkMap, Map<DuccId, DuccId>>(work, processToJob);
             }
 
        } catch ( Exception e ) {

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java Mon Dec  7 20:41:22 2015
@@ -97,7 +97,7 @@ public class RmStatePersistence
     	throws Exception
     {
     	this.logger = logger;
-        String stateUrl = System.getProperty("ducc.state.database.url");
+        String stateUrl = System.getProperty(DbManager.URL_PROPERTY);
         init(stateUrl);
         DbHandle h = dbManager.open();
 
@@ -274,7 +274,7 @@ public class RmStatePersistence
     public Map<String, Map<String, Object>> getAllMachines()
     	throws Exception
     {
-    	String methodName = "getAllMachiens";
+    	//String methodName = "getAllMachines";
         Map<String, Map<String, Object>> ret = new HashMap<String, Map<String, Object>>();
         String cql = "SELECT * FROM " + RM_NODE_TABLE;
         DbHandle h = dbManager.open();

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java Mon Dec  7 20:41:22 2015
@@ -96,7 +96,7 @@ public class StateServicesDb
     	throws Exception
     {
     	this.logger = logger;
-        String stateUrl = System.getProperty("ducc.state.database.url");
+        String stateUrl = System.getProperty(DbManager.URL_PROPERTY);
         return init(stateUrl, null);
     }
 
@@ -105,7 +105,7 @@ public class StateServicesDb
     	throws Exception
     {
     	this.logger = logger;
-        String stateUrl = System.getProperty("ducc.state.database.url");
+        String stateUrl = System.getProperty(DbManager.URL_PROPERTY);
         return init(stateUrl, dbManager);
     }
 
@@ -154,9 +154,7 @@ public class StateServicesDb
             return ret;    // avoid NPE in caller
         }
 
-        DbHandle h = null;
         try {
-            h = dbManager.open();
             Map<Long, DuccProperties> svcset  = getProperties(SVC_TABLE, IStateServices.SvcRegProps.values(), isArchived);
             Map<Long, DuccProperties> metaset = getProperties(META_TABLE, IStateServices.SvcMetaProps.values(), isArchived);
             

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java Mon Dec  7 20:41:22 2015
@@ -174,11 +174,11 @@ public class Machine
         if ( heartbeats == 0 ) return;
         heartbeats = 0;
         try {
-            logger.info(methodName, null, "Reset heartbeat to 0 from", heartbeats);
+            logger.info(methodName, null, id, "Reset heartbeat to 0 from", heartbeats);
 			persistence.setNodeProperty(id, RmNodes.Heartbeats, 0);
-            logger.info(methodName, null, "Time to reset heartbeat", System.currentTimeMillis() - now);
+            logger.info(methodName, null, id, "Time to reset heartbeat", System.currentTimeMillis() - now);
 		} catch (Exception e) {
-            logger.warn(methodName, null, "Cannot update heartbeat count in database:", e);
+            logger.warn(methodName, null, id, "Cannot update heartbeat count in database:", e);
 		}
     }
 
@@ -190,11 +190,11 @@ public class Machine
         if ( c < 2 ) return;                    // we allow a couple because timing and races can create false negatives
         heartbeats = c;
         try {
-            logger.info(methodName, null, "Missed heartbeat count", c);
+            logger.info(methodName, null, id, "Missed heartbeat count", c);
 			persistence.setNodeProperty(id, RmNodes.Heartbeats, c);
-            logger.info(methodName, null, "Time to record misssed heartbeat", System.currentTimeMillis() - now);
+            logger.info(methodName, null, id, "Time to record misssed heartbeat", System.currentTimeMillis() - now);
 		} catch (Exception e) {
-            logger.warn(methodName, null, "Cannot update heartbeat count in database:", e);
+            logger.warn(methodName, null, id, "Cannot update heartbeat count in database:", e);
 		}
     }
 

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java Mon Dec  7 20:41:22 2015
@@ -540,6 +540,14 @@ class NodePool
     public Machine getMachine(Node n)
     {
         Machine m = allMachines.get(n);
+
+        if ( m == null ) {
+            m = unresponsiveMachines.get(n);
+        }
+        if ( m == null ) {
+            m = offlineMachines.get(n);
+        }
+
         if ( m == null ) {
             for ( NodePool np : children.values() ) {
                 m = np.getMachine(n);
@@ -1061,7 +1069,6 @@ class NodePool
 
         if ( offlineMachines.containsKey(node) ) {               // if it's offline it can't be restored like this.
             Machine m = offlineMachines.get(node);
-            signalDb(m, RmNodes.Responsive, true);
             logger.trace(methodName, null, "Node ", m.getId(), " is offline, not activating.");
             return m;
         }
@@ -1127,7 +1134,7 @@ class NodePool
 
     void disable(Machine m, HashMap<Node, Machine> disableMap)
     {
-        String methodName = "nodeLeaves";
+        String methodName = "disable";
 
         if ( allMachines.containsKey(m.key()) ) {
             logger.info(methodName, null, "Nodepool:", id, "Host disabled:", m.getId(), "Looking for shares to clear");

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java?rev=1718459&r1=1718458&r2=1718459&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java Mon Dec  7 20:41:22 2015
@@ -825,6 +825,12 @@ public class Scheduler
         synchronized(this) {
             for ( Node n : nodeUpdates.keySet() ) {
                 Machine m = getMachine(n);
+
+                if ( m == null ) {
+                    logger.warn(methodName, null, "Cannot find any record of machine", n.getNodeIdentity().getName());
+                    continue;
+                }
+
                 int count = nodeUpdates.get(n);
                 if ( count == 0 ) {
                     m.heartbeatArrives();