You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2013/04/24 19:38:36 UTC
svn commit: r1471555 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm:
ServiceHandler.java ServiceManagerComponent.java ServiceSet.java
Author: challngr
Date: Wed Apr 24 17:38:35 2013
New Revision: 1471555
URL: http://svn.apache.org/r1471555
Log:
UIMA-2833
Save service registrations and associated items to history.
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java?rev=1471555&r1=1471554&r2=1471555&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java Wed Apr 24 17:38:35 2013
@@ -1089,19 +1089,6 @@ public class ServiceHandler
}
sset.deleteProperties();
-
- // String metafn = sset.getMetaFilename();
- // String propsfn = sset.getPropsFilename();
-
- // if ( metafn != null ) {
- // File mf = new File(metafn);
- // mf.delete();
- // }
- // if ( propsfn != null ) {
- // File pf = new File(propsfn);
- // pf.delete();
- // }
-
}
String extractId(long friendly, String epname)
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java?rev=1471555&r1=1471554&r2=1471555&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java Wed Apr 24 17:38:35 2013
@@ -133,6 +133,10 @@ public class ServiceManagerComponent
if ( ! descdir.exists() ) {
descdir.mkdirs();
}
+ File histdir = new File(serviceHistoryLocation());
+ if ( ! histdir.exists() ) {
+ histdir.mkdirs();
+ }
String[] desclist = descdir.list();
for ( String d : desclist) {
if ( d.endsWith(".svc") ) {
@@ -151,18 +155,7 @@ public class ServiceManagerComponent
String sc = metaprops.getProperty("service-class");
if ( (sc != null) && ( sc.equals("Implicit") || sc.equals("Submitted") ) ) {
logger.info(methodName, null, "Scrubbing", sc, "service", stem);
- try {
- File mf = new File(meta_filename);
- mf.delete();
- } catch ( Throwable t ) {
- // nothing to do about it, ignore.
- }
- try {
- File pf = new File(props_filename);
- pf.delete();
- } catch ( Throwable t ) {
- // nothing to do about it, ignore.
- }
+ deleteProperties(stem, meta_filename, metaprops, props_filename, props);
continue;
}
@@ -566,11 +559,16 @@ public class ServiceManagerComponent
notify();
}
- private String serviceFileLocation()
+ static String serviceFileLocation()
{
return System.getProperty("DUCC_HOME") + "/state/services";
}
+ static String serviceHistoryLocation()
+ {
+ return System.getProperty("DUCC_HOME") + "/history/services-registry/";
+ }
+
private String serviceFileKey(String fn)
{
return serviceFileLocation() + "/" + fn;
@@ -722,14 +720,58 @@ public class ServiceManagerComponent
//ev.setReply(ServiceCode.OK, "Service not implemented.", "no-endpoint", null);
}
- public synchronized DuccId newId()
+ Object idSync = new Object();
+ public DuccId newId()
throws Exception
{
- DuccId id = idFactory.next();
- sm_props.setProperty(service_seqno, id.toString());
- FileOutputStream fos = new FileOutputStream(state_file);
- sm_props.store(fos, "Service Manager Properties");
- fos.close();
+ DuccId id = null;
+ synchronized(idSync) {
+ id = idFactory.next();
+ sm_props.setProperty(service_seqno, id.toString());
+ FileOutputStream fos = new FileOutputStream(state_file);
+ sm_props.store(fos, "Service Manager Properties");
+ fos.close();
+ }
return id;
}
+
+
+ static void deleteProperties(String id, String meta_filename, Properties meta_props, String props_filename, Properties job_props)
+ {
+ // NOTE: During init we may now know the ID as a DuccId so it has to be passed in as a string
+
+ String methodName = "deleteProperties";
+ // Save a copy in history, and then delete the original
+ String history_dir = serviceHistoryLocation();
+ if ( meta_filename != null ) {
+ File mfh = new File(history_dir + id + ".meta");
+ try {
+ FileOutputStream fos = new FileOutputStream(mfh);
+ meta_props.store(fos, "Archived meta descriptor");
+ fos.close();
+ } catch (Exception e) {
+ logger.warn(methodName, null, id + ": Unable to save history to \"" + mfh.toString(), ": ", e.toString() + "\"");
+ }
+
+ File mf = new File(meta_filename);
+ mf.delete();
+ }
+ meta_filename = null;
+
+ if ( props_filename != null ) {
+ File pfh = new File(history_dir + id + ".svc");
+ try {
+ FileOutputStream fos = new FileOutputStream(pfh);
+ job_props.store(fos, "Archived meta descriptor");
+ fos.close();
+ } catch (Exception e) {
+ logger.warn(methodName, null, id + ":Unable to save history to \"" + pfh.toString(), ": ", e.toString() + "\"");
+ }
+
+ File pf = new File(props_filename);
+ pf.delete();
+ }
+ props_filename = null;
+ }
+
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java?rev=1471555&r1=1471554&r2=1471555&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java Wed Apr 24 17:38:35 2013
@@ -19,7 +19,6 @@
package org.apache.uima.ducc.sm;
import java.io.BufferedReader;
-import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -75,6 +74,7 @@ public class ServiceSet
// For a registered service, here is my registered id
DuccId id;
HashMap<Long, DuccId> friendly_ids = new HashMap<Long, DuccId>();
+ String history_key = "work-instances";
// incoming nodes, for dup checking
List<ServiceSet> predecessors = new ArrayList<ServiceSet>();
@@ -312,17 +312,18 @@ public class ServiceSet
synchronized void deleteProperties()
{
- if ( meta_filename != null ) {
- File mf = new File(meta_filename);
- mf.delete();
- }
- meta_filename = null;
- if ( props_filename != null ) {
- File pf = new File(props_filename);
- pf.delete();
- }
- props_filename = null;
+ // be sure to move any services that seem not to have croaked yet to history
+ String history = meta_props.getStringProperty(history_key, "");
+ for ( Long id : friendly_ids.keySet() ) {
+ history = history + " " + id.toString();
+ }
+ meta_props.put(history_key, history);
+ meta_props.remove("implementors");
+
+ ServiceManagerComponent.deleteProperties(id.toString(), meta_filename, meta_props, props_filename, job_props);
+ meta_filename = null;
+ props_filename = null;
}
void setIncoming(ServiceSet sset)
@@ -422,6 +423,7 @@ public class ServiceSet
void synchronizeImplementors(Map<DuccId, JobState> work)
{
HashMap<Long, DuccId> newmap = new HashMap<Long, DuccId>();
+ // first loop synchronized 'friendly_ids' with live jobs comining in
for ( DuccId id : work.keySet() ) {
long fid = id.getFriendly();
@@ -432,6 +434,16 @@ public class ServiceSet
}
}
+ // second loop synchronizes history with jobs that used to be live and aren't any more (because of restart)
+ String history = meta_props.getStringProperty(history_key, "");
+ for ( Long friendly : friendly_ids.keySet() ) {
+ DuccId id = newmap.get(friendly);
+ if ( id == null ) {
+ history = history + " " + friendly;
+ }
+ }
+ meta_props.put(history_key, history);
+
friendly_ids = newmap; // replace persisted version with validated version from OR state
persistImplementors();
}
@@ -711,10 +723,15 @@ public class ServiceSet
public void removeImplementor(DuccId id)
{
+ String methodName = "removeImplementors";
if ( ! implementors.containsKey(id ) ) return; // quick short circuit if it's already gone
+ logger.debug(methodName, this.id, "Removing implementor", id);
implementors.remove(id);
friendly_ids.remove(id.getFriendly());
+ String history = meta_props.getStringProperty(history_key, "");
+ history = history + " " + id.toString();
+ meta_props.put(history_key, history);
persistImplementors();
if ( implementors.size() == 0 ) {
stopPingThread();