You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2016/10/18 14:56:01 UTC
svn commit: r1765450 - in /uima/uima-ducc/trunk:
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/
uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orch...
Author: degenaro
Date: Tue Oct 18 14:56:01 2016
New Revision: 1765450
URL: http://svn.apache.org/viewvc?rev=1765450&view=rev
Log:
UIMA-5060 DUCC Orchestrator (OR) "warm" restart issues
- for restoration of next job/reservation sequence number, use the greater of seqno in state/orchestrator.properties and historical/ckpt from database comprising jobs+reservations
- log a WARNing in or.log if historical/ckpt data is used
Added:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java (with props)
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IPropertiesFileManager.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/IDuccIdFactory.java
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java
Modified: uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IPropertiesFileManager.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IPropertiesFileManager.java?rev=1765450&r1=1765449&r2=1765450&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IPropertiesFileManager.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IPropertiesFileManager.java Tue Oct 18 14:56:01 2016
@@ -20,4 +20,6 @@ package org.apache.uima.ducc.common.pers
public interface IPropertiesFileManager {
public int increment(String key);
+ public String get(String key, String defaultValue);
+ public void set(String key, String value);
}
Modified: uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java?rev=1765450&r1=1765449&r2=1765450&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java Tue Oct 18 14:56:01 2016
@@ -62,4 +62,32 @@ public class DuccIdFactory implements ID
return new DuccId(seqno.get());
}
}
+
+ /**
+ * set seqno to candidate if it is bigger than previous, return previous value
+ */
+ public long setIfMax(long candidate) {
+ long previous = seqno.get();
+ synchronized(this) {
+ if(propertiesFileManager != null) {
+ try {
+ String propertiesFileValue = propertiesFileManager.get(propertiesFileKey, ""+previous);
+ previous = Long.parseLong(propertiesFileValue);
+ }
+ catch(Exception e) {
+ // No worries
+ }
+ if(candidate > previous) {
+ propertiesFileManager.set(propertiesFileKey, ""+candidate);
+ seqno.set(candidate);
+ }
+ }
+ else {
+ if(candidate > previous) {
+ seqno.set(candidate);
+ }
+ }
+ }
+ return previous;
+ }
}
Modified: uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/IDuccIdFactory.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/IDuccIdFactory.java?rev=1765450&r1=1765449&r2=1765450&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/IDuccIdFactory.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/IDuccIdFactory.java Tue Oct 18 14:56:01 2016
@@ -21,5 +21,5 @@ package org.apache.uima.ducc.common.util
public interface IDuccIdFactory {
public DuccId next();
-
+ public long setIfMax(long s);
}
Modified: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java?rev=1765450&r1=1765449&r2=1765450&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java Tue Oct 18 14:56:01 2016
@@ -46,6 +46,7 @@ public class OrchestratorCommonArea {
private static OrchestratorCommonArea orchestratorCommonArea = null;
private static final DuccLogger logger = DuccLoggerComponents.getOrLogger(OrchestratorCommonArea.class.getName());
+ private static final DuccId jobid = null;
private static DuccPropertiesResolver dpr = DuccPropertiesResolver.getInstance();
@@ -118,7 +119,8 @@ public class OrchestratorCommonArea {
ComponentHelper.oneInstance(IDuccEnv.DUCC_STATE_DIR,"orchestrator");
}
// </Jira 3414>
- setPropertiesFileManager(new PropertiesFileManager(IDuccLoggerComponents.abbrv_orchestrator, IDuccEnv.DUCC_STATE_DIR, constOrchestratorProperties, false, true));
+ PropertiesFileManager pfm = new PropertiesFileManager(IDuccLoggerComponents.abbrv_orchestrator, IDuccEnv.DUCC_STATE_DIR, constOrchestratorProperties, false, true);
+ setPropertiesFileManager(pfm);
initSeqNo();
setDuccIdFactory(new DuccIdFactory(propertiesFileManager,constSeqNo));
workMap = new DuccWorkMap();
@@ -134,6 +136,13 @@ public class OrchestratorCommonArea {
System.exit(1); // what should we do here? exit or acquire the NullHistoryManager?
}
logger.info(methodName, null, "Got history manager of class", historyPersistenceManager.getClass().getName());
+ //
+ OrchestratorRecovery orchestratorRecovery = new OrchestratorRecovery(historyPersistenceManager);
+ long historicSeqNo = orchestratorRecovery.recoverSeqNo();
+ long previousSeqNo = getDuccIdFactory().setIfMax(historicSeqNo);
+ if(previousSeqNo != historicSeqNo) {
+ logger.warn(methodName, jobid, "properties:"+previousSeqNo+" "+""+"historic:"+historicSeqNo);
+ }
}
public String getStateDirectory() {
Added: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java?rev=1765450&view=auto
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java (added)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java Tue Oct 18 14:56:01 2016
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.orchestrator;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
+import org.apache.uima.ducc.transport.event.common.history.IHistoryPersistenceManager;
+
+/**
+ * Purpose: scan job+reservation history+checkpoint data for the largest sequence number
+ */
+public class OrchestratorRecovery {
+
+ private static final DuccLogger logger = DuccLoggerComponents.getOrLogger(OrchestratorRecovery.class.getName());
+ private static final DuccId jobid = null;
+
+ IHistoryPersistenceManager historyPersistenceManager = null;
+
+ public OrchestratorRecovery(IHistoryPersistenceManager historyPersistenceManager) {
+ setHistoryPersistenceManager(historyPersistenceManager);
+ }
+
+ private void setHistoryPersistenceManager(IHistoryPersistenceManager value) {
+ historyPersistenceManager = value;
+ }
+
+ private Long findHistoryLastJob() {
+ String location = "findHistoryLastJob";
+ Long retVal = new Long(-1);
+ try {
+ List<IDuccWorkJob> duccWorkJobs = historyPersistenceManager.restoreJobs(1);
+ if(duccWorkJobs != null) {
+ if(!duccWorkJobs.isEmpty()) {
+ for(IDuccWorkJob job : duccWorkJobs) {
+ DuccId duccId = job.getDuccId();
+ retVal = duccId.getFriendly();
+ logger.info(location, jobid, "hist job "+retVal);
+ break;
+ }
+ }
+ else {
+ logger.debug(location, jobid, "hist job empty");
+ }
+ }
+ else {
+ logger.debug(location, jobid, "hist job not found");
+ }
+ }
+ catch(Exception e) {
+ logger.error(location, jobid, e);
+ }
+ return retVal;
+ }
+
+ private Long findHistoryLastReservation() {
+ String location = "findHistoryLastReservation";
+ Long retVal = new Long(-1);
+ try {
+ List<IDuccWorkReservation> duccWorkReservations = historyPersistenceManager.restoreReservations(1);
+ if(duccWorkReservations != null) {
+ if(!duccWorkReservations.isEmpty()) {
+ for(IDuccWorkReservation reservation : duccWorkReservations) {
+ DuccId duccId = reservation.getDuccId();
+ retVal = duccId.getFriendly();
+ logger.info(location, jobid, "hist reservation "+retVal);
+ break;
+ }
+ }
+ else {
+ logger.debug(location, jobid, "hist reservation empty");
+ }
+ }
+ else {
+ logger.debug(location, jobid, "hist reservation not found");
+ }
+ }
+ catch(Exception e) {
+ logger.error(location, jobid, e);
+ }
+ return retVal;
+ }
+
+ private Long findCheckpointLastJob() {
+ String location = "findCheckpointLastJob";
+ Long retVal = new Long(-1);
+ DuccWorkMap workMap = OrchestratorCommonArea.getInstance().getWorkMap();
+ try {
+ if(workMap != null) {
+ DuccId duccId = getMax(workMap.getJobKeySet());
+ if(duccId != null) {
+ retVal = duccId.getFriendly();
+ logger.info(location, jobid, "ckpt job "+retVal);
+ }
+ else {
+ logger.debug(location, jobid, "ckpt job not found");
+ }
+ }
+ else {
+ logger.debug(location, jobid, "workmap not found");
+ }
+ }
+ catch(Exception e) {
+ logger.error(location, jobid, e);
+ }
+ return retVal;
+ }
+
+ private Long findCheckpointLastReservation() {
+ String location = "findCheckpointLastReservation";
+ Long retVal = new Long(-1);
+ DuccWorkMap workMap = OrchestratorCommonArea.getInstance().getWorkMap();
+ try {
+ if(workMap != null) {
+ DuccId duccId = getMax(workMap.getReservationKeySet());
+ if(duccId != null) {
+ retVal = duccId.getFriendly();
+ logger.info(location, jobid, "ckpt reservation "+retVal);
+ }
+ else {
+ logger.debug(location, jobid, "ckpt reservation not found");
+ }
+ }
+ else {
+ logger.debug(location, jobid, "workmap not found");
+ }
+ }
+ catch(Exception e) {
+ logger.error(location, jobid, e);
+ }
+ return retVal;
+ }
+
+ private DuccId getMax(Set<DuccId> keys) {
+ DuccId retVal = null;
+ if(keys != null) {
+ for(DuccId key : keys) {
+ if(retVal == null) {
+ retVal = key;
+ }
+ else if(key.getFriendly() > retVal.getFriendly()) {
+ retVal = key;
+ }
+ }
+ }
+ return retVal;
+ }
+
+ private Long maxOf(Long v0, Long v1, String text) {
+ String location = "maxOf";
+ Long retVal = v0;
+ if(v1 > v0) {
+ retVal = v1+1;
+ logger.info(location, jobid, text);
+ }
+ return retVal;
+ }
+
+ public Long recoverSeqNo() {
+ String location = "recoverSeqNo";
+ Long seqNo = new Long(-1);
+ seqNo = maxOf(seqNo, findHistoryLastJob(), "history:job");
+ seqNo = maxOf(seqNo, findHistoryLastReservation(), "history:reservation");
+ seqNo = maxOf(seqNo, findCheckpointLastJob(), "checkpoint:job");
+ seqNo = maxOf(seqNo, findCheckpointLastReservation(), "checkpoint:reservation");
+ logger.info(location, jobid, seqNo);
+ return seqNo;
+ }
+}
Propchange: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java
------------------------------------------------------------------------------
svn:mime-type = text/plain