You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ke...@apache.org on 2014/04/03 00:47:27 UTC
[1/3] git commit: updated refs/heads/master to cd8801f
Repository: cloudstack
Updated Branches:
refs/heads/master e9c81c78b -> cd8801f6f
Don't let VMsync touch the VM if VM has been scheduled for HA but HA is not yet running.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/66bece59
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/66bece59
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/66bece59
Branch: refs/heads/master
Commit: 66bece5906c34002b42a26523614fe5e31b84fd2
Parents: e9c81c7
Author: Kelven Yang <ke...@gmail.com>
Authored: Fri Mar 21 10:58:14 2014 -0700
Committer: Kelven Yang <ke...@gmail.com>
Committed: Wed Apr 2 14:58:35 2014 -0700
----------------------------------------------------------------------
.../com/cloud/ha/HighAvailabilityManagerImpl.java | 2 +-
.../src/com/cloud/ha/dao/HighAvailabilityDao.java | 2 ++
.../com/cloud/ha/dao/HighAvailabilityDaoImpl.java | 17 +++++++++++++++++
3 files changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/66bece59/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
index 406b9b2..1318d9b 100755
--- a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
+++ b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
@@ -967,7 +967,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai
@Override
public boolean hasPendingHaWork(long vmId) {
- List<HaWorkVO> haWorks = _haDao.listRunningHaWorkForVm(vmId);
+ List<HaWorkVO> haWorks = _haDao.listPendingHaWorkForVm(vmId);
return haWorks.size() > 0;
}
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/66bece59/server/src/com/cloud/ha/dao/HighAvailabilityDao.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/ha/dao/HighAvailabilityDao.java b/server/src/com/cloud/ha/dao/HighAvailabilityDao.java
index 7f532f2..85135bb 100644
--- a/server/src/com/cloud/ha/dao/HighAvailabilityDao.java
+++ b/server/src/com/cloud/ha/dao/HighAvailabilityDao.java
@@ -81,4 +81,6 @@ public interface HighAvailabilityDao extends GenericDao<HaWorkVO, Long> {
* @return List of work items
*/
List<HaWorkVO> listRunningHaWorkForVm(long vmId);
+
+ List<HaWorkVO> listPendingHaWorkForVm(long vmId);
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/66bece59/server/src/com/cloud/ha/dao/HighAvailabilityDaoImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/ha/dao/HighAvailabilityDaoImpl.java b/server/src/com/cloud/ha/dao/HighAvailabilityDaoImpl.java
index c0da883..d25c6a7 100644
--- a/server/src/com/cloud/ha/dao/HighAvailabilityDaoImpl.java
+++ b/server/src/com/cloud/ha/dao/HighAvailabilityDaoImpl.java
@@ -49,6 +49,7 @@ public class HighAvailabilityDaoImpl extends GenericDaoBase<HaWorkVO, Long> impl
private final SearchBuilder<HaWorkVO> ReleaseSearch;
private final SearchBuilder<HaWorkVO> FutureHaWorkSearch;
private final SearchBuilder<HaWorkVO> RunningHaWorkSearch;
+ private final SearchBuilder<HaWorkVO> PendingHaWorkSearch;
protected HighAvailabilityDaoImpl() {
super();
@@ -106,6 +107,22 @@ public class HighAvailabilityDaoImpl extends GenericDaoBase<HaWorkVO, Long> impl
RunningHaWorkSearch.and("taken", RunningHaWorkSearch.entity().getDateTaken(), Op.NNULL);
RunningHaWorkSearch.and("step", RunningHaWorkSearch.entity().getStep(), Op.NIN);
RunningHaWorkSearch.done();
+
+ PendingHaWorkSearch = createSearchBuilder();
+ PendingHaWorkSearch.and("instance", PendingHaWorkSearch.entity().getInstanceId(), Op.EQ);
+ PendingHaWorkSearch.and("type", PendingHaWorkSearch.entity().getType(), Op.EQ);
+ PendingHaWorkSearch.and("step", PendingHaWorkSearch.entity().getStep(), Op.NIN);
+ PendingHaWorkSearch.done();
+ }
+
+ @Override
+ public List<HaWorkVO> listPendingHaWorkForVm(long vmId) {
+ SearchCriteria<HaWorkVO> sc = PendingHaWorkSearch.create();
+ sc.setParameters("instance", vmId);
+ sc.setParameters("type", WorkType.HA);
+ sc.setParameters("step", Step.Done, Step.Error, Step.Cancelled);
+
+ return search(sc, null);
}
@Override
[3/3] git commit: updated refs/heads/master to cd8801f
Posted by ke...@apache.org.
Do app-level iteration to avoid mysql deadlocks
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/cd8801f6
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/cd8801f6
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/cd8801f6
Branch: refs/heads/master
Commit: cd8801f6f7849e46d55f7880ff60f0d73c3a80e0
Parents: 66486d4
Author: Kelven Yang <ke...@gmail.com>
Authored: Wed Apr 2 15:46:50 2014 -0700
Committer: Kelven Yang <ke...@gmail.com>
Committed: Wed Apr 2 15:46:50 2014 -0700
----------------------------------------------------------------------
.../com/cloud/vm/VirtualMachineManagerImpl.java | 4 +-
.../framework/jobs/dao/VmWorkJobDaoImpl.java | 54 ++++++++++----------
2 files changed, 28 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/cd8801f6/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java
----------------------------------------------------------------------
diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java
index 6a2a5f8..c5a7efb 100755
--- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java
+++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java
@@ -1,4 +1,4 @@
-// Licensed to the Apache Software Foundation (ASF) under one
+// Licensed to the Apacohe Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
@@ -569,7 +569,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
@Override
public boolean start() {
// TODO, initial delay is hardcoded
- _executor.scheduleAtFixedRate(new CleanupTask(), 5000, VmJobStateReportInterval.value(), TimeUnit.SECONDS);
+ _executor.scheduleAtFixedRate(new CleanupTask(), 5, VmJobStateReportInterval.value(), TimeUnit.SECONDS);
_executor.scheduleAtFixedRate(new TransitionTask(), VmOpCleanupInterval.value(), VmOpCleanupInterval.value(), TimeUnit.SECONDS);
cancelWorkItems(_nodeId);
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/cd8801f6/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImpl.java
----------------------------------------------------------------------
diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImpl.java
index cf3e173..ee8748a 100644
--- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImpl.java
+++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImpl.java
@@ -20,9 +20,10 @@ import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Date;
import java.util.List;
-import java.util.TimeZone;
-
import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+
+import org.apache.log4j.Logger;
import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO;
import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO.Step;
@@ -41,9 +42,14 @@ import com.cloud.utils.db.TransactionStatus;
import com.cloud.vm.VirtualMachine;
public class VmWorkJobDaoImpl extends GenericDaoBase<VmWorkJobVO, Long> implements VmWorkJobDao {
+ private static final Logger s_logger = Logger.getLogger(VmWorkJobDaoImpl.class);
protected SearchBuilder<VmWorkJobVO> PendingWorkJobSearch;
protected SearchBuilder<VmWorkJobVO> PendingWorkJobByCommandSearch;
+ protected SearchBuilder<VmWorkJobVO> ExpungingWorkJobSearch;
+
+ @Inject
+ protected AsyncJobDao _baseJobDao;
public VmWorkJobDaoImpl() {
}
@@ -63,6 +69,12 @@ public class VmWorkJobDaoImpl extends GenericDaoBase<VmWorkJobVO, Long> implemen
PendingWorkJobByCommandSearch.and("step", PendingWorkJobByCommandSearch.entity().getStep(), Op.NEQ);
PendingWorkJobByCommandSearch.and("cmd", PendingWorkJobByCommandSearch.entity().getCmd(), Op.EQ);
PendingWorkJobByCommandSearch.done();
+
+ ExpungingWorkJobSearch = createSearchBuilder();
+ ExpungingWorkJobSearch.and("jobStatus", ExpungingWorkJobSearch.entity().getStatus(), Op.NEQ);
+ ExpungingWorkJobSearch.and("cutDate", ExpungingWorkJobSearch.entity().getLastUpdated(), Op.LT);
+ ExpungingWorkJobSearch.and("dispatcher", ExpungingWorkJobSearch.entity().getDispatcher(), Op.EQ);
+ ExpungingWorkJobSearch.done();
}
@Override
@@ -124,33 +136,19 @@ public class VmWorkJobDaoImpl extends GenericDaoBase<VmWorkJobVO, Long> implemen
expunge(sc);
*/
- Transaction.execute(new TransactionCallbackNoReturn() {
- @Override
- public void doInTransactionWithoutResult(TransactionStatus status) {
- TransactionLegacy txn = TransactionLegacy.currentTxn();
- PreparedStatement pstmt = null;
- try {
- pstmt = txn.prepareAutoCloseStatement(
- "DELETE FROM vm_work_job WHERE id IN (SELECT id FROM async_job WHERE job_dispatcher='VmWorkJobDispatcher' AND job_status != 0 AND last_updated < ?)");
- pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
-
- pstmt.execute();
- } catch (SQLException e) {
- } catch (Throwable e) {
- }
-
- try {
- pstmt = txn.prepareAutoCloseStatement(
- "DELETE FROM async_job WHERE job_dispatcher='VmWorkJobDispatcher' AND job_status != 0 AND last_updated < ?");
- pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
-
- pstmt.execute();
- } catch (SQLException e) {
- } catch (Throwable e) {
- }
- }
- });
+ // loop at application level to avoid mysql deadlock issues
+ SearchCriteria<VmWorkJobVO> sc = ExpungingWorkJobSearch.create();
+ sc.setParameters("jobStatus", JobInfo.Status.IN_PROGRESS);
+ sc.setParameters("lastUpdated", cutDate);
+ sc.setParameters("dispatcher", "VmWorkJobDispatcher");
+ List<VmWorkJobVO> expungeList = listBy(sc);
+ for (VmWorkJobVO job : expungeList) {
+ if (s_logger.isDebugEnabled())
+ s_logger.debug("Expunge completed work job-" + job.getId());
+ expunge(job.getId());
+ _baseJobDao.expunge(job.getId());
+ }
}
@Override
[2/3] git commit: updated refs/heads/master to cd8801f
Posted by ke...@apache.org.
BUG-ID: CS-19697: Deal with possible DB deadlock issue, return the job into queue for next turn
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/66486d43
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/66486d43
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/66486d43
Branch: refs/heads/master
Commit: 66486d4322121c2a8903ab94db60fb894c5ad149
Parents: 66bece5
Author: Kelven Yang <ke...@gmail.com>
Authored: Mon Mar 24 17:50:27 2014 -0700
Committer: Kelven Yang <ke...@gmail.com>
Committed: Wed Apr 2 14:59:04 2014 -0700
----------------------------------------------------------------------
.../framework/messagebus/MessageDetector.java | 4 ++-
.../jobs/impl/AsyncJobManagerImpl.java | 38 ++++++++++++++++----
2 files changed, 34 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/66486d43/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java
----------------------------------------------------------------------
diff --git a/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java b/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java
index 3fb620c..fae9bf3 100644
--- a/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java
+++ b/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java
@@ -31,8 +31,10 @@ public class MessageDetector implements MessageSubscriber {
}
public boolean waitAny(long timeoutInMiliseconds) {
- _signalled = false;
synchronized (this) {
+ if (_signalled)
+ return true;
+
try {
wait(timeoutInMiliseconds);
} catch (InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/66486d43/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
----------------------------------------------------------------------
diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
index 4acfdbf..42148be 100644
--- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
+++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
@@ -445,9 +445,9 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
CallContext.registerPlaceHolderContext();
if (job.getRelated() != null && !job.getRelated().isEmpty())
- NDC.push("Job-" + job.getRelated() + "/" + "Job-" + job.getId());
+ NDC.push("job-" + job.getRelated() + "/" + "job-" + job.getId());
else
- NDC.push("Job-" + job.getId());
+ NDC.push("job-" + job.getId());
try {
super.run();
} finally {
@@ -560,17 +560,41 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
job.setSyncSource(item);
- job.setExecutingMsid(getMsid());
- _jobDao.update(job.getId(), job);
+ //
+ // TODO: a temporary solution to work-around DB deadlock situation
+ //
+ // to live with DB deadlocks, we will give a chance for job to be rescheduled
+ // in case of exceptions (most-likely DB deadlock exceptions)
+ try {
+ job.setExecutingMsid(getMsid());
+ _jobDao.update(job.getId(), job);
+ } catch (Exception e) {
+ s_logger.warn("Unexpected exception while dispatching job-" + item.getContentId(), e);
+
+ try {
+ _queueMgr.returnItem(item.getId());
+ } catch (Throwable thr) {
+ s_logger.error("Unexpected exception while returning job-" + item.getContentId() + " to queue", thr);
+ }
+ }
try {
scheduleExecution(job);
} catch (RejectedExecutionException e) {
s_logger.warn("Execution for job-" + job.getId() + " is rejected, return it to the queue for next turn");
- _queueMgr.returnItem(item.getId());
- job.setExecutingMsid(null);
- _jobDao.update(job.getId(), job);
+ try {
+ _queueMgr.returnItem(item.getId());
+ } catch (Exception e2) {
+ s_logger.error("Unexpected exception while returning job-" + item.getContentId() + " to queue", e2);
+ }
+
+ try {
+ job.setExecutingMsid(null);
+ _jobDao.update(job.getId(), job);
+ } catch (Exception e3) {
+ s_logger.warn("Unexpected exception while update job-" + item.getContentId() + " msid for bookkeeping");
+ }
}
} else {