You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@batchee.apache.org by rm...@apache.org on 2013/11/05 08:39:00 UTC
[23/62] importing batchee from github - a fork from the IBm RI
http://git-wip-us.apache.org/repos/asf/incubator-batchee/blob/efa64877/jbatch/src/main/java/org/apache/batchee/container/services/persistence/JPAPersistenceService.java
----------------------------------------------------------------------
diff --git a/jbatch/src/main/java/org/apache/batchee/container/services/persistence/JPAPersistenceService.java b/jbatch/src/main/java/org/apache/batchee/container/services/persistence/JPAPersistenceService.java
new file mode 100644
index 0000000..8c0f6f0
--- /dev/null
+++ b/jbatch/src/main/java/org/apache/batchee/container/services/persistence/JPAPersistenceService.java
@@ -0,0 +1,847 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.batchee.container.services.persistence;
+
+import org.apache.batchee.container.exception.BatchContainerRuntimeException;
+import org.apache.batchee.container.exception.PersistenceException;
+import org.apache.batchee.container.impl.JobExecutionImpl;
+import org.apache.batchee.container.impl.JobInstanceImpl;
+import org.apache.batchee.container.impl.StepContextImpl;
+import org.apache.batchee.container.impl.StepExecutionImpl;
+import org.apache.batchee.container.impl.controller.PartitionedStepBuilder;
+import org.apache.batchee.container.impl.controller.chunk.CheckpointData;
+import org.apache.batchee.container.impl.controller.chunk.CheckpointDataKey;
+import org.apache.batchee.container.impl.controller.chunk.PersistentDataWrapper;
+import org.apache.batchee.container.impl.jobinstance.RuntimeFlowInSplitExecution;
+import org.apache.batchee.container.impl.jobinstance.RuntimeJobExecution;
+import org.apache.batchee.container.services.InternalJobExecution;
+import org.apache.batchee.container.services.persistence.jpa.EntityManagerProvider;
+import org.apache.batchee.container.services.persistence.jpa.TransactionProvider;
+import org.apache.batchee.container.services.persistence.jpa.domain.CheckpointEntity;
+import org.apache.batchee.container.services.persistence.jpa.domain.JobExecutionEntity;
+import org.apache.batchee.container.services.persistence.jpa.domain.JobInstanceEntity;
+import org.apache.batchee.container.services.persistence.jpa.domain.StepExecutionEntity;
+import org.apache.batchee.container.services.persistence.jpa.provider.DefaultEntityManagerProvider;
+import org.apache.batchee.container.services.persistence.jpa.provider.DefaultTransactionProvider;
+import org.apache.batchee.container.status.JobStatus;
+import org.apache.batchee.container.status.StepStatus;
+import org.apache.batchee.spi.PersistenceManagerService;
+
+import javax.batch.operations.NoSuchJobExecutionException;
+import javax.batch.runtime.BatchStatus;
+import javax.batch.runtime.JobInstance;
+import javax.batch.runtime.Metric;
+import javax.batch.runtime.StepExecution;
+import javax.persistence.EntityManager;
+import javax.persistence.NoResultException;
+import javax.persistence.TypedQuery;
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import static org.apache.batchee.container.util.Serializations.deserialize;
+import static org.apache.batchee.container.util.Serializations.serialize;
+
+public class JPAPersistenceService implements PersistenceManagerService {
+ private static final String[] DELETE_QUERIES = {
+ StepExecutionEntity.Queries.DELETE_BY_INSTANCE_ID, CheckpointEntity.Queries.DELETE_BY_INSTANCE_ID,
+ JobExecutionEntity.Queries.DELETE_BY_INSTANCE_ID, JobInstanceEntity.Queries.DELETE_BY_INSTANCE_ID
+ };
+
+ private EntityManagerProvider emProvider;
+ private TransactionProvider txProvider;
+
+ @Override
+ public void cleanUp(final long instanceId) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final Object tx = txProvider.start(em);
+ try {
+ for (final String query : DELETE_QUERIES) {
+ em.createQuery(query).setParameter("instanceId", instanceId).executeUpdate();
+ }
+ txProvider.commit(tx);
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public StepStatus getStepStatus(final long instanceId, final String stepName) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final List<StepExecutionEntity> list = em.createNamedQuery(StepExecutionEntity.Queries.FIND_BY_INSTANCE_AND_NAME, StepExecutionEntity.class)
+ .setParameter("instanceId", instanceId)
+ .setParameter("step", stepName)
+ .getResultList();
+ if (list != null && !list.isEmpty()) {
+ final StepExecutionEntity entity = list.iterator().next();
+ final StepStatus status = new StepStatus(entity.getId(), entity.getStartCount());
+ status.setBatchStatus(entity.getBatchStatus());
+ status.setExitStatus(entity.getExitStatus());
+ status.setNumPartitions(entity.getNumPartitions());
+ status.setLastRunStepExecutionId(entity.getLastRunStepExecutionId());
+ if (entity.getPersistentData() != null) {
+ status.setPersistentUserData(new PersistentDataWrapper(entity.getPersistentData()));
+ }
+ return status;
+ }
+ return null;
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public Map<Long, String> jobOperatorGetExternalJobInstanceData() {
+ final Map<Long, String> data = new HashMap<Long, String>();
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final List<JobInstanceEntity> list = em.createNamedQuery(JobInstanceEntity.Queries.FIND_EXTERNALS, JobInstanceEntity.class)
+ .setParameter("pattern", PartitionedStepBuilder.JOB_ID_SEPARATOR)
+ .getResultList();
+ if (list != null) {
+ for (final JobInstanceEntity elt : list) {
+ data.put(elt.getJobInstanceId(), elt.getName());
+ }
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ return data;
+ }
+
+ @Override
+ public JobStatus getJobStatusFromExecution(final long executionId) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final JobInstanceEntity entity = em.createNamedQuery(JobInstanceEntity.Queries.FIND_FROM_EXECUTION, JobInstanceEntity.class)
+ .setParameter("executionId", executionId)
+ .getSingleResult();
+ final JobStatus status = new JobStatus(entity.getJobInstanceId());
+ setJobStatusData(status, entity);
+ return status;
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public StepExecution getStepExecutionByStepExecutionId(final long stepExecId) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final StepExecutionEntity entity = em.find(StepExecutionEntity.class, stepExecId);
+ final StepExecutionImpl stepEx = new StepExecutionImpl(entity.getExecution().getExecutionId(), stepExecId);
+ setStepExecutionData(entity, stepEx);
+ return stepEx;
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ private void setStepExecutionData(final StepExecutionEntity entity, final StepExecutionImpl stepEx) {
+ stepEx.setBatchStatus(entity.getBatchStatus());
+ stepEx.setExitStatus(entity.getExitStatus());
+ stepEx.setStepName(entity.getStepName());
+ stepEx.setReadCount(entity.getRead());
+ stepEx.setWriteCount(entity.getWrite());
+ stepEx.setCommitCount(entity.getCommit());
+ stepEx.setRollbackCount(entity.getRollback());
+ stepEx.setReadSkipCount(entity.getReadSkip());
+ stepEx.setProcessSkipCount(entity.getProcessSkip());
+ stepEx.setFilterCount(entity.getFilter());
+ stepEx.setWriteSkipCount(entity.getWriteSkip());
+ stepEx.setStartTime(entity.getStartTime());
+ stepEx.setEndTime(entity.getEndTime());
+ try {
+ stepEx.setPersistentUserData(deserialize(entity.getPersistentData()));
+ } catch (final Exception e) {
+ throw new PersistenceException(e);
+ }
+ }
+
+ @Override
+ public List<StepExecution> getStepExecutionsForJobExecution(final long execid) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final List<StepExecutionEntity> steps = em.createNamedQuery(StepExecutionEntity.Queries.FIND_BY_EXECUTION, StepExecutionEntity.class)
+ .setParameter("executionId", execid)
+ .getResultList();
+
+ if (steps == null) {
+ return Collections.emptyList();
+ }
+
+ final List<StepExecution> executions = new ArrayList<StepExecution>(steps.size());
+ for (final StepExecutionEntity entity : steps) {
+ final StepExecutionImpl stepEx = new StepExecutionImpl(execid, entity.getId());
+ setStepExecutionData(entity, stepEx);
+ executions.add(stepEx);
+ }
+
+ return executions;
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public void updateStepExecution(final long jobExecId, final StepContextImpl stepContext) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final Object tx = txProvider.start(em);
+ try {
+ final StepExecutionEntity entity = em.find(StepExecutionEntity.class, stepContext.getStepExecutionId());
+ setStepData(em, jobExecId, stepContext,
+ stepContext.metricsAsMap().get(Metric.MetricType.READ_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.WRITE_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.COMMIT_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.ROLLBACK_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.READ_SKIP_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.PROCESS_SKIP_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.FILTER_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.WRITE_SKIP_COUNT.name()).getValue(),
+ entity);
+ txProvider.commit(tx);
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public StepExecutionImpl createStepExecution(final long jobExecId, final StepContextImpl stepContext) {
+ final StepExecutionEntity entity = new StepExecutionEntity();
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ setStepData(em, jobExecId, stepContext,
+ stepContext.metricsAsMap().get(Metric.MetricType.READ_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.WRITE_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.COMMIT_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.ROLLBACK_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.READ_SKIP_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.PROCESS_SKIP_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.FILTER_COUNT.name()).getValue(),
+ stepContext.metricsAsMap().get(Metric.MetricType.WRITE_SKIP_COUNT.name()).getValue(),
+ entity);
+
+ final Object tx = txProvider.start(em);
+ try {
+ em.persist(entity);
+ txProvider.commit(tx);
+
+ final StepExecutionImpl stepExecution = new StepExecutionImpl(jobExecId, entity.getId());
+ stepExecution.setStepName(stepContext.getStepName());
+ return stepExecution;
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ private void setStepData(final EntityManager em,
+ final long jobExecId, final StepContextImpl stepContext,
+ final long readCount, final long writeCount,
+ final long commitCount, final long rollbackCount,
+ final long readSkipCount, final long processSkipCount,
+ final long filterCount, final long writeSkipCount,
+ final StepExecutionEntity entity) {
+ entity.setExecution(em.find(JobExecutionEntity.class, jobExecId));
+ entity.setBatchStatus(stepContext.getBatchStatus());
+ entity.setExitStatus(stepContext.getExitStatus());
+ entity.setStepName(stepContext.getStepName());
+ entity.setRead(readCount);
+ entity.setWrite(writeCount);
+ entity.setCommit(commitCount);
+ entity.setRollback(rollbackCount);
+ entity.setReadSkip(readSkipCount);
+ entity.setProcessSkip(processSkipCount);
+ entity.setFilter(filterCount);
+ entity.setWriteSkip(writeSkipCount);
+ entity.setStartTime(stepContext.getStartTimeTS());
+ entity.setEndTime(stepContext.getEndTimeTS());
+ try {
+ entity.setPersistentData(serialize(stepContext.getPersistentUserData()));
+ } catch (final IOException e) {
+ throw new PersistenceException(e);
+ }
+ }
+
+ @Override
+ public long getJobInstanceIdByExecutionId(final long executionId) throws NoSuchJobExecutionException {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ return em.find(JobExecutionEntity.class, executionId).getInstance().getJobInstanceId();
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public Set<Long> jobOperatorGetRunningExecutions(final String jobName) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final List<JobExecutionEntity> list = em.createNamedQuery(JobExecutionEntity.Queries.FIND_RUNNING, JobExecutionEntity.class)
+ .setParameter("name", jobName)
+ .setParameter("statuses", JobExecutionEntity.Queries.RUNNING_STATUSES)
+ .getResultList();
+
+ if (list == null) {
+ return Collections.emptySet();
+ }
+
+ final Set<Long> ids= new HashSet<Long>(list.size());
+ for (final JobExecutionEntity entity : list) {
+ ids.add(entity.getExecutionId());
+ }
+ return ids;
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public List<InternalJobExecution> jobOperatorGetJobExecutions(final long jobInstanceId) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final List<JobExecutionEntity> list = em.createNamedQuery(JobExecutionEntity.Queries.FIND_BY_INSTANCE, JobExecutionEntity.class)
+ .setParameter("instanceId", jobInstanceId).getResultList();
+
+ if (list == null) {
+ return Collections.emptyList();
+ }
+
+ final List<InternalJobExecution> result = new ArrayList<InternalJobExecution>(list.size());
+ for (final JobExecutionEntity entity : list) {
+ final JobExecutionImpl jobEx = new JobExecutionImpl(entity.getExecutionId(), jobInstanceId);
+ jobEx.setCreateTime(entity.getCreateTime());
+ jobEx.setStartTime(entity.getStartTime());
+ jobEx.setEndTime(entity.getEndTime());
+ jobEx.setLastUpdateTime(entity.getUpdateTime());
+ jobEx.setBatchStatus(entity.getBatchStatus().name());
+ jobEx.setExitStatus(entity.getExitStatus());
+ jobEx.setJobName(entity.getInstance().getName());
+
+ result.add(jobEx);
+ }
+ return result;
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public void updateWithFinalExecutionStatusesAndTimestamps(final long key, final BatchStatus batchStatus, final String exitStatus, final Timestamp updatets) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final Object tx = txProvider.start(em);
+ try {
+ final JobExecutionEntity instance = em.find(JobExecutionEntity.class, key);
+ instance.setBatchStatus(batchStatus);
+ instance.setUpdateTime(updatets);
+ instance.setEndTime(updatets);
+ instance.setExitStatus(exitStatus);
+
+ em.merge(instance);
+ txProvider.commit(tx);
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public InternalJobExecution jobOperatorGetJobExecution(final long jobExecutionId) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final JobExecutionEntity instance = em.find(JobExecutionEntity.class, jobExecutionId);
+
+ final JobExecutionImpl jobEx = new JobExecutionImpl(jobExecutionId, instance.getInstance().getJobInstanceId());
+ jobEx.setCreateTime(instance.getCreateTime());
+ jobEx.setStartTime(instance.getStartTime());
+ jobEx.setEndTime(instance.getEndTime());
+ jobEx.setJobParameters(instance.getJobProperties());
+ jobEx.setLastUpdateTime(instance.getUpdateTime());
+ if (instance.getBatchStatus() != null) {
+ jobEx.setBatchStatus(instance.getBatchStatus().name());
+ }
+ jobEx.setExitStatus(instance.getExitStatus());
+ jobEx.setJobName(instance.getInstance().getName());
+ return jobEx;
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public void markJobStarted(final long key, final Timestamp startTS) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final Object tx = txProvider.start(em);
+ try {
+ final JobExecutionEntity instance = em.find(JobExecutionEntity.class, key);
+ instance.setBatchStatus(BatchStatus.STARTED);
+ instance.setStartTime(startTS);
+ instance.setUpdateTime(startTS);
+
+ em.merge(instance);
+ txProvider.commit(tx);
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public Properties getParameters(final long executionId) throws NoSuchJobExecutionException {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ return em.find(JobExecutionEntity.class, executionId).getJobProperties();
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public String jobOperatorQueryJobExecutionExitStatus(final long key) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ return em.find(JobExecutionEntity.class, key).getExitStatus();
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public String jobOperatorQueryJobExecutionBatchStatus(final long key) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ return em.find(JobExecutionEntity.class, key).getBatchStatus().name();
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public Timestamp jobOperatorQueryJobExecutionTimestamp(final long key, final TimestampType timetype) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final JobExecutionEntity entity = em.find(JobExecutionEntity.class, key);
+ if (timetype.equals(TimestampType.CREATE)) {
+ return entity.getCreateTime();
+ } else if (timetype.equals(TimestampType.END)) {
+ return entity.getEndTime();
+ } else if (timetype.equals(TimestampType.LAST_UPDATED)) {
+ return entity.getUpdateTime();
+ } else if (timetype.equals(TimestampType.STARTED)) {
+ return entity.getStartTime();
+ } else {
+ throw new IllegalArgumentException("Unexpected enum value.");
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public RuntimeFlowInSplitExecution createFlowInSplitExecution(final JobInstance jobInstance, final BatchStatus batchStatus) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final JobExecutionEntity instance = new JobExecutionEntity();
+ instance.setCreateTime(new Timestamp(System.currentTimeMillis()));
+ instance.setUpdateTime(instance.getCreateTime());
+ instance.setBatchStatus(batchStatus);
+
+ final Object tx = txProvider.start(em);
+ try {
+ instance.setInstance(em.find(JobInstanceEntity.class, jobInstance.getInstanceId()));
+
+ em.persist(instance);
+ txProvider.commit(tx);
+
+ final RuntimeFlowInSplitExecution jobExecution = new RuntimeFlowInSplitExecution(jobInstance, instance.getExecutionId());
+ jobExecution.setBatchStatus(batchStatus.name());
+ jobExecution.setCreateTime(instance.getCreateTime());
+ jobExecution.setLastUpdateTime(instance.getCreateTime());
+ return jobExecution;
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public long getMostRecentExecutionId(final long jobInstanceId) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ return em.createNamedQuery(JobExecutionEntity.Queries.MOST_RECENT, JobExecutionEntity.class)
+ .setParameter("instanceId", jobInstanceId)
+ .setMaxResults(1)
+ .getSingleResult().getExecutionId();
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public void updateBatchStatusOnly(final long executionId, final BatchStatus batchStatus, final Timestamp timestamp) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final Object tx = txProvider.start(em);
+ final JobExecutionEntity instance = em.find(JobExecutionEntity.class, executionId);
+ instance.setBatchStatus(batchStatus);
+ instance.setUpdateTime(timestamp);
+
+ try {
+ em.merge(instance);
+ txProvider.commit(tx);
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public RuntimeJobExecution createJobExecution(final JobInstance jobInstance, final Properties jobParameters, final BatchStatus batchStatus) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final JobExecutionEntity execution = new JobExecutionEntity();
+ execution.setJobProperties(jobParameters);
+ execution.setCreateTime(new Timestamp(System.currentTimeMillis()));
+ execution.setUpdateTime(execution.getCreateTime());
+ execution.setBatchStatus(batchStatus);
+
+ final Object tx = txProvider.start(em);
+ try {
+ execution.setInstance(em.find(JobInstanceEntity.class, jobInstance.getInstanceId()));
+
+ em.persist(execution);
+ txProvider.commit(tx);
+
+ final RuntimeJobExecution jobExecution = new RuntimeJobExecution(jobInstance, execution.getExecutionId());
+ jobExecution.setBatchStatus(batchStatus.name());
+ jobExecution.setCreateTime(execution.getCreateTime());
+ jobExecution.setLastUpdateTime(execution.getCreateTime());
+ return jobExecution;
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public JobInstance createSubJobInstance(final String name, final String apptag) {
+ return createJobInstance(name, apptag, null);
+ }
+
+ @Override
+ public List<Long> jobOperatorGetJobInstanceIds(final String jobName, final int start, final int count) {
+ return jobOperatorGetJobInstanceIds(jobName, null, start, count);
+ }
+
+ @Override
+ public List<Long> jobOperatorGetJobInstanceIds(final String jobName, final String appTag, final int start, final int count) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final TypedQuery<JobInstanceEntity> query;
+ if (appTag != null) {
+ query = em.createNamedQuery(JobInstanceEntity.Queries.FIND_BY_NAME, JobInstanceEntity.class).setParameter("tag", appTag);
+ } else {
+ query = em.createNamedQuery(JobInstanceEntity.Queries.FIND_BY_NAME, JobInstanceEntity.class);
+ }
+ query.setParameter("name", jobName);
+
+ final List<JobInstanceEntity> resultList = query
+ .setFirstResult(start)
+ .setMaxResults(count)
+ .getResultList();
+
+ if (resultList == null) {
+ return Collections.emptyList();
+ }
+
+ final List<Long> result = new ArrayList<Long>(resultList.size());
+ for (final JobInstanceEntity entity : resultList) {
+ result.add(entity.getJobInstanceId());
+ }
+ return result;
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public JobInstance createJobInstance(final String name, final String apptag, final String jobXml) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final JobInstanceEntity instance = new JobInstanceEntity();
+ instance.setTag(apptag);
+ instance.setName(name);
+ instance.setJobXml(jobXml);
+
+ final Object tx = txProvider.start(em);
+ try {
+ em.persist(instance);
+ txProvider.commit(tx);
+
+ final JobInstanceImpl jobInstance = new JobInstanceImpl(instance.getJobInstanceId(), jobXml);
+ jobInstance.setJobName(name);
+ return jobInstance;
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public int jobOperatorGetJobInstanceCount(final String jobName) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ return em.createNamedQuery(JobInstanceEntity.Queries.COUNT_BY_NAME, Number.class)
+ .setParameter("name", jobName)
+ .getSingleResult().intValue();
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public int jobOperatorGetJobInstanceCount(final String jobName, final String appTag) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ return em.createNamedQuery(JobInstanceEntity.Queries.COUNT_BY_NAME, Number.class)
+ .setParameter("name", jobName)
+ .setParameter("tag", appTag)
+ .getSingleResult().intValue();
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public void updateStepStatus(final long stepExecutionId, final StepStatus stepStatus) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final Object tx = txProvider.start(em);
+ try {
+ final StepExecutionEntity entity = em.find(StepExecutionEntity.class, stepExecutionId);
+ entity.setBatchStatus(stepStatus.getBatchStatus());
+ entity.setExitStatus(stepStatus.getExitStatus());
+ entity.setLastRunStepExecutionId(stepStatus.getLastRunStepExecutionId());
+ entity.setNumPartitions(stepStatus.getNumPartitions());
+ entity.setPersistentData(stepStatus.getRawPersistentUserData());
+ entity.setStartCount(stepStatus.getStartCount());
+
+ em.merge(entity);
+ txProvider.commit(tx);
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public StepStatus createStepStatus(final long stepExecId) {
+ return new StepStatus(stepExecId); // instance already created
+ }
+
+ @Override
+ public void updateJobStatus(final long instanceId, final JobStatus jobStatus) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final Object tx = txProvider.start(em);
+ try {
+ final JobInstanceEntity entity = em.find(JobInstanceEntity.class, instanceId);
+ entity.setBatchStatus(jobStatus.getBatchStatus());
+ entity.setStep(jobStatus.getCurrentStepId());
+ entity.setLatestExecution(jobStatus.getLatestExecutionId());
+ entity.setExitStatus(jobStatus.getExitStatus());
+ entity.setRestartOn(jobStatus.getRestartOn());
+ if (jobStatus.getJobInstance() != null) {
+ entity.setName(jobStatus.getJobInstance().getJobName());
+ }
+ em.merge(entity);
+
+ final List<JobExecutionEntity> executions = em.createNamedQuery(JobExecutionEntity.Queries.FIND_BY_INSTANCE, JobExecutionEntity.class)
+ .setParameter("instanceId", instanceId)
+ .getResultList();
+ if (executions != null) {
+ for (final JobExecutionEntity e : executions) {
+ e.setInstance(entity);
+ em.merge(e);
+ }
+ }
+ txProvider.commit(tx);
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public JobStatus getJobStatus(final long instanceId) {
+ final JobStatus status = new JobStatus(instanceId);
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final Object tx = txProvider.start(em);
+ try {
+ setJobStatusData(status, em.find(JobInstanceEntity.class, instanceId));
+ txProvider.commit(tx);
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ return status;
+ }
+
+ private void setJobStatusData(final JobStatus status, final JobInstanceEntity entity) {
+ status.setBatchStatus(entity.getBatchStatus());
+ status.setCurrentStepId(entity.getStep());
+ status.setLatestExecutionId(entity.getLatestExecution());
+ status.setExitStatus(entity.getExitStatus());
+ status.setRestartOn(entity.getRestartOn());
+ status.setJobInstance(entity.toJobInstance());
+ }
+
+ @Override
+ public JobStatus createJobStatus(final long jobInstanceId) {
+ return new JobStatus(jobInstanceId); // instance already created
+ }
+
+ @Override
+ public void setCheckpointData(final CheckpointDataKey key, final CheckpointData value) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final Object tx = txProvider.start(em);
+ try {
+ final List<CheckpointEntity> checkpoints = em.createNamedQuery(CheckpointEntity.Queries.FIND, CheckpointEntity.class)
+ .setParameter("jobInstanceId", key.getJobInstanceId())
+ .setParameter("stepName", key.getStepName())
+ .setParameter("type", key.getType())
+ .getResultList();
+
+ final CheckpointEntity checkpoint;
+ final boolean isNew = checkpoints == null || checkpoints.isEmpty();
+ if (isNew) {
+ checkpoint = new CheckpointEntity();
+ checkpoint.setInstance(em.find(JobInstanceEntity.class, key.getJobInstanceId()));
+ checkpoint.setStepName(key.getStepName());
+ checkpoint.setType(key.getType());
+ } else {
+ checkpoint = checkpoints.iterator().next();
+ }
+
+ checkpoint.setData(value.getRestartToken());
+
+ if (isNew) {
+ em.persist(checkpoint);
+ } else {
+ em.merge(checkpoint);
+ }
+ txProvider.commit(tx);
+ } catch (final Exception e) {
+ txProvider.rollback(tx, e);
+ throw new BatchContainerRuntimeException(e);
+ }
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public CheckpointData getCheckpointData(final CheckpointDataKey key) {
+ final EntityManager em = emProvider.newEntityManager();
+ try {
+ final CheckpointEntity checkpoint = em.createNamedQuery(CheckpointEntity.Queries.FIND, CheckpointEntity.class)
+ .setParameter("jobInstanceId", key.getJobInstanceId())
+ .setParameter("stepName", key.getStepName())
+ .setParameter("type", key.getType())
+ .getSingleResult();
+
+ final CheckpointData data = new CheckpointData(checkpoint.getInstance().getJobInstanceId(), checkpoint.getStepName(), checkpoint.getType());
+ data.setRestartToken(checkpoint.getData());
+ return data;
+ } catch (final NoResultException nre) {
+ return null;
+ } finally {
+ emProvider.release(em);
+ }
+ }
+
+ @Override
+ public void init(final Properties batchConfig) {
+ final String txProviderClass = batchConfig.getProperty("persistence.jpa.transaction-provider", DefaultTransactionProvider.class.getName());
+ try {
+ txProvider = TransactionProvider.class.cast(Thread.currentThread().getContextClassLoader().loadClass(txProviderClass).newInstance());
+ } catch (final Exception e) {
+ throw new BatchContainerRuntimeException(e);
+ }
+ txProvider.init(batchConfig);
+
+ final String providerClass = batchConfig.getProperty("persistence.jpa.entity-manager-provider", DefaultEntityManagerProvider.class.getName());
+ try {
+ emProvider = EntityManagerProvider.class.cast(Thread.currentThread().getContextClassLoader().loadClass(providerClass).newInstance());
+ } catch (final Exception e) {
+ throw new BatchContainerRuntimeException(e);
+ }
+ emProvider.init(batchConfig);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-batchee/blob/efa64877/jbatch/src/main/java/org/apache/batchee/container/services/persistence/MemoryPersistenceManager.java
----------------------------------------------------------------------
diff --git a/jbatch/src/main/java/org/apache/batchee/container/services/persistence/MemoryPersistenceManager.java b/jbatch/src/main/java/org/apache/batchee/container/services/persistence/MemoryPersistenceManager.java
new file mode 100644
index 0000000..34ab68d
--- /dev/null
+++ b/jbatch/src/main/java/org/apache/batchee/container/services/persistence/MemoryPersistenceManager.java
@@ -0,0 +1,654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.batchee.container.services.persistence;
+
+import org.apache.batchee.container.impl.JobExecutionImpl;
+import org.apache.batchee.container.impl.JobInstanceImpl;
+import org.apache.batchee.container.impl.MetricImpl;
+import org.apache.batchee.container.impl.StepContextImpl;
+import org.apache.batchee.container.impl.StepExecutionImpl;
+import org.apache.batchee.container.impl.controller.PartitionedStepBuilder;
+import org.apache.batchee.container.impl.controller.chunk.CheckpointData;
+import org.apache.batchee.container.impl.controller.chunk.CheckpointDataKey;
+import org.apache.batchee.container.impl.jobinstance.RuntimeFlowInSplitExecution;
+import org.apache.batchee.container.impl.jobinstance.RuntimeJobExecution;
+import org.apache.batchee.container.services.InternalJobExecution;
+import org.apache.batchee.container.status.JobStatus;
+import org.apache.batchee.container.status.StepStatus;
+import org.apache.batchee.spi.PersistenceManagerService;
+
+import javax.batch.operations.NoSuchJobExecutionException;
+import javax.batch.runtime.BatchStatus;
+import javax.batch.runtime.JobInstance;
+import javax.batch.runtime.Metric;
+import javax.batch.runtime.StepExecution;
+import java.io.Serializable;
+import java.sql.Timestamp;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class MemoryPersistenceManager implements PersistenceManagerService {
+ private static final Collection<BatchStatus> RUNNING_STATUSES = new CopyOnWriteArrayList<BatchStatus>() {{
+ add(BatchStatus.STARTED);
+ add(BatchStatus.STARTING);
+ add(BatchStatus.STOPPING);
+ }};
+
+ protected static interface Structures extends Serializable {
+ static class JobInstanceData implements Structures {
+ protected JobInstanceImpl instance;
+ protected String tag;
+ protected JobStatus status;
+ protected final List<ExecutionInstanceData> executions = new LinkedList<ExecutionInstanceData>();
+ protected final Collection<CheckpointDataKey> checkpoints = new LinkedList<CheckpointDataKey>();
+ }
+
+ static class ExecutionInstanceData implements Structures {
+ protected final List<StepExecution> stepExecutions = new LinkedList<StepExecution>();
+ protected JobExecutionImpl execution;
+ }
+
+ static class StepExecutionInstanceData implements Structures {
+ protected ExecutionInstanceData jobExec;
+ protected StepExecutionImpl execution;
+ protected StepStatus status;
+ }
+ }
+
+ static class Data {
+ protected final AtomicLong lastCleanedJobInstanceId = new AtomicLong(0);
+ protected final AtomicLong jobInstanceIdGenerator = new AtomicLong();
+ protected final AtomicLong executionInstanceIdGenerator = new AtomicLong();
+ protected final AtomicLong stepExecutionIdGenerator = new AtomicLong();
+
+ protected final Map<CheckpointDataKey, CheckpointData> checkpointData = new ConcurrentHashMap<CheckpointDataKey, CheckpointData>();
+ protected final Map<Long, Structures.JobInstanceData> jobInstanceData = new ConcurrentHashMap<Long, Structures.JobInstanceData>();
+ protected final Map<Long, Structures.ExecutionInstanceData> executionInstanceData = new ConcurrentHashMap<Long, Structures.ExecutionInstanceData>();
+ protected final Map<Long, Structures.StepExecutionInstanceData> stepExecutionInstanceData = new ConcurrentHashMap<Long, Structures.StepExecutionInstanceData>();
+ }
+
+ private static final Data GLOBAL_DATA = new Data();
+
+ private Data data;
+ private int maxSize;
+
+ // an extension point to be able to use whatever map you want, a distributed/filtered/size-limited one for instance
+ protected Data newData() {
+ return new Data();
+ }
+
+ @Override
+ public void init(final Properties batchConfig) {
+ if ("true".equalsIgnoreCase(batchConfig.getProperty("persistence.memory.global", "false"))) {
+ data = GLOBAL_DATA;
+ } else {
+ data = newData();
+ }
+ maxSize = Integer.parseInt(batchConfig.getProperty("persistence.memory.max-jobs-instances", "1000"));
+ }
+
+ @Override
+ public int jobOperatorGetJobInstanceCount(final String jobName) {
+ return jobOperatorGetJobInstanceCount(jobName, null);
+ }
+
+ @Override
+ public int jobOperatorGetJobInstanceCount(final String jobName, final String appTag) {
+ final boolean hasTag = appTag != null;
+ int i = 0;
+ for (final Structures.JobInstanceData jobInstanceData : data.jobInstanceData.values()) {
+ if (jobInstanceData.instance.getJobName().equals(jobName) && (!hasTag || appTag.equals(jobInstanceData.tag))) {
+ i++;
+ }
+ }
+ return i;
+ }
+
+ @Override
+ public Map<Long, String> jobOperatorGetExternalJobInstanceData() {
+ final Map<Long, String> out = new HashMap<Long, String>();
+ for (final Structures.JobInstanceData jobInstanceData : data.jobInstanceData.values()) {
+ if (jobInstanceData.instance.getJobName() != null && !jobInstanceData.instance.getJobName().startsWith(PartitionedStepBuilder.JOB_ID_SEPARATOR)) {
+ out.put(jobInstanceData.instance.getInstanceId(), jobInstanceData.instance.getJobName());
+ }
+ }
+ return out;
+ }
+
+ @Override
+ public List<Long> jobOperatorGetJobInstanceIds(final String jobName, final int start, final int count) {
+ return jobOperatorGetJobInstanceIds(jobName, null, start, count);
+ }
+
+ @Override
+ public List<Long> jobOperatorGetJobInstanceIds(final String jobName, final String appTag, final int start, final int count) {
+ final List<Long> out = new LinkedList<Long>();
+ for (final Structures.JobInstanceData jobInstanceData : data.jobInstanceData.values()) {
+ if (jobInstanceData.instance.getJobName() != null && jobInstanceData.instance.getJobName().equals(jobName)) {
+ final boolean hasTag = appTag != null;
+ if (!hasTag || appTag.equals(jobInstanceData.tag)) {
+ out.add(jobInstanceData.instance.getInstanceId());
+ }
+ }
+ }
+
+ // sorting could be optimized a bit but is it necessary for this impl?
+ Collections.sort(out);
+ Collections.reverse(out);
+
+ if (out.size() > 0) {
+ try {
+ return out.subList(start, start + count);
+ } catch (final IndexOutOfBoundsException oobEx) {
+ return out.subList(start, out.size());
+ }
+ }
+
+ return out;
+ }
+
+ @Override
+ public Timestamp jobOperatorQueryJobExecutionTimestamp(final long key, final TimestampType timestampType) {
+ return null; // avoid infinite loops
+ }
+
+ @Override
+ public String jobOperatorQueryJobExecutionBatchStatus(final long key) {
+ return null; // avoid infinite loops
+ }
+
+ @Override
+ public String jobOperatorQueryJobExecutionExitStatus(final long key) {
+ return null; // avoid infinite loops
+ }
+
+ @Override
+ public List<StepExecution> getStepExecutionsForJobExecution(final long execid) {
+ final Structures.ExecutionInstanceData executionInstanceData = data.executionInstanceData.get(execid);
+ synchronized (executionInstanceData.stepExecutions) {
+ return executionInstanceData.stepExecutions;
+ }
+ }
+
+ @Override
+ public void updateBatchStatusOnly(final long executionId, final BatchStatus batchStatus, final Timestamp timestamp) {
+ final Structures.ExecutionInstanceData toUpdate = data.executionInstanceData.get(executionId);
+ toUpdate.execution.setBatchStatus(batchStatus.name());
+ toUpdate.execution.setLastUpdateTime(timestamp);
+ }
+
+ @Override
+ public void markJobStarted(final long key, final Timestamp startTS) {
+ final Structures.ExecutionInstanceData toUpdate = data.executionInstanceData.get(key);
+ toUpdate.execution.setBatchStatus(BatchStatus.STARTED.name());
+ toUpdate.execution.setLastUpdateTime(startTS);
+ toUpdate.execution.setStartTime(startTS);
+ }
+
+ @Override
+ public void updateWithFinalExecutionStatusesAndTimestamps(final long key, final BatchStatus batchStatus, final String exitStatus, final Timestamp updatets) {
+ final Structures.ExecutionInstanceData toUpdate = data.executionInstanceData.get(key);
+ if (batchStatus != null) {
+ toUpdate.execution.setBatchStatus(batchStatus.name());
+ } else {
+ toUpdate.execution.setBatchStatus(null);
+ }
+ toUpdate.execution.setExitStatus(exitStatus);
+ toUpdate.execution.setLastUpdateTime(updatets);
+ toUpdate.execution.setEndTime(updatets);
+ }
+
+ @Override
+ public InternalJobExecution jobOperatorGetJobExecution(final long jobExecutionId) {
+ final Structures.ExecutionInstanceData executionInstanceData = data.executionInstanceData.get(jobExecutionId);
+ if (executionInstanceData == null) {
+ return null;
+ }
+ return executionInstanceData.execution;
+ }
+
+ @Override
+ public Properties getParameters(final long executionId) throws NoSuchJobExecutionException {
+ return data.executionInstanceData.get(executionId).execution.getJobParameters();
+ }
+
+ @Override
+ public List<InternalJobExecution> jobOperatorGetJobExecutions(final long jobInstanceId) {
+ final List<InternalJobExecution> list = new LinkedList<InternalJobExecution>();
+ final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(jobInstanceId);
+ synchronized (jobInstanceData.executions) {
+ for (final Structures.ExecutionInstanceData executionInstanceData : jobInstanceData.executions) {
+ list.add(executionInstanceData.execution);
+ }
+ }
+ return list;
+ }
+
+ @Override
+ public Set<Long> jobOperatorGetRunningExecutions(final String jobName) {
+ Structures.JobInstanceData jobInstanceData = null;
+ for (final Structures.JobInstanceData instanceData : data.jobInstanceData.values()) {
+ if (instanceData.instance.getJobName().equals(jobName)) {
+ jobInstanceData = instanceData;
+ break;
+ }
+ }
+
+ if (jobInstanceData == null) {
+ return Collections.emptySet();
+ }
+
+ final Set<Long> set = new HashSet<Long>();
+
+ synchronized (jobInstanceData.executions) {
+ for (final Structures.ExecutionInstanceData executionInstanceData : jobInstanceData.executions) {
+ if (RUNNING_STATUSES.contains(executionInstanceData.execution.getBatchStatus())) {
+ set.add(executionInstanceData.execution.getExecutionId());
+ }
+ }
+ }
+ return set;
+ }
+
+ @Override
+ public JobStatus getJobStatusFromExecution(final long executionId) {
+ final Structures.ExecutionInstanceData executionInstanceData = data.executionInstanceData.get(executionId);
+ if (executionInstanceData == null) {
+ return null;
+ }
+
+ final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(executionInstanceData.execution.getInstanceId());
+ if (jobInstanceData == null) {
+ return null;
+ }
+ return jobInstanceData.status;
+ }
+
+ @Override
+ public long getJobInstanceIdByExecutionId(final long executionId) throws NoSuchJobExecutionException {
+ return data.executionInstanceData.get(executionId).execution.getInstanceId();
+ }
+
+ @Override
+ public JobInstance createJobInstance(final String name, final String apptag, final String jobXml) {
+ final JobInstanceImpl jobInstance = new JobInstanceImpl(data.jobInstanceIdGenerator.getAndIncrement(), jobXml);
+ jobInstance.setJobName(name);
+
+ final Structures.JobInstanceData jobInstanceData = new Structures.JobInstanceData();
+ jobInstanceData.tag = apptag;
+ jobInstanceData.instance = jobInstance;
+ if (maxSize > 0 && data.jobInstanceData.size() >= maxSize) {
+ synchronized (this) {
+ while (data.jobInstanceData.size() >= maxSize) {
+ cleanUp(data.lastCleanedJobInstanceId.getAndIncrement());
+ }
+ }
+ }
+ data.jobInstanceData.put(jobInstance.getInstanceId(), jobInstanceData);
+
+ return jobInstance;
+ }
+
+ @Override
+ public RuntimeJobExecution createJobExecution(final JobInstance jobInstance, final Properties jobParameters, final BatchStatus batchStatus) {
+ final Timestamp now = new Timestamp(System.currentTimeMillis());
+
+ final Structures.ExecutionInstanceData executionInstanceData = createRuntimeJobExecutionEntry(jobInstance, jobParameters, batchStatus, now);
+ executionInstanceData.execution.setJobName(jobInstance.getJobName());
+
+ final RuntimeJobExecution jobExecution = new RuntimeJobExecution(jobInstance, executionInstanceData.execution.getExecutionId());
+ jobExecution.setBatchStatus(batchStatus.name());
+ jobExecution.setCreateTime(now);
+ jobExecution.setLastUpdateTime(now);
+
+ return jobExecution;
+ }
+
+ private Structures.ExecutionInstanceData createRuntimeJobExecutionEntry(final JobInstance jobInstance, final Properties jobParameters, final BatchStatus batchStatus, final Timestamp now) {
+ final Structures.ExecutionInstanceData executionInstanceData = new Structures.ExecutionInstanceData();
+ final long id = data.executionInstanceIdGenerator.getAndIncrement();
+ executionInstanceData.execution = new JobExecutionImpl(id, jobInstance.getInstanceId());
+ executionInstanceData.execution.setExecutionId(id);
+ executionInstanceData.execution.setInstanceId(jobInstance.getInstanceId());
+ executionInstanceData.execution.setBatchStatus(batchStatus.name());
+ executionInstanceData.execution.setCreateTime(now);
+ executionInstanceData.execution.setLastUpdateTime(now);
+ executionInstanceData.execution.setJobParameters(jobParameters);
+
+ data.executionInstanceData.put(id, executionInstanceData);
+ final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(jobInstance.getInstanceId());
+ synchronized (jobInstanceData.executions) {
+ jobInstanceData.executions.add(executionInstanceData);
+ }
+ return executionInstanceData;
+ }
+
+ @Override
+ public StepExecutionImpl createStepExecution(final long rootJobExecId, final StepContextImpl stepContext) {
+ final String batchStatus = stepContext.getBatchStatus() == null ? BatchStatus.STARTING.name() : stepContext.getBatchStatus().name();
+ final String exitStatus = stepContext.getExitStatus();
+ final String stepName = stepContext.getStepName();
+
+ long readCount = 0;
+ long writeCount = 0;
+ long commitCount = 0;
+ long rollbackCount = 0;
+ long readSkipCount = 0;
+ long processSkipCount = 0;
+ long filterCount = 0;
+ long writeSkipCount = 0;
+ Timestamp startTime = stepContext.getStartTimeTS();
+ Timestamp endTime = stepContext.getEndTimeTS();
+
+ final Metric[] metrics = stepContext.getMetrics();
+ for (final Metric metric : metrics) {
+ if (metric.getType().equals(MetricImpl.MetricType.READ_COUNT)) {
+ readCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.WRITE_COUNT)) {
+ writeCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.PROCESS_SKIP_COUNT)) {
+ processSkipCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.COMMIT_COUNT)) {
+ commitCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.ROLLBACK_COUNT)) {
+ rollbackCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.READ_SKIP_COUNT)) {
+ readSkipCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.FILTER_COUNT)) {
+ filterCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.WRITE_SKIP_COUNT)) {
+ writeSkipCount = metric.getValue();
+ }
+ }
+ final Serializable persistentData = stepContext.getPersistentUserData();
+
+ return createStepExecution(rootJobExecId, batchStatus, exitStatus, stepName, readCount,
+ writeCount, commitCount, rollbackCount, readSkipCount, processSkipCount, filterCount, writeSkipCount, startTime,
+ endTime, persistentData);
+ }
+
+ private StepExecutionImpl createStepExecution(final long rootJobExecId, final String batchStatus, final String exitStatus, final String stepName,
+ final long readCount, final long writeCount, final long commitCount, final long rollbackCount,
+ final long readSkipCount, final long processSkipCount, final long filterCount, final long writeSkipCount,
+ final Timestamp startTime, final Timestamp endTime, final Serializable persistentData) {
+
+ final StepExecutionImpl stepExecution = new StepExecutionImpl(rootJobExecId, data.stepExecutionIdGenerator.getAndIncrement());
+ stepExecution.setStepName(stepName);
+ final Structures.ExecutionInstanceData executionInstanceData = data.executionInstanceData.get(rootJobExecId);
+ synchronized (executionInstanceData.stepExecutions) {
+ executionInstanceData.stepExecutions.add(stepExecution);
+ }
+
+ final Structures.StepExecutionInstanceData stepExecutionInstanceData = new Structures.StepExecutionInstanceData();
+ stepExecutionInstanceData.execution = stepExecution;
+ updateStepExecutionInstanceData(executionInstanceData, batchStatus, exitStatus, stepName,
+ readCount, writeCount, commitCount, rollbackCount,
+ readSkipCount, processSkipCount, filterCount, writeSkipCount,
+ startTime, endTime, persistentData,
+ stepExecutionInstanceData);
+ data.stepExecutionInstanceData.put(stepExecution.getStepExecutionId(), stepExecutionInstanceData);
+
+ return stepExecution;
+ }
+
+ private void updateStepExecutionInstanceData(final Structures.ExecutionInstanceData exec,
+ final String batchStatus,final String exitStatus, final String stepName,
+ final long readCount, final long writeCount, final long commitCount, final long rollbackCount,
+ final long readSkipCount, final long processSkipCount, final long filterCount, final long writeSkipCount,
+ final Timestamp startTime, final Timestamp endTime,
+ final Serializable persistentData,
+ final Structures.StepExecutionInstanceData stepExecutionInstanceData) {
+ stepExecutionInstanceData.jobExec = exec;
+ stepExecutionInstanceData.execution.setExitStatus(exitStatus);
+ stepExecutionInstanceData.execution.setBatchStatus(BatchStatus.valueOf(batchStatus));
+ stepExecutionInstanceData.execution.setRollbackCount(rollbackCount);
+ stepExecutionInstanceData.execution.setStepName(stepName);
+ stepExecutionInstanceData.execution.setReadCount(readCount);
+ stepExecutionInstanceData.execution.setWriteCount(writeCount);
+ stepExecutionInstanceData.execution.setCommitCount(commitCount);
+ stepExecutionInstanceData.execution.setRollbackCount(rollbackCount);
+ stepExecutionInstanceData.execution.setReadSkipCount(readSkipCount);
+ stepExecutionInstanceData.execution.setProcessSkipCount(processSkipCount);
+ stepExecutionInstanceData.execution.setWriteSkipCount(writeSkipCount);
+ stepExecutionInstanceData.execution.setFilterCount(filterCount);
+ stepExecutionInstanceData.execution.setWriteSkipCount(writeSkipCount);
+ stepExecutionInstanceData.execution.setStartTime(startTime);
+ stepExecutionInstanceData.execution.setEndTime(endTime);
+ stepExecutionInstanceData.execution.setPersistentUserData(persistentData);
+ }
+
+ @Override
+ public void updateStepExecution(final long jobExecId, final StepContextImpl stepContext) {
+ final long stepExecutionId = stepContext.getStepExecutionId();
+ final String batchStatus = stepContext.getBatchStatus() == null ? BatchStatus.STARTING.name() : stepContext.getBatchStatus().name();
+ final String exitStatus = stepContext.getExitStatus();
+ final String stepName = stepContext.getStepName();
+
+ long readCount = 0;
+ long writeCount = 0;
+ long commitCount = 0;
+ long rollbackCount = 0;
+ long readSkipCount = 0;
+ long processSkipCount = 0;
+ long filterCount = 0;
+ long writeSkipCount = 0;
+ Timestamp startTime = stepContext.getStartTimeTS();
+ Timestamp endTime = stepContext.getEndTimeTS();
+
+ final Metric[] metrics = stepContext.getMetrics();
+ for (final Metric metric : metrics) {
+ if (metric.getType().equals(MetricImpl.MetricType.READ_COUNT)) {
+ readCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.WRITE_COUNT)) {
+ writeCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.PROCESS_SKIP_COUNT)) {
+ processSkipCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.COMMIT_COUNT)) {
+ commitCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.ROLLBACK_COUNT)) {
+ rollbackCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.READ_SKIP_COUNT)) {
+ readSkipCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.FILTER_COUNT)) {
+ filterCount = metric.getValue();
+ } else if (metric.getType().equals(MetricImpl.MetricType.WRITE_SKIP_COUNT)) {
+ writeSkipCount = metric.getValue();
+ }
+ }
+ final Serializable persistentData = stepContext.getPersistentUserData();
+
+ updateStepExecution(stepExecutionId, jobExecId, batchStatus, exitStatus, stepName, readCount,
+ writeCount, commitCount, rollbackCount, readSkipCount, processSkipCount, filterCount,
+ writeSkipCount, startTime, endTime, persistentData);
+ }
+
+ private void updateStepExecution(final long stepExecutionId, final long jobExecId, final String batchStatus, final String exitStatus, final String stepName,
+ final long readCount, final long writeCount, final long commitCount, final long rollbackCount,
+ final long readSkipCount, final long processSkipCount, final long filterCount, final long writeSkipCount,
+ final Timestamp startTime, final Timestamp endTime, final Serializable persistentData) {
+ final Structures.ExecutionInstanceData executionInstanceData = data.executionInstanceData.get(jobExecId);
+ if (executionInstanceData == null) {
+ return;
+ }
+
+ synchronized (executionInstanceData.stepExecutions) {
+ for (final StepExecution execution : executionInstanceData.stepExecutions) {
+ if (execution.getStepExecutionId() == stepExecutionId) {
+ updateStepExecutionInstanceData(executionInstanceData, batchStatus, exitStatus, stepName,
+ readCount, writeCount, commitCount, rollbackCount,
+ readSkipCount, processSkipCount, filterCount, writeSkipCount,
+ startTime, endTime, persistentData,
+ data.stepExecutionInstanceData.get(stepExecutionId));
+ break;
+ }
+ }
+ }
+ }
+
+ @Override
+ public JobStatus createJobStatus(final long jobInstanceId) {
+ final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(jobInstanceId);
+ jobInstanceData.status = new JobStatus(jobInstanceId);
+ return jobInstanceData.status;
+ }
+
+ @Override
+ public JobStatus getJobStatus(final long instanceId) {
+ final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(instanceId);
+ if (jobInstanceData == null) {
+ return null;
+ }
+ return jobInstanceData.status;
+ }
+
+ @Override
+ public void updateJobStatus(final long instanceId, final JobStatus jobStatus) {
+ final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(instanceId);
+ if (jobInstanceData != null) {
+ jobInstanceData.status = jobStatus;
+ }
+ }
+
+ @Override
+ public StepStatus createStepStatus(final long stepExecId) {
+ final StepStatus stepStatus = new StepStatus(stepExecId);
+ data.stepExecutionInstanceData.get(stepExecId).status = stepStatus;
+ return stepStatus;
+ }
+
+ @Override
+ public StepStatus getStepStatus(final long instanceId, final String stepName) {
+ final Map<Date, StepStatus> statusMap = new TreeMap<Date, StepStatus>(ReverseDateComparator.INSTANCE);
+ for (final Structures.StepExecutionInstanceData stepExecutionInstanceData : data.stepExecutionInstanceData.values()) {
+ if (stepExecutionInstanceData.jobExec.execution.getInstanceId() == instanceId
+ && stepExecutionInstanceData.execution.getStepName().equals(stepName)
+ && stepExecutionInstanceData.status != null) {
+ statusMap.put(stepExecutionInstanceData.jobExec.execution.getCreateTime(), stepExecutionInstanceData.status);
+ }
+ }
+ if (statusMap.isEmpty()) {
+ return null;
+ }
+ return statusMap.values().iterator().next();
+ }
+
+ @Override
+ public void updateStepStatus(final long stepExecutionId, final StepStatus stepStatus) {
+ final Structures.StepExecutionInstanceData stepExecutionInstanceData = data.stepExecutionInstanceData.get(stepExecutionId);
+ if (stepExecutionInstanceData != null) {
+ stepExecutionInstanceData.status = stepStatus;
+ }
+ }
+
+ @Override
+ public void setCheckpointData(final CheckpointDataKey key, final CheckpointData value) {
+ data.checkpointData.put(key, value);
+ final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(key.getJobInstanceId());
+ synchronized (jobInstanceData.checkpoints) {
+ jobInstanceData.checkpoints.add(key);
+ }
+ }
+
+ @Override
+ public CheckpointData getCheckpointData(final CheckpointDataKey key) {
+ return data.checkpointData.get(key);
+ }
+
+ @Override
+ public long getMostRecentExecutionId(final long jobInstanceId) {
+ final Map<Date, Structures.ExecutionInstanceData> filter = new TreeMap<Date, Structures.ExecutionInstanceData>(ReverseDateComparator.INSTANCE);
+ final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(jobInstanceId);
+ synchronized (jobInstanceData.executions) {
+ for (final Structures.ExecutionInstanceData exec : jobInstanceData.executions) {
+ if (exec.execution.getInstanceId() == jobInstanceId) {
+ filter.put(exec.execution.getCreateTime(), exec);
+ }
+ }
+ }
+ if (filter.isEmpty()) {
+ return -1;
+ }
+ return filter.values().iterator().next().execution.getExecutionId();
+ }
+
+ @Override
+ public JobInstance createSubJobInstance(final String name, final String apptag) {
+ return createJobInstance(name, apptag, null);
+ }
+
+ @Override
+ public RuntimeFlowInSplitExecution createFlowInSplitExecution(final JobInstance jobInstance, final BatchStatus batchStatus) {
+ final Timestamp now = new Timestamp(System.currentTimeMillis());
+ final Structures.ExecutionInstanceData executionInstanceData = createRuntimeJobExecutionEntry(jobInstance, null, batchStatus, now);
+ final RuntimeFlowInSplitExecution flowExecution = new RuntimeFlowInSplitExecution(jobInstance, executionInstanceData.execution.getExecutionId());
+ flowExecution.setBatchStatus(batchStatus.name());
+ flowExecution.setCreateTime(now);
+ flowExecution.setLastUpdateTime(now);
+ return flowExecution;
+ }
+
+ @Override
+ public StepExecution getStepExecutionByStepExecutionId(final long stepExecId) {
+ return data.stepExecutionInstanceData.get(stepExecId).execution;
+ }
+
+ @Override
+ public void cleanUp(final long instanceId) {
+ final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.remove(instanceId);
+ if (jobInstanceData == null) {
+ return;
+ }
+
+ synchronized (jobInstanceData.executions) {
+ for (final Structures.ExecutionInstanceData executionInstanceData : jobInstanceData.executions) {
+ data.executionInstanceData.remove(executionInstanceData.execution.getExecutionId());
+ synchronized (executionInstanceData.stepExecutions) {
+ for (final StepExecution stepExecution : executionInstanceData.stepExecutions) {
+ data.stepExecutionInstanceData.remove(stepExecution.getStepExecutionId());
+ }
+ }
+ }
+ }
+ synchronized (jobInstanceData.checkpoints) {
+ for (final CheckpointDataKey key : jobInstanceData.checkpoints) {
+ data.checkpointData.remove(key);
+ }
+ }
+ }
+
+ private static class ReverseDateComparator implements Comparator<Date> {
+ public static final ReverseDateComparator INSTANCE = new ReverseDateComparator();
+
+ @Override
+ public int compare(final Date o1, final Date o2) {
+ return o2.compareTo(o1);
+ }
+ }
+}