You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@batchee.apache.org by rm...@apache.org on 2013/11/05 08:39:00 UTC

[23/62] importing batchee from github - a fork from the IBm RI

http://git-wip-us.apache.org/repos/asf/incubator-batchee/blob/efa64877/jbatch/src/main/java/org/apache/batchee/container/services/persistence/JPAPersistenceService.java
----------------------------------------------------------------------
diff --git a/jbatch/src/main/java/org/apache/batchee/container/services/persistence/JPAPersistenceService.java b/jbatch/src/main/java/org/apache/batchee/container/services/persistence/JPAPersistenceService.java
new file mode 100644
index 0000000..8c0f6f0
--- /dev/null
+++ b/jbatch/src/main/java/org/apache/batchee/container/services/persistence/JPAPersistenceService.java
@@ -0,0 +1,847 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.batchee.container.services.persistence;
+
+import org.apache.batchee.container.exception.BatchContainerRuntimeException;
+import org.apache.batchee.container.exception.PersistenceException;
+import org.apache.batchee.container.impl.JobExecutionImpl;
+import org.apache.batchee.container.impl.JobInstanceImpl;
+import org.apache.batchee.container.impl.StepContextImpl;
+import org.apache.batchee.container.impl.StepExecutionImpl;
+import org.apache.batchee.container.impl.controller.PartitionedStepBuilder;
+import org.apache.batchee.container.impl.controller.chunk.CheckpointData;
+import org.apache.batchee.container.impl.controller.chunk.CheckpointDataKey;
+import org.apache.batchee.container.impl.controller.chunk.PersistentDataWrapper;
+import org.apache.batchee.container.impl.jobinstance.RuntimeFlowInSplitExecution;
+import org.apache.batchee.container.impl.jobinstance.RuntimeJobExecution;
+import org.apache.batchee.container.services.InternalJobExecution;
+import org.apache.batchee.container.services.persistence.jpa.EntityManagerProvider;
+import org.apache.batchee.container.services.persistence.jpa.TransactionProvider;
+import org.apache.batchee.container.services.persistence.jpa.domain.CheckpointEntity;
+import org.apache.batchee.container.services.persistence.jpa.domain.JobExecutionEntity;
+import org.apache.batchee.container.services.persistence.jpa.domain.JobInstanceEntity;
+import org.apache.batchee.container.services.persistence.jpa.domain.StepExecutionEntity;
+import org.apache.batchee.container.services.persistence.jpa.provider.DefaultEntityManagerProvider;
+import org.apache.batchee.container.services.persistence.jpa.provider.DefaultTransactionProvider;
+import org.apache.batchee.container.status.JobStatus;
+import org.apache.batchee.container.status.StepStatus;
+import org.apache.batchee.spi.PersistenceManagerService;
+
+import javax.batch.operations.NoSuchJobExecutionException;
+import javax.batch.runtime.BatchStatus;
+import javax.batch.runtime.JobInstance;
+import javax.batch.runtime.Metric;
+import javax.batch.runtime.StepExecution;
+import javax.persistence.EntityManager;
+import javax.persistence.NoResultException;
+import javax.persistence.TypedQuery;
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import static org.apache.batchee.container.util.Serializations.deserialize;
+import static org.apache.batchee.container.util.Serializations.serialize;
+
+public class JPAPersistenceService implements PersistenceManagerService {
+    private static final String[] DELETE_QUERIES = {
+        StepExecutionEntity.Queries.DELETE_BY_INSTANCE_ID, CheckpointEntity.Queries.DELETE_BY_INSTANCE_ID,
+        JobExecutionEntity.Queries.DELETE_BY_INSTANCE_ID, JobInstanceEntity.Queries.DELETE_BY_INSTANCE_ID
+    };
+
+    private EntityManagerProvider emProvider;
+    private TransactionProvider txProvider;
+
+    @Override
+    public void cleanUp(final long instanceId) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final Object tx = txProvider.start(em);
+            try {
+                for (final String query : DELETE_QUERIES) {
+                    em.createQuery(query).setParameter("instanceId", instanceId).executeUpdate();
+                }
+                txProvider.commit(tx);
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public StepStatus getStepStatus(final long instanceId, final String stepName) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final List<StepExecutionEntity> list = em.createNamedQuery(StepExecutionEntity.Queries.FIND_BY_INSTANCE_AND_NAME, StepExecutionEntity.class)
+                .setParameter("instanceId", instanceId)
+                .setParameter("step", stepName)
+                .getResultList();
+            if (list != null && !list.isEmpty()) {
+                final StepExecutionEntity entity = list.iterator().next();
+                final StepStatus status = new StepStatus(entity.getId(), entity.getStartCount());
+                status.setBatchStatus(entity.getBatchStatus());
+                status.setExitStatus(entity.getExitStatus());
+                status.setNumPartitions(entity.getNumPartitions());
+                status.setLastRunStepExecutionId(entity.getLastRunStepExecutionId());
+                if (entity.getPersistentData() != null) {
+                    status.setPersistentUserData(new PersistentDataWrapper(entity.getPersistentData()));
+                }
+                return status;
+            }
+            return null;
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public Map<Long, String> jobOperatorGetExternalJobInstanceData() {
+        final Map<Long, String> data = new HashMap<Long, String>();
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final List<JobInstanceEntity> list = em.createNamedQuery(JobInstanceEntity.Queries.FIND_EXTERNALS, JobInstanceEntity.class)
+                .setParameter("pattern", PartitionedStepBuilder.JOB_ID_SEPARATOR)
+                .getResultList();
+            if (list != null) {
+                for (final JobInstanceEntity elt : list) {
+                    data.put(elt.getJobInstanceId(), elt.getName());
+                }
+            }
+        } finally {
+            emProvider.release(em);
+        }
+        return data;
+    }
+
+    @Override
+    public JobStatus getJobStatusFromExecution(final long executionId) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final JobInstanceEntity entity = em.createNamedQuery(JobInstanceEntity.Queries.FIND_FROM_EXECUTION, JobInstanceEntity.class)
+                .setParameter("executionId", executionId)
+                .getSingleResult();
+            final JobStatus status = new JobStatus(entity.getJobInstanceId());
+            setJobStatusData(status, entity);
+            return status;
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public StepExecution getStepExecutionByStepExecutionId(final long stepExecId) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final StepExecutionEntity entity = em.find(StepExecutionEntity.class, stepExecId);
+            final StepExecutionImpl stepEx = new StepExecutionImpl(entity.getExecution().getExecutionId(), stepExecId);
+            setStepExecutionData(entity, stepEx);
+            return stepEx;
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    private void setStepExecutionData(final StepExecutionEntity entity, final StepExecutionImpl stepEx) {
+        stepEx.setBatchStatus(entity.getBatchStatus());
+        stepEx.setExitStatus(entity.getExitStatus());
+        stepEx.setStepName(entity.getStepName());
+        stepEx.setReadCount(entity.getRead());
+        stepEx.setWriteCount(entity.getWrite());
+        stepEx.setCommitCount(entity.getCommit());
+        stepEx.setRollbackCount(entity.getRollback());
+        stepEx.setReadSkipCount(entity.getReadSkip());
+        stepEx.setProcessSkipCount(entity.getProcessSkip());
+        stepEx.setFilterCount(entity.getFilter());
+        stepEx.setWriteSkipCount(entity.getWriteSkip());
+        stepEx.setStartTime(entity.getStartTime());
+        stepEx.setEndTime(entity.getEndTime());
+        try {
+            stepEx.setPersistentUserData(deserialize(entity.getPersistentData()));
+        } catch (final Exception e) {
+            throw new PersistenceException(e);
+        }
+    }
+
+    @Override
+    public List<StepExecution> getStepExecutionsForJobExecution(final long execid) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final List<StepExecutionEntity> steps = em.createNamedQuery(StepExecutionEntity.Queries.FIND_BY_EXECUTION, StepExecutionEntity.class)
+                .setParameter("executionId", execid)
+                .getResultList();
+
+            if (steps == null) {
+                return Collections.emptyList();
+            }
+
+            final List<StepExecution> executions = new ArrayList<StepExecution>(steps.size());
+            for (final StepExecutionEntity entity : steps) {
+                final StepExecutionImpl stepEx = new StepExecutionImpl(execid, entity.getId());
+                setStepExecutionData(entity, stepEx);
+                executions.add(stepEx);
+            }
+
+            return executions;
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public void updateStepExecution(final long jobExecId, final StepContextImpl stepContext) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final Object tx = txProvider.start(em);
+            try {
+                final StepExecutionEntity entity = em.find(StepExecutionEntity.class, stepContext.getStepExecutionId());
+                setStepData(em, jobExecId, stepContext,
+                    stepContext.metricsAsMap().get(Metric.MetricType.READ_COUNT.name()).getValue(),
+                    stepContext.metricsAsMap().get(Metric.MetricType.WRITE_COUNT.name()).getValue(),
+                    stepContext.metricsAsMap().get(Metric.MetricType.COMMIT_COUNT.name()).getValue(),
+                    stepContext.metricsAsMap().get(Metric.MetricType.ROLLBACK_COUNT.name()).getValue(),
+                    stepContext.metricsAsMap().get(Metric.MetricType.READ_SKIP_COUNT.name()).getValue(),
+                    stepContext.metricsAsMap().get(Metric.MetricType.PROCESS_SKIP_COUNT.name()).getValue(),
+                    stepContext.metricsAsMap().get(Metric.MetricType.FILTER_COUNT.name()).getValue(),
+                    stepContext.metricsAsMap().get(Metric.MetricType.WRITE_SKIP_COUNT.name()).getValue(),
+                    entity);
+                txProvider.commit(tx);
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public StepExecutionImpl createStepExecution(final long jobExecId, final StepContextImpl stepContext) {
+        final StepExecutionEntity entity = new StepExecutionEntity();
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            setStepData(em, jobExecId, stepContext,
+                stepContext.metricsAsMap().get(Metric.MetricType.READ_COUNT.name()).getValue(),
+                stepContext.metricsAsMap().get(Metric.MetricType.WRITE_COUNT.name()).getValue(),
+                stepContext.metricsAsMap().get(Metric.MetricType.COMMIT_COUNT.name()).getValue(),
+                stepContext.metricsAsMap().get(Metric.MetricType.ROLLBACK_COUNT.name()).getValue(),
+                stepContext.metricsAsMap().get(Metric.MetricType.READ_SKIP_COUNT.name()).getValue(),
+                stepContext.metricsAsMap().get(Metric.MetricType.PROCESS_SKIP_COUNT.name()).getValue(),
+                stepContext.metricsAsMap().get(Metric.MetricType.FILTER_COUNT.name()).getValue(),
+                stepContext.metricsAsMap().get(Metric.MetricType.WRITE_SKIP_COUNT.name()).getValue(),
+                entity);
+
+            final Object tx = txProvider.start(em);
+            try {
+                em.persist(entity);
+                txProvider.commit(tx);
+
+                final StepExecutionImpl stepExecution = new StepExecutionImpl(jobExecId, entity.getId());
+                stepExecution.setStepName(stepContext.getStepName());
+                return stepExecution;
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    private void setStepData(final EntityManager em,
+                             final long jobExecId, final StepContextImpl stepContext,
+                             final long readCount, final long writeCount,
+                             final long commitCount, final long rollbackCount,
+                             final long readSkipCount, final long processSkipCount,
+                             final long filterCount, final long writeSkipCount,
+                             final StepExecutionEntity entity) {
+        entity.setExecution(em.find(JobExecutionEntity.class, jobExecId));
+        entity.setBatchStatus(stepContext.getBatchStatus());
+        entity.setExitStatus(stepContext.getExitStatus());
+        entity.setStepName(stepContext.getStepName());
+        entity.setRead(readCount);
+        entity.setWrite(writeCount);
+        entity.setCommit(commitCount);
+        entity.setRollback(rollbackCount);
+        entity.setReadSkip(readSkipCount);
+        entity.setProcessSkip(processSkipCount);
+        entity.setFilter(filterCount);
+        entity.setWriteSkip(writeSkipCount);
+        entity.setStartTime(stepContext.getStartTimeTS());
+        entity.setEndTime(stepContext.getEndTimeTS());
+        try {
+            entity.setPersistentData(serialize(stepContext.getPersistentUserData()));
+        } catch (final IOException e) {
+            throw new PersistenceException(e);
+        }
+    }
+
+    @Override
+    public long getJobInstanceIdByExecutionId(final long executionId) throws NoSuchJobExecutionException {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            return em.find(JobExecutionEntity.class, executionId).getInstance().getJobInstanceId();
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public Set<Long> jobOperatorGetRunningExecutions(final String jobName) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final List<JobExecutionEntity> list = em.createNamedQuery(JobExecutionEntity.Queries.FIND_RUNNING, JobExecutionEntity.class)
+                .setParameter("name", jobName)
+                .setParameter("statuses", JobExecutionEntity.Queries.RUNNING_STATUSES)
+                .getResultList();
+
+            if (list == null) {
+                return Collections.emptySet();
+            }
+
+            final Set<Long> ids= new HashSet<Long>(list.size());
+            for (final JobExecutionEntity entity : list) {
+                ids.add(entity.getExecutionId());
+            }
+            return ids;
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public List<InternalJobExecution> jobOperatorGetJobExecutions(final long jobInstanceId) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final List<JobExecutionEntity> list = em.createNamedQuery(JobExecutionEntity.Queries.FIND_BY_INSTANCE, JobExecutionEntity.class)
+                .setParameter("instanceId", jobInstanceId).getResultList();
+
+            if (list == null) {
+                return Collections.emptyList();
+            }
+
+            final List<InternalJobExecution> result = new ArrayList<InternalJobExecution>(list.size());
+            for (final JobExecutionEntity entity : list) {
+                final JobExecutionImpl jobEx = new JobExecutionImpl(entity.getExecutionId(), jobInstanceId);
+                jobEx.setCreateTime(entity.getCreateTime());
+                jobEx.setStartTime(entity.getStartTime());
+                jobEx.setEndTime(entity.getEndTime());
+                jobEx.setLastUpdateTime(entity.getUpdateTime());
+                jobEx.setBatchStatus(entity.getBatchStatus().name());
+                jobEx.setExitStatus(entity.getExitStatus());
+                jobEx.setJobName(entity.getInstance().getName());
+
+                result.add(jobEx);
+            }
+            return result;
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public void updateWithFinalExecutionStatusesAndTimestamps(final long key, final BatchStatus batchStatus, final String exitStatus, final Timestamp updatets) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final Object tx = txProvider.start(em);
+            try {
+                final JobExecutionEntity instance = em.find(JobExecutionEntity.class, key);
+                instance.setBatchStatus(batchStatus);
+                instance.setUpdateTime(updatets);
+                instance.setEndTime(updatets);
+                instance.setExitStatus(exitStatus);
+
+                em.merge(instance);
+                txProvider.commit(tx);
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public InternalJobExecution jobOperatorGetJobExecution(final long jobExecutionId) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final JobExecutionEntity instance = em.find(JobExecutionEntity.class, jobExecutionId);
+
+            final JobExecutionImpl jobEx = new JobExecutionImpl(jobExecutionId, instance.getInstance().getJobInstanceId());
+            jobEx.setCreateTime(instance.getCreateTime());
+            jobEx.setStartTime(instance.getStartTime());
+            jobEx.setEndTime(instance.getEndTime());
+            jobEx.setJobParameters(instance.getJobProperties());
+            jobEx.setLastUpdateTime(instance.getUpdateTime());
+            if (instance.getBatchStatus() != null) {
+                jobEx.setBatchStatus(instance.getBatchStatus().name());
+            }
+            jobEx.setExitStatus(instance.getExitStatus());
+            jobEx.setJobName(instance.getInstance().getName());
+            return jobEx;
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public void markJobStarted(final long key, final Timestamp startTS) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final Object tx = txProvider.start(em);
+            try {
+                final JobExecutionEntity instance = em.find(JobExecutionEntity.class, key);
+                instance.setBatchStatus(BatchStatus.STARTED);
+                instance.setStartTime(startTS);
+                instance.setUpdateTime(startTS);
+
+                em.merge(instance);
+                txProvider.commit(tx);
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public Properties getParameters(final long executionId) throws NoSuchJobExecutionException {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            return em.find(JobExecutionEntity.class, executionId).getJobProperties();
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public String jobOperatorQueryJobExecutionExitStatus(final long key) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            return em.find(JobExecutionEntity.class, key).getExitStatus();
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public String jobOperatorQueryJobExecutionBatchStatus(final long key) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            return em.find(JobExecutionEntity.class, key).getBatchStatus().name();
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public Timestamp jobOperatorQueryJobExecutionTimestamp(final long key, final TimestampType timetype) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final JobExecutionEntity entity = em.find(JobExecutionEntity.class, key);
+            if (timetype.equals(TimestampType.CREATE)) {
+                return entity.getCreateTime();
+            } else if (timetype.equals(TimestampType.END)) {
+                return entity.getEndTime();
+            } else if (timetype.equals(TimestampType.LAST_UPDATED)) {
+                return entity.getUpdateTime();
+            } else if (timetype.equals(TimestampType.STARTED)) {
+                return entity.getStartTime();
+            } else {
+                throw new IllegalArgumentException("Unexpected enum value.");
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public RuntimeFlowInSplitExecution createFlowInSplitExecution(final JobInstance jobInstance, final BatchStatus batchStatus) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final JobExecutionEntity instance = new JobExecutionEntity();
+            instance.setCreateTime(new Timestamp(System.currentTimeMillis()));
+            instance.setUpdateTime(instance.getCreateTime());
+            instance.setBatchStatus(batchStatus);
+
+            final Object tx = txProvider.start(em);
+            try {
+                instance.setInstance(em.find(JobInstanceEntity.class, jobInstance.getInstanceId()));
+
+                em.persist(instance);
+                txProvider.commit(tx);
+
+                final RuntimeFlowInSplitExecution jobExecution = new RuntimeFlowInSplitExecution(jobInstance, instance.getExecutionId());
+                jobExecution.setBatchStatus(batchStatus.name());
+                jobExecution.setCreateTime(instance.getCreateTime());
+                jobExecution.setLastUpdateTime(instance.getCreateTime());
+                return jobExecution;
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public long getMostRecentExecutionId(final long jobInstanceId) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            return em.createNamedQuery(JobExecutionEntity.Queries.MOST_RECENT, JobExecutionEntity.class)
+                .setParameter("instanceId", jobInstanceId)
+                .setMaxResults(1)
+                .getSingleResult().getExecutionId();
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public void updateBatchStatusOnly(final long executionId, final BatchStatus batchStatus, final Timestamp timestamp) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final Object tx = txProvider.start(em);
+            final JobExecutionEntity instance = em.find(JobExecutionEntity.class, executionId);
+            instance.setBatchStatus(batchStatus);
+            instance.setUpdateTime(timestamp);
+
+            try {
+                em.merge(instance);
+                txProvider.commit(tx);
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public RuntimeJobExecution createJobExecution(final JobInstance jobInstance, final Properties jobParameters, final BatchStatus batchStatus) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final JobExecutionEntity execution = new JobExecutionEntity();
+            execution.setJobProperties(jobParameters);
+            execution.setCreateTime(new Timestamp(System.currentTimeMillis()));
+            execution.setUpdateTime(execution.getCreateTime());
+            execution.setBatchStatus(batchStatus);
+
+            final Object tx = txProvider.start(em);
+            try {
+                execution.setInstance(em.find(JobInstanceEntity.class, jobInstance.getInstanceId()));
+
+                em.persist(execution);
+                txProvider.commit(tx);
+
+                final RuntimeJobExecution jobExecution = new RuntimeJobExecution(jobInstance, execution.getExecutionId());
+                jobExecution.setBatchStatus(batchStatus.name());
+                jobExecution.setCreateTime(execution.getCreateTime());
+                jobExecution.setLastUpdateTime(execution.getCreateTime());
+                return jobExecution;
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public JobInstance createSubJobInstance(final String name, final String apptag) {
+        return createJobInstance(name, apptag, null);
+    }
+
+    @Override
+    public List<Long> jobOperatorGetJobInstanceIds(final String jobName, final int start, final int count) {
+        return jobOperatorGetJobInstanceIds(jobName, null, start, count);
+    }
+
+    @Override
+    public List<Long> jobOperatorGetJobInstanceIds(final String jobName, final String appTag, final int start, final int count) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final TypedQuery<JobInstanceEntity> query;
+            if (appTag != null) {
+                query = em.createNamedQuery(JobInstanceEntity.Queries.FIND_BY_NAME, JobInstanceEntity.class).setParameter("tag", appTag);
+            } else {
+                query = em.createNamedQuery(JobInstanceEntity.Queries.FIND_BY_NAME, JobInstanceEntity.class);
+            }
+            query.setParameter("name", jobName);
+
+            final List<JobInstanceEntity> resultList = query
+                .setFirstResult(start)
+                .setMaxResults(count)
+                .getResultList();
+
+            if (resultList == null) {
+                return Collections.emptyList();
+            }
+
+            final List<Long> result = new ArrayList<Long>(resultList.size());
+            for (final JobInstanceEntity entity : resultList) {
+                result.add(entity.getJobInstanceId());
+            }
+            return result;
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public JobInstance createJobInstance(final String name, final String apptag, final String jobXml) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final JobInstanceEntity instance = new JobInstanceEntity();
+            instance.setTag(apptag);
+            instance.setName(name);
+            instance.setJobXml(jobXml);
+
+            final Object tx = txProvider.start(em);
+            try {
+                em.persist(instance);
+                txProvider.commit(tx);
+
+                final JobInstanceImpl jobInstance = new JobInstanceImpl(instance.getJobInstanceId(), jobXml);
+                jobInstance.setJobName(name);
+                return jobInstance;
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public int jobOperatorGetJobInstanceCount(final String jobName) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            return em.createNamedQuery(JobInstanceEntity.Queries.COUNT_BY_NAME, Number.class)
+                .setParameter("name", jobName)
+                .getSingleResult().intValue();
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public int jobOperatorGetJobInstanceCount(final String jobName, final String appTag) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            return em.createNamedQuery(JobInstanceEntity.Queries.COUNT_BY_NAME, Number.class)
+                .setParameter("name", jobName)
+                .setParameter("tag", appTag)
+                .getSingleResult().intValue();
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public void updateStepStatus(final long stepExecutionId, final StepStatus stepStatus) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final Object tx = txProvider.start(em);
+            try {
+                final StepExecutionEntity entity = em.find(StepExecutionEntity.class, stepExecutionId);
+                entity.setBatchStatus(stepStatus.getBatchStatus());
+                entity.setExitStatus(stepStatus.getExitStatus());
+                entity.setLastRunStepExecutionId(stepStatus.getLastRunStepExecutionId());
+                entity.setNumPartitions(stepStatus.getNumPartitions());
+                entity.setPersistentData(stepStatus.getRawPersistentUserData());
+                entity.setStartCount(stepStatus.getStartCount());
+
+                em.merge(entity);
+                txProvider.commit(tx);
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public StepStatus createStepStatus(final long stepExecId) {
+        return new StepStatus(stepExecId); // instance already created
+    }
+
+    @Override
+    public void updateJobStatus(final long instanceId, final JobStatus jobStatus) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final Object tx = txProvider.start(em);
+            try {
+                final JobInstanceEntity entity = em.find(JobInstanceEntity.class, instanceId);
+                entity.setBatchStatus(jobStatus.getBatchStatus());
+                entity.setStep(jobStatus.getCurrentStepId());
+                entity.setLatestExecution(jobStatus.getLatestExecutionId());
+                entity.setExitStatus(jobStatus.getExitStatus());
+                entity.setRestartOn(jobStatus.getRestartOn());
+                if (jobStatus.getJobInstance() != null) {
+                    entity.setName(jobStatus.getJobInstance().getJobName());
+                }
+                em.merge(entity);
+
+                final List<JobExecutionEntity> executions = em.createNamedQuery(JobExecutionEntity.Queries.FIND_BY_INSTANCE, JobExecutionEntity.class)
+                    .setParameter("instanceId", instanceId)
+                    .getResultList();
+                if (executions != null) {
+                    for (final JobExecutionEntity e : executions) {
+                        e.setInstance(entity);
+                        em.merge(e);
+                    }
+                }
+                txProvider.commit(tx);
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public JobStatus getJobStatus(final long instanceId) {
+        final JobStatus status = new JobStatus(instanceId);
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final Object tx = txProvider.start(em);
+            try {
+                setJobStatusData(status, em.find(JobInstanceEntity.class, instanceId));
+                txProvider.commit(tx);
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+        return status;
+    }
+
+    private void setJobStatusData(final JobStatus status, final JobInstanceEntity entity) {
+        status.setBatchStatus(entity.getBatchStatus());
+        status.setCurrentStepId(entity.getStep());
+        status.setLatestExecutionId(entity.getLatestExecution());
+        status.setExitStatus(entity.getExitStatus());
+        status.setRestartOn(entity.getRestartOn());
+        status.setJobInstance(entity.toJobInstance());
+    }
+
+    @Override
+    public JobStatus createJobStatus(final long jobInstanceId) {
+        return new JobStatus(jobInstanceId); // instance already created
+    }
+
+    @Override
+    public void setCheckpointData(final CheckpointDataKey key, final CheckpointData value) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final Object tx = txProvider.start(em);
+            try {
+                final List<CheckpointEntity> checkpoints = em.createNamedQuery(CheckpointEntity.Queries.FIND, CheckpointEntity.class)
+                    .setParameter("jobInstanceId", key.getJobInstanceId())
+                    .setParameter("stepName", key.getStepName())
+                    .setParameter("type", key.getType())
+                    .getResultList();
+
+                final CheckpointEntity checkpoint;
+                final boolean isNew = checkpoints == null || checkpoints.isEmpty();
+                if (isNew) {
+                    checkpoint = new CheckpointEntity();
+                    checkpoint.setInstance(em.find(JobInstanceEntity.class, key.getJobInstanceId()));
+                    checkpoint.setStepName(key.getStepName());
+                    checkpoint.setType(key.getType());
+                } else {
+                    checkpoint = checkpoints.iterator().next();
+                }
+
+                checkpoint.setData(value.getRestartToken());
+
+                if (isNew) {
+                    em.persist(checkpoint);
+                } else {
+                    em.merge(checkpoint);
+                }
+                txProvider.commit(tx);
+            } catch (final Exception e) {
+                txProvider.rollback(tx, e);
+                throw new BatchContainerRuntimeException(e);
+            }
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public CheckpointData getCheckpointData(final CheckpointDataKey key) {
+        final EntityManager em = emProvider.newEntityManager();
+        try {
+            final CheckpointEntity checkpoint = em.createNamedQuery(CheckpointEntity.Queries.FIND, CheckpointEntity.class)
+                .setParameter("jobInstanceId", key.getJobInstanceId())
+                .setParameter("stepName", key.getStepName())
+                .setParameter("type", key.getType())
+                .getSingleResult();
+
+            final CheckpointData data = new CheckpointData(checkpoint.getInstance().getJobInstanceId(), checkpoint.getStepName(), checkpoint.getType());
+            data.setRestartToken(checkpoint.getData());
+            return data;
+        } catch (final NoResultException nre) {
+            return null;
+        } finally {
+            emProvider.release(em);
+        }
+    }
+
+    @Override
+    public void init(final Properties batchConfig) {
+        final String txProviderClass = batchConfig.getProperty("persistence.jpa.transaction-provider", DefaultTransactionProvider.class.getName());
+        try {
+            txProvider = TransactionProvider.class.cast(Thread.currentThread().getContextClassLoader().loadClass(txProviderClass).newInstance());
+        } catch (final Exception e) {
+            throw new BatchContainerRuntimeException(e);
+        }
+        txProvider.init(batchConfig);
+
+        final String providerClass = batchConfig.getProperty("persistence.jpa.entity-manager-provider", DefaultEntityManagerProvider.class.getName());
+        try {
+            emProvider = EntityManagerProvider.class.cast(Thread.currentThread().getContextClassLoader().loadClass(providerClass).newInstance());
+        } catch (final Exception e) {
+            throw new BatchContainerRuntimeException(e);
+        }
+        emProvider.init(batchConfig);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-batchee/blob/efa64877/jbatch/src/main/java/org/apache/batchee/container/services/persistence/MemoryPersistenceManager.java
----------------------------------------------------------------------
diff --git a/jbatch/src/main/java/org/apache/batchee/container/services/persistence/MemoryPersistenceManager.java b/jbatch/src/main/java/org/apache/batchee/container/services/persistence/MemoryPersistenceManager.java
new file mode 100644
index 0000000..34ab68d
--- /dev/null
+++ b/jbatch/src/main/java/org/apache/batchee/container/services/persistence/MemoryPersistenceManager.java
@@ -0,0 +1,654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.batchee.container.services.persistence;
+
+import org.apache.batchee.container.impl.JobExecutionImpl;
+import org.apache.batchee.container.impl.JobInstanceImpl;
+import org.apache.batchee.container.impl.MetricImpl;
+import org.apache.batchee.container.impl.StepContextImpl;
+import org.apache.batchee.container.impl.StepExecutionImpl;
+import org.apache.batchee.container.impl.controller.PartitionedStepBuilder;
+import org.apache.batchee.container.impl.controller.chunk.CheckpointData;
+import org.apache.batchee.container.impl.controller.chunk.CheckpointDataKey;
+import org.apache.batchee.container.impl.jobinstance.RuntimeFlowInSplitExecution;
+import org.apache.batchee.container.impl.jobinstance.RuntimeJobExecution;
+import org.apache.batchee.container.services.InternalJobExecution;
+import org.apache.batchee.container.status.JobStatus;
+import org.apache.batchee.container.status.StepStatus;
+import org.apache.batchee.spi.PersistenceManagerService;
+
+import javax.batch.operations.NoSuchJobExecutionException;
+import javax.batch.runtime.BatchStatus;
+import javax.batch.runtime.JobInstance;
+import javax.batch.runtime.Metric;
+import javax.batch.runtime.StepExecution;
+import java.io.Serializable;
+import java.sql.Timestamp;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class MemoryPersistenceManager implements PersistenceManagerService {
+    private static final Collection<BatchStatus> RUNNING_STATUSES = new CopyOnWriteArrayList<BatchStatus>() {{
+        add(BatchStatus.STARTED);
+        add(BatchStatus.STARTING);
+        add(BatchStatus.STOPPING);
+    }};
+
+    protected static interface Structures extends Serializable {
+        static class JobInstanceData implements Structures {
+            protected JobInstanceImpl instance;
+            protected String tag;
+            protected JobStatus status;
+            protected final List<ExecutionInstanceData> executions = new LinkedList<ExecutionInstanceData>();
+            protected final Collection<CheckpointDataKey> checkpoints = new LinkedList<CheckpointDataKey>();
+        }
+
+        static class ExecutionInstanceData implements Structures {
+            protected final List<StepExecution> stepExecutions = new LinkedList<StepExecution>();
+            protected JobExecutionImpl execution;
+        }
+
+        static class StepExecutionInstanceData implements Structures {
+            protected ExecutionInstanceData jobExec;
+            protected StepExecutionImpl execution;
+            protected StepStatus status;
+        }
+    }
+
+    static class Data {
+        protected final AtomicLong lastCleanedJobInstanceId = new AtomicLong(0);
+        protected final AtomicLong jobInstanceIdGenerator = new AtomicLong();
+        protected final AtomicLong executionInstanceIdGenerator = new AtomicLong();
+        protected final AtomicLong stepExecutionIdGenerator = new AtomicLong();
+
+        protected final Map<CheckpointDataKey, CheckpointData> checkpointData = new ConcurrentHashMap<CheckpointDataKey, CheckpointData>();
+        protected final Map<Long, Structures.JobInstanceData> jobInstanceData = new ConcurrentHashMap<Long, Structures.JobInstanceData>();
+        protected final Map<Long, Structures.ExecutionInstanceData> executionInstanceData = new ConcurrentHashMap<Long, Structures.ExecutionInstanceData>();
+        protected final Map<Long, Structures.StepExecutionInstanceData> stepExecutionInstanceData = new ConcurrentHashMap<Long, Structures.StepExecutionInstanceData>();
+    }
+
+    private static final Data GLOBAL_DATA = new Data();
+
+    private Data data;
+    private int maxSize;
+
+    // an extension point to be able to use whatever map you want, a distributed/filtered/size-limited one for instance
+    protected Data newData() {
+        return new Data();
+    }
+
+    @Override
+    public void init(final Properties batchConfig) {
+        if ("true".equalsIgnoreCase(batchConfig.getProperty("persistence.memory.global", "false"))) {
+            data = GLOBAL_DATA;
+        } else {
+            data = newData();
+        }
+        maxSize = Integer.parseInt(batchConfig.getProperty("persistence.memory.max-jobs-instances", "1000"));
+    }
+
+    @Override
+    public int jobOperatorGetJobInstanceCount(final String jobName) {
+        return jobOperatorGetJobInstanceCount(jobName, null);
+    }
+
+    @Override
+    public int jobOperatorGetJobInstanceCount(final String jobName, final String appTag) {
+        final boolean hasTag = appTag != null;
+        int i = 0;
+        for (final Structures.JobInstanceData jobInstanceData : data.jobInstanceData.values()) {
+            if (jobInstanceData.instance.getJobName().equals(jobName) && (!hasTag || appTag.equals(jobInstanceData.tag))) {
+                i++;
+            }
+        }
+        return i;
+    }
+
+    @Override
+    public Map<Long, String> jobOperatorGetExternalJobInstanceData() {
+        final Map<Long, String> out = new HashMap<Long, String>();
+        for (final Structures.JobInstanceData jobInstanceData : data.jobInstanceData.values()) {
+            if (jobInstanceData.instance.getJobName() != null && !jobInstanceData.instance.getJobName().startsWith(PartitionedStepBuilder.JOB_ID_SEPARATOR)) {
+                out.put(jobInstanceData.instance.getInstanceId(), jobInstanceData.instance.getJobName());
+            }
+        }
+        return out;
+    }
+
+    @Override
+    public List<Long> jobOperatorGetJobInstanceIds(final String jobName, final int start, final int count) {
+        return jobOperatorGetJobInstanceIds(jobName, null, start, count);
+    }
+
+    @Override
+    public List<Long> jobOperatorGetJobInstanceIds(final String jobName, final String appTag, final int start, final int count) {
+        final List<Long> out = new LinkedList<Long>();
+        for (final Structures.JobInstanceData jobInstanceData : data.jobInstanceData.values()) {
+            if (jobInstanceData.instance.getJobName() != null && jobInstanceData.instance.getJobName().equals(jobName)) {
+                final boolean hasTag = appTag != null;
+                if (!hasTag || appTag.equals(jobInstanceData.tag)) {
+                    out.add(jobInstanceData.instance.getInstanceId());
+                }
+            }
+        }
+
+        // sorting could be optimized a bit but is it necessary for this impl?
+        Collections.sort(out);
+        Collections.reverse(out);
+
+        if (out.size() > 0) {
+            try {
+                return out.subList(start, start + count);
+            } catch (final IndexOutOfBoundsException oobEx) {
+                return out.subList(start, out.size());
+            }
+        }
+
+        return out;
+    }
+
+    @Override
+    public Timestamp jobOperatorQueryJobExecutionTimestamp(final long key, final TimestampType timestampType) {
+        return null; // avoid infinite loops
+    }
+
+    @Override
+    public String jobOperatorQueryJobExecutionBatchStatus(final long key) {
+        return null; // avoid infinite loops
+    }
+
+    @Override
+    public String jobOperatorQueryJobExecutionExitStatus(final long key) {
+        return null; // avoid infinite loops
+    }
+
+    @Override
+    public List<StepExecution> getStepExecutionsForJobExecution(final long execid) {
+        final Structures.ExecutionInstanceData executionInstanceData = data.executionInstanceData.get(execid);
+        synchronized (executionInstanceData.stepExecutions) {
+            return executionInstanceData.stepExecutions;
+        }
+    }
+
+    @Override
+    public void updateBatchStatusOnly(final long executionId, final BatchStatus batchStatus, final Timestamp timestamp) {
+        final Structures.ExecutionInstanceData toUpdate = data.executionInstanceData.get(executionId);
+        toUpdate.execution.setBatchStatus(batchStatus.name());
+        toUpdate.execution.setLastUpdateTime(timestamp);
+    }
+
+    @Override
+    public void markJobStarted(final long key, final Timestamp startTS) {
+        final Structures.ExecutionInstanceData toUpdate = data.executionInstanceData.get(key);
+        toUpdate.execution.setBatchStatus(BatchStatus.STARTED.name());
+        toUpdate.execution.setLastUpdateTime(startTS);
+        toUpdate.execution.setStartTime(startTS);
+    }
+
+    @Override
+    public void updateWithFinalExecutionStatusesAndTimestamps(final long key, final BatchStatus batchStatus, final String exitStatus, final Timestamp updatets) {
+        final Structures.ExecutionInstanceData toUpdate = data.executionInstanceData.get(key);
+        if (batchStatus != null) {
+            toUpdate.execution.setBatchStatus(batchStatus.name());
+        } else {
+            toUpdate.execution.setBatchStatus(null);
+        }
+        toUpdate.execution.setExitStatus(exitStatus);
+        toUpdate.execution.setLastUpdateTime(updatets);
+        toUpdate.execution.setEndTime(updatets);
+    }
+
+    @Override
+    public InternalJobExecution jobOperatorGetJobExecution(final long jobExecutionId) {
+        final Structures.ExecutionInstanceData executionInstanceData = data.executionInstanceData.get(jobExecutionId);
+        if (executionInstanceData == null) {
+            return null;
+        }
+        return executionInstanceData.execution;
+    }
+
+    @Override
+    public Properties getParameters(final long executionId) throws NoSuchJobExecutionException {
+        return data.executionInstanceData.get(executionId).execution.getJobParameters();
+    }
+
+    @Override
+    public List<InternalJobExecution> jobOperatorGetJobExecutions(final long jobInstanceId) {
+        final List<InternalJobExecution> list = new LinkedList<InternalJobExecution>();
+        final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(jobInstanceId);
+        synchronized (jobInstanceData.executions) {
+            for (final Structures.ExecutionInstanceData executionInstanceData : jobInstanceData.executions) {
+                list.add(executionInstanceData.execution);
+            }
+        }
+        return list;
+    }
+
+    @Override
+    public Set<Long> jobOperatorGetRunningExecutions(final String jobName) {
+        Structures.JobInstanceData jobInstanceData = null;
+        for (final Structures.JobInstanceData instanceData : data.jobInstanceData.values()) {
+            if (instanceData.instance.getJobName().equals(jobName)) {
+                jobInstanceData = instanceData;
+                break;
+            }
+        }
+
+        if (jobInstanceData == null) {
+            return Collections.emptySet();
+        }
+
+        final Set<Long> set = new HashSet<Long>();
+
+        synchronized (jobInstanceData.executions) {
+            for (final Structures.ExecutionInstanceData executionInstanceData : jobInstanceData.executions) {
+                if (RUNNING_STATUSES.contains(executionInstanceData.execution.getBatchStatus())) {
+                    set.add(executionInstanceData.execution.getExecutionId());
+                }
+            }
+        }
+        return set;
+    }
+
+    @Override
+    public JobStatus getJobStatusFromExecution(final long executionId) {
+        final Structures.ExecutionInstanceData executionInstanceData = data.executionInstanceData.get(executionId);
+        if (executionInstanceData == null) {
+            return null;
+        }
+
+        final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(executionInstanceData.execution.getInstanceId());
+        if (jobInstanceData == null) {
+            return null;
+        }
+        return jobInstanceData.status;
+    }
+
+    @Override
+    public long getJobInstanceIdByExecutionId(final long executionId) throws NoSuchJobExecutionException {
+        return data.executionInstanceData.get(executionId).execution.getInstanceId();
+    }
+
+    @Override
+    public JobInstance createJobInstance(final String name, final String apptag, final String jobXml) {
+        final JobInstanceImpl jobInstance = new JobInstanceImpl(data.jobInstanceIdGenerator.getAndIncrement(), jobXml);
+        jobInstance.setJobName(name);
+
+        final Structures.JobInstanceData jobInstanceData = new Structures.JobInstanceData();
+        jobInstanceData.tag = apptag;
+        jobInstanceData.instance = jobInstance;
+        if (maxSize > 0 && data.jobInstanceData.size() >= maxSize) {
+            synchronized (this) {
+                while (data.jobInstanceData.size() >= maxSize) {
+                    cleanUp(data.lastCleanedJobInstanceId.getAndIncrement());
+                }
+            }
+        }
+        data.jobInstanceData.put(jobInstance.getInstanceId(), jobInstanceData);
+
+        return jobInstance;
+    }
+
+    @Override
+    public RuntimeJobExecution createJobExecution(final JobInstance jobInstance, final Properties jobParameters, final BatchStatus batchStatus) {
+        final Timestamp now = new Timestamp(System.currentTimeMillis());
+
+        final Structures.ExecutionInstanceData executionInstanceData = createRuntimeJobExecutionEntry(jobInstance, jobParameters, batchStatus, now);
+        executionInstanceData.execution.setJobName(jobInstance.getJobName());
+
+        final RuntimeJobExecution jobExecution = new RuntimeJobExecution(jobInstance, executionInstanceData.execution.getExecutionId());
+        jobExecution.setBatchStatus(batchStatus.name());
+        jobExecution.setCreateTime(now);
+        jobExecution.setLastUpdateTime(now);
+
+        return jobExecution;
+    }
+
+    private Structures.ExecutionInstanceData createRuntimeJobExecutionEntry(final JobInstance jobInstance, final Properties jobParameters, final BatchStatus batchStatus, final Timestamp now) {
+        final Structures.ExecutionInstanceData executionInstanceData = new Structures.ExecutionInstanceData();
+        final long id = data.executionInstanceIdGenerator.getAndIncrement();
+        executionInstanceData.execution = new JobExecutionImpl(id, jobInstance.getInstanceId());
+        executionInstanceData.execution.setExecutionId(id);
+        executionInstanceData.execution.setInstanceId(jobInstance.getInstanceId());
+        executionInstanceData.execution.setBatchStatus(batchStatus.name());
+        executionInstanceData.execution.setCreateTime(now);
+        executionInstanceData.execution.setLastUpdateTime(now);
+        executionInstanceData.execution.setJobParameters(jobParameters);
+
+        data.executionInstanceData.put(id, executionInstanceData);
+        final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(jobInstance.getInstanceId());
+        synchronized (jobInstanceData.executions) {
+            jobInstanceData.executions.add(executionInstanceData);
+        }
+        return executionInstanceData;
+    }
+
+    @Override
+    public StepExecutionImpl createStepExecution(final long rootJobExecId, final StepContextImpl stepContext) {
+        final String batchStatus = stepContext.getBatchStatus() == null ? BatchStatus.STARTING.name() : stepContext.getBatchStatus().name();
+        final String exitStatus = stepContext.getExitStatus();
+        final String stepName = stepContext.getStepName();
+
+        long readCount = 0;
+        long writeCount = 0;
+        long commitCount = 0;
+        long rollbackCount = 0;
+        long readSkipCount = 0;
+        long processSkipCount = 0;
+        long filterCount = 0;
+        long writeSkipCount = 0;
+        Timestamp startTime = stepContext.getStartTimeTS();
+        Timestamp endTime = stepContext.getEndTimeTS();
+
+        final Metric[] metrics = stepContext.getMetrics();
+        for (final Metric metric : metrics) {
+            if (metric.getType().equals(MetricImpl.MetricType.READ_COUNT)) {
+                readCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.WRITE_COUNT)) {
+                writeCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.PROCESS_SKIP_COUNT)) {
+                processSkipCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.COMMIT_COUNT)) {
+                commitCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.ROLLBACK_COUNT)) {
+                rollbackCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.READ_SKIP_COUNT)) {
+                readSkipCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.FILTER_COUNT)) {
+                filterCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.WRITE_SKIP_COUNT)) {
+                writeSkipCount = metric.getValue();
+            }
+        }
+        final Serializable persistentData = stepContext.getPersistentUserData();
+
+        return createStepExecution(rootJobExecId, batchStatus, exitStatus, stepName, readCount,
+            writeCount, commitCount, rollbackCount, readSkipCount, processSkipCount, filterCount, writeSkipCount, startTime,
+            endTime, persistentData);
+    }
+
+    private StepExecutionImpl createStepExecution(final long rootJobExecId, final String batchStatus, final String exitStatus, final String stepName,
+                                                  final long readCount, final long writeCount, final long commitCount, final long rollbackCount,
+                                                  final long readSkipCount, final long processSkipCount, final long filterCount, final long writeSkipCount,
+                                                  final Timestamp startTime, final Timestamp endTime, final Serializable persistentData) {
+
+        final StepExecutionImpl stepExecution = new StepExecutionImpl(rootJobExecId, data.stepExecutionIdGenerator.getAndIncrement());
+        stepExecution.setStepName(stepName);
+        final Structures.ExecutionInstanceData executionInstanceData = data.executionInstanceData.get(rootJobExecId);
+        synchronized (executionInstanceData.stepExecutions) {
+            executionInstanceData.stepExecutions.add(stepExecution);
+        }
+
+        final Structures.StepExecutionInstanceData stepExecutionInstanceData = new Structures.StepExecutionInstanceData();
+        stepExecutionInstanceData.execution = stepExecution;
+        updateStepExecutionInstanceData(executionInstanceData, batchStatus, exitStatus, stepName,
+            readCount, writeCount, commitCount, rollbackCount,
+            readSkipCount, processSkipCount, filterCount, writeSkipCount,
+            startTime, endTime, persistentData,
+            stepExecutionInstanceData);
+        data.stepExecutionInstanceData.put(stepExecution.getStepExecutionId(), stepExecutionInstanceData);
+
+        return stepExecution;
+    }
+
+    private void updateStepExecutionInstanceData(final Structures.ExecutionInstanceData exec,
+                                                 final String batchStatus,final  String exitStatus, final String stepName,
+                                                 final long readCount, final long writeCount, final long commitCount, final long rollbackCount,
+                                                 final long readSkipCount, final long processSkipCount, final long filterCount, final long writeSkipCount,
+                                                 final Timestamp startTime, final Timestamp endTime,
+                                                 final Serializable persistentData,
+                                                 final Structures.StepExecutionInstanceData stepExecutionInstanceData) {
+        stepExecutionInstanceData.jobExec = exec;
+        stepExecutionInstanceData.execution.setExitStatus(exitStatus);
+        stepExecutionInstanceData.execution.setBatchStatus(BatchStatus.valueOf(batchStatus));
+        stepExecutionInstanceData.execution.setRollbackCount(rollbackCount);
+        stepExecutionInstanceData.execution.setStepName(stepName);
+        stepExecutionInstanceData.execution.setReadCount(readCount);
+        stepExecutionInstanceData.execution.setWriteCount(writeCount);
+        stepExecutionInstanceData.execution.setCommitCount(commitCount);
+        stepExecutionInstanceData.execution.setRollbackCount(rollbackCount);
+        stepExecutionInstanceData.execution.setReadSkipCount(readSkipCount);
+        stepExecutionInstanceData.execution.setProcessSkipCount(processSkipCount);
+        stepExecutionInstanceData.execution.setWriteSkipCount(writeSkipCount);
+        stepExecutionInstanceData.execution.setFilterCount(filterCount);
+        stepExecutionInstanceData.execution.setWriteSkipCount(writeSkipCount);
+        stepExecutionInstanceData.execution.setStartTime(startTime);
+        stepExecutionInstanceData.execution.setEndTime(endTime);
+        stepExecutionInstanceData.execution.setPersistentUserData(persistentData);
+    }
+
+    @Override
+    public void updateStepExecution(final long jobExecId, final StepContextImpl stepContext) {
+        final long stepExecutionId = stepContext.getStepExecutionId();
+        final String batchStatus = stepContext.getBatchStatus() == null ? BatchStatus.STARTING.name() : stepContext.getBatchStatus().name();
+        final String exitStatus = stepContext.getExitStatus();
+        final String stepName = stepContext.getStepName();
+
+        long readCount = 0;
+        long writeCount = 0;
+        long commitCount = 0;
+        long rollbackCount = 0;
+        long readSkipCount = 0;
+        long processSkipCount = 0;
+        long filterCount = 0;
+        long writeSkipCount = 0;
+        Timestamp startTime = stepContext.getStartTimeTS();
+        Timestamp endTime = stepContext.getEndTimeTS();
+
+        final Metric[] metrics = stepContext.getMetrics();
+        for (final Metric metric : metrics) {
+            if (metric.getType().equals(MetricImpl.MetricType.READ_COUNT)) {
+                readCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.WRITE_COUNT)) {
+                writeCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.PROCESS_SKIP_COUNT)) {
+                processSkipCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.COMMIT_COUNT)) {
+                commitCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.ROLLBACK_COUNT)) {
+                rollbackCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.READ_SKIP_COUNT)) {
+                readSkipCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.FILTER_COUNT)) {
+                filterCount = metric.getValue();
+            } else if (metric.getType().equals(MetricImpl.MetricType.WRITE_SKIP_COUNT)) {
+                writeSkipCount = metric.getValue();
+            }
+        }
+        final Serializable persistentData = stepContext.getPersistentUserData();
+
+        updateStepExecution(stepExecutionId, jobExecId, batchStatus, exitStatus, stepName, readCount,
+            writeCount, commitCount, rollbackCount, readSkipCount, processSkipCount, filterCount,
+            writeSkipCount, startTime, endTime, persistentData);
+    }
+
+    private void updateStepExecution(final long stepExecutionId, final long jobExecId, final String batchStatus, final String exitStatus, final String stepName,
+                                     final long readCount, final long writeCount, final long commitCount, final long rollbackCount,
+                                     final long readSkipCount, final long processSkipCount, final long filterCount, final long writeSkipCount,
+                                     final Timestamp startTime, final Timestamp endTime, final Serializable persistentData) {
+        final Structures.ExecutionInstanceData executionInstanceData = data.executionInstanceData.get(jobExecId);
+        if (executionInstanceData == null) {
+            return;
+        }
+
+        synchronized (executionInstanceData.stepExecutions) {
+            for (final StepExecution execution : executionInstanceData.stepExecutions) {
+                if (execution.getStepExecutionId() == stepExecutionId) {
+                    updateStepExecutionInstanceData(executionInstanceData, batchStatus, exitStatus, stepName,
+                        readCount, writeCount, commitCount, rollbackCount,
+                        readSkipCount, processSkipCount, filterCount, writeSkipCount,
+                        startTime, endTime, persistentData,
+                        data.stepExecutionInstanceData.get(stepExecutionId));
+                    break;
+                }
+            }
+        }
+    }
+
+    @Override
+    public JobStatus createJobStatus(final long jobInstanceId) {
+        final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(jobInstanceId);
+        jobInstanceData.status = new JobStatus(jobInstanceId);
+        return jobInstanceData.status;
+    }
+
+    @Override
+    public JobStatus getJobStatus(final long instanceId) {
+        final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(instanceId);
+        if (jobInstanceData == null) {
+            return null;
+        }
+        return jobInstanceData.status;
+    }
+
+    @Override
+    public void updateJobStatus(final long instanceId, final JobStatus jobStatus) {
+        final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(instanceId);
+        if (jobInstanceData != null) {
+            jobInstanceData.status = jobStatus;
+        }
+    }
+
+    @Override
+    public StepStatus createStepStatus(final long stepExecId) {
+        final StepStatus stepStatus = new StepStatus(stepExecId);
+        data.stepExecutionInstanceData.get(stepExecId).status = stepStatus;
+        return stepStatus;
+    }
+
+    @Override
+    public StepStatus getStepStatus(final long instanceId, final String stepName) {
+        final Map<Date, StepStatus> statusMap = new TreeMap<Date, StepStatus>(ReverseDateComparator.INSTANCE);
+        for (final Structures.StepExecutionInstanceData stepExecutionInstanceData : data.stepExecutionInstanceData.values()) {
+            if (stepExecutionInstanceData.jobExec.execution.getInstanceId() == instanceId
+                    && stepExecutionInstanceData.execution.getStepName().equals(stepName)
+                    && stepExecutionInstanceData.status != null) {
+                statusMap.put(stepExecutionInstanceData.jobExec.execution.getCreateTime(), stepExecutionInstanceData.status);
+            }
+        }
+        if (statusMap.isEmpty()) {
+            return null;
+        }
+        return statusMap.values().iterator().next();
+    }
+
+    @Override
+    public void updateStepStatus(final long stepExecutionId, final StepStatus stepStatus) {
+        final Structures.StepExecutionInstanceData stepExecutionInstanceData = data.stepExecutionInstanceData.get(stepExecutionId);
+        if (stepExecutionInstanceData != null) {
+            stepExecutionInstanceData.status = stepStatus;
+        }
+    }
+
+    @Override
+    public void setCheckpointData(final CheckpointDataKey key, final CheckpointData value) {
+        data.checkpointData.put(key, value);
+        final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(key.getJobInstanceId());
+        synchronized (jobInstanceData.checkpoints) {
+            jobInstanceData.checkpoints.add(key);
+        }
+    }
+
+    @Override
+    public CheckpointData getCheckpointData(final CheckpointDataKey key) {
+        return data.checkpointData.get(key);
+    }
+
+    @Override
+    public long getMostRecentExecutionId(final long jobInstanceId) {
+        final Map<Date, Structures.ExecutionInstanceData> filter = new TreeMap<Date, Structures.ExecutionInstanceData>(ReverseDateComparator.INSTANCE);
+        final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.get(jobInstanceId);
+        synchronized (jobInstanceData.executions) {
+            for (final Structures.ExecutionInstanceData exec : jobInstanceData.executions) {
+                if (exec.execution.getInstanceId() == jobInstanceId) {
+                    filter.put(exec.execution.getCreateTime(), exec);
+                }
+            }
+        }
+        if (filter.isEmpty()) {
+            return -1;
+        }
+        return filter.values().iterator().next().execution.getExecutionId();
+    }
+
+    @Override
+    public JobInstance createSubJobInstance(final String name, final String apptag) {
+        return createJobInstance(name, apptag, null);
+    }
+
+    @Override
+    public RuntimeFlowInSplitExecution createFlowInSplitExecution(final JobInstance jobInstance, final BatchStatus batchStatus) {
+        final Timestamp now = new Timestamp(System.currentTimeMillis());
+        final Structures.ExecutionInstanceData executionInstanceData = createRuntimeJobExecutionEntry(jobInstance, null, batchStatus, now);
+        final RuntimeFlowInSplitExecution flowExecution = new RuntimeFlowInSplitExecution(jobInstance, executionInstanceData.execution.getExecutionId());
+        flowExecution.setBatchStatus(batchStatus.name());
+        flowExecution.setCreateTime(now);
+        flowExecution.setLastUpdateTime(now);
+        return flowExecution;
+    }
+
+    @Override
+    public StepExecution getStepExecutionByStepExecutionId(final long stepExecId) {
+        return data.stepExecutionInstanceData.get(stepExecId).execution;
+    }
+
+    @Override
+    public void cleanUp(final long instanceId) {
+        final Structures.JobInstanceData jobInstanceData = data.jobInstanceData.remove(instanceId);
+        if (jobInstanceData == null) {
+            return;
+        }
+
+        synchronized (jobInstanceData.executions) {
+            for (final Structures.ExecutionInstanceData executionInstanceData : jobInstanceData.executions) {
+                data.executionInstanceData.remove(executionInstanceData.execution.getExecutionId());
+                synchronized (executionInstanceData.stepExecutions) {
+                    for (final StepExecution stepExecution : executionInstanceData.stepExecutions) {
+                        data.stepExecutionInstanceData.remove(stepExecution.getStepExecutionId());
+                    }
+                }
+            }
+        }
+        synchronized (jobInstanceData.checkpoints) {
+            for (final CheckpointDataKey key : jobInstanceData.checkpoints) {
+                data.checkpointData.remove(key);
+            }
+        }
+    }
+
+    private static class ReverseDateComparator implements Comparator<Date> {
+        public static final ReverseDateComparator INSTANCE = new ReverseDateComparator();
+
+        @Override
+        public int compare(final Date o1, final Date o2) {
+            return o2.compareTo(o1);
+        }
+    }
+}