You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2012/02/05 07:20:52 UTC
svn commit: r1240665 - in /oodt/trunk: ./
workflow/src/main/java/org/apache/oodt/cas/workflow/tools/
workflow/src/test/org/apache/oodt/cas/workflow/tools/
workflow/src/testdata/testinstrepo/
Author: mattmann
Date: Sun Feb 5 06:20:51 2012
New Revision: 1240665
URL: http://svn.apache.org/viewvc?rev=1240665&view=rev
Log:
- fix for OODT-356: Tool to clean Workflow Instance repositories
- also hardened the LuceneWorkflowInstanceRepository so that it's backwards compatible with OODT 0.3 created inst repos
- thanks to bfoster for the review!
Added:
oodt/trunk/workflow/src/main/java/org/apache/oodt/cas/workflow/tools/
oodt/trunk/workflow/src/main/java/org/apache/oodt/cas/workflow/tools/InstanceRepoCleaner.java
oodt/trunk/workflow/src/test/org/apache/oodt/cas/workflow/tools/
oodt/trunk/workflow/src/test/org/apache/oodt/cas/workflow/tools/TestInstanceRepoCleaner.java
oodt/trunk/workflow/src/testdata/testinstrepo/
oodt/trunk/workflow/src/testdata/testinstrepo/_43.cfs (with props)
oodt/trunk/workflow/src/testdata/testinstrepo/deletable (with props)
oodt/trunk/workflow/src/testdata/testinstrepo/segments (with props)
Modified:
oodt/trunk/CHANGES.txt
Modified: oodt/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/oodt/trunk/CHANGES.txt?rev=1240665&r1=1240664&r2=1240665&view=diff
==============================================================================
--- oodt/trunk/CHANGES.txt (original)
+++ oodt/trunk/CHANGES.txt Sun Feb 5 06:20:51 2012
@@ -4,6 +4,8 @@ Apache OODT Change Log
Release 0.4: Current Development
--------------------------------------------
+* OODT-356 Tool to clean Workflow Instance repositories (mattmann, bfoster)
+
* OODT-372 Correct LDAPAuthenticationProvider class name (Shakeh Khudikyan via ahart)
* OODT-256 updateMetadata needed in XmlRpcFileManager (mattmann)
Added: oodt/trunk/workflow/src/main/java/org/apache/oodt/cas/workflow/tools/InstanceRepoCleaner.java
URL: http://svn.apache.org/viewvc/oodt/trunk/workflow/src/main/java/org/apache/oodt/cas/workflow/tools/InstanceRepoCleaner.java?rev=1240665&view=auto
==============================================================================
--- oodt/trunk/workflow/src/main/java/org/apache/oodt/cas/workflow/tools/InstanceRepoCleaner.java (added)
+++ oodt/trunk/workflow/src/main/java/org/apache/oodt/cas/workflow/tools/InstanceRepoCleaner.java Sun Feb 5 06:20:51 2012
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.workflow.tools;
+
+//JDK imports
+import java.net.URL;
+import java.util.Calendar;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+//OODT imports
+import org.apache.oodt.cas.workflow.instrepo.LuceneWorkflowInstanceRepository;
+import org.apache.oodt.cas.workflow.structs.WorkflowInstance;
+import org.apache.oodt.cas.workflow.structs.WorkflowInstancePage;
+import org.apache.oodt.cas.workflow.structs.WorkflowStatus;
+import org.apache.oodt.cas.workflow.system.XmlRpcWorkflowManagerClient;
+import org.apache.oodt.commons.date.DateUtils;
+
+/**
+ *
+ * Cleans a workflow manager instance repository of ghost jobs that will never
+ * execute and cleans up job history and repository status.
+ *
+ * @author mattmann
+ *
+ */
+public class InstanceRepoCleaner {
+
+ /* PGE task statuses */
+ public static final String STAGING_INPUT = "STAGING INPUT";
+
+ public static final String CONF_FILE_BUILD = "BUILDING CONFIG FILE";
+
+ public static final String RUNNING_PGE = "PGE EXEC";
+
+ public static final String CRAWLING = "CRAWLING";
+
+ private static final Logger LOG = Logger.getLogger(InstanceRepoCleaner.class
+ .getName());
+
+ private XmlRpcWorkflowManagerClient wm;
+
+ private LuceneWorkflowInstanceRepository rep;
+
+ public InstanceRepoCleaner() {
+ }
+
+ public InstanceRepoCleaner(String wmUrlStr) throws Exception {
+ this.wm = new XmlRpcWorkflowManagerClient(new URL(wmUrlStr));
+ }
+
+ public void setInstanceRepo(String idxPath) {
+ this.rep = new LuceneWorkflowInstanceRepository(idxPath, 1000);
+ }
+
+ public static void main(String[] args) throws Exception {
+ String usage = "InstanceRepoCleaner [options]\n"
+ + "<workflow manager url>\n" + "--idxPath <path>\n";
+ if (args.length != 1 && args.length != 2) {
+ System.err.println(usage);
+ System.exit(1);
+ }
+
+ InstanceRepoCleaner clean = null;
+ if (args.length == 1) {
+ String wmUrlStr = args[0];
+ clean = new InstanceRepoCleaner(wmUrlStr);
+ } else {
+ String idxPath = args[1];
+ clean = new InstanceRepoCleaner();
+ clean.setInstanceRepo(idxPath);
+ }
+ clean.cleanRepository();
+ }
+
+ public void cleanRepository() throws Exception {
+ WorkflowInstancePage page = wm != null ? wm.getFirstPage() : rep
+ .getFirstPage();
+ while (page != null && page.getPageWorkflows() != null
+ && page.getPageWorkflows().size() > 0) {
+
+ LOG.log(Level.INFO,
+ "Cleaning workflow instances: page: [" + page.getPageNum() + "] of ["
+ + page.getTotalPages() + "]: page size: [" + page.getPageSize()
+ + "]");
+ for (WorkflowInstance inst : (List<WorkflowInstance>) page
+ .getPageWorkflows()) {
+ if (inst.getStatus().equals(WorkflowStatus.CREATED)
+ || inst.getStatus().equals(WorkflowStatus.STARTED)
+ || inst.getStatus().equals(WorkflowStatus.QUEUED)
+ || inst.getStatus().equals(WorkflowStatus.RESMGR_SUBMIT)
+ || inst.getStatus().equals(CONF_FILE_BUILD)
+ || inst.getStatus().equals(CRAWLING)
+ || inst.getStatus().equals(RUNNING_PGE)
+ || inst.getStatus().equals(STAGING_INPUT)) {
+ String endDateTimeIsoStr = DateUtils.toString(Calendar.getInstance());
+ LOG.log(Level.INFO, "Updated workflow instance id: [" + inst.getId()
+ + "]: setting end date time to: [" + endDateTimeIsoStr + "]");
+ LOG.log(Level.INFO, "Existing status: [" + inst.getStatus()
+ + "]: setting to [" + WorkflowStatus.FINISHED + "]");
+ inst.setEndDateTimeIsoStr(endDateTimeIsoStr);
+ if (inst.getStartDateTimeIsoStr() == null
+ || (inst.getStartDateTimeIsoStr() != null && inst
+ .getStartDateTimeIsoStr().equals(""))) {
+ inst.setStartDateTimeIsoStr(endDateTimeIsoStr);
+ }
+ inst.setStatus(WorkflowStatus.FINISHED);
+ if (wm != null) {
+ wm.updateWorkflowInstance(inst);
+ } else {
+ rep.updateWorkflowInstance(inst);
+ }
+ }
+ }
+
+ if (page.isLastPage()) {
+ LOG.log(Level.INFO, "Last set of workflow instances cleaned.");
+ break;
+ }
+
+ page = wm != null ? wm.getNextPage(page) : rep.getNextPage(page);
+
+ }
+
+ }
+
+}
Added: oodt/trunk/workflow/src/test/org/apache/oodt/cas/workflow/tools/TestInstanceRepoCleaner.java
URL: http://svn.apache.org/viewvc/oodt/trunk/workflow/src/test/org/apache/oodt/cas/workflow/tools/TestInstanceRepoCleaner.java?rev=1240665&view=auto
==============================================================================
--- oodt/trunk/workflow/src/test/org/apache/oodt/cas/workflow/tools/TestInstanceRepoCleaner.java (added)
+++ oodt/trunk/workflow/src/test/org/apache/oodt/cas/workflow/tools/TestInstanceRepoCleaner.java Sun Feb 5 06:20:51 2012
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.workflow.tools;
+
+//JDK imports
+import java.io.File;
+import java.util.List;
+
+//APACHE imports
+import org.apache.commons.io.FileUtils;
+
+//OODT imports
+import org.apache.oodt.cas.workflow.instrepo.LuceneWorkflowInstanceRepository;
+import org.apache.oodt.cas.workflow.instrepo.WorkflowInstanceRepository;
+import org.apache.oodt.cas.workflow.structs.WorkflowInstance;
+import org.apache.oodt.cas.workflow.structs.WorkflowStatus;
+import org.apache.oodt.cas.workflow.structs.exceptions.InstanceRepositoryException;
+
+//Junit imports
+import junit.framework.TestCase;
+
+/**
+ *
+ * Test harness for the {@link InstanceRepoCleaner}.
+ *
+ * @author mattmann
+ * @version $Revision$
+ * @since
+ *
+ */
+public class TestInstanceRepoCleaner extends TestCase {
+
+ private String instRepoPath;
+
+ public void testClean() {
+ InstanceRepoCleaner cleaner = new InstanceRepoCleaner();
+ cleaner.setInstanceRepo(instRepoPath);
+ try {
+ cleaner.cleanRepository();
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+
+ WorkflowInstanceRepository repo = new LuceneWorkflowInstanceRepository(
+ instRepoPath, 20);
+ try {
+ assertEquals(10, repo.getNumWorkflowInstances());
+ for (WorkflowInstance inst : (List<WorkflowInstance>) repo
+ .getWorkflowInstances()) {
+ if (!inst.getStatus().equals(WorkflowStatus.FINISHED)) {
+ fail("Workflow Instance: [" + inst.getId()
+ + "] does was not marked as finished by the cleaner: status: ["
+ + inst.getStatus() + "]");
+ }
+ }
+
+ } catch (InstanceRepositoryException e) {
+ fail(e.getMessage());
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see junit.framework.TestCase#setUp()
+ */
+ @Override
+ protected void setUp() throws Exception {
+ // get a temp directory path
+ File tempDir = File.createTempFile("bogus", "txt").getParentFile();
+ FileUtils.copyDirectory(new File("./src/testdata/testinstrepo"), new File(
+ tempDir.getAbsolutePath() + "/" + "testinstrepo"));
+ instRepoPath = tempDir.getAbsolutePath().endsWith("/") ? (tempDir
+ .getAbsolutePath() + "testinstrepo")
+ : (tempDir.getAbsolutePath() + "/" + "testinstrepo");
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see junit.framework.TestCase#tearDown()
+ */
+ @Override
+ protected void tearDown() throws Exception {
+ FileUtils.deleteDirectory(new File(instRepoPath));
+
+ }
+
+}
Added: oodt/trunk/workflow/src/testdata/testinstrepo/_43.cfs
URL: http://svn.apache.org/viewvc/oodt/trunk/workflow/src/testdata/testinstrepo/_43.cfs?rev=1240665&view=auto
==============================================================================
Binary file - no diff available.
Propchange: oodt/trunk/workflow/src/testdata/testinstrepo/_43.cfs
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: oodt/trunk/workflow/src/testdata/testinstrepo/deletable
URL: http://svn.apache.org/viewvc/oodt/trunk/workflow/src/testdata/testinstrepo/deletable?rev=1240665&view=auto
==============================================================================
Binary file - no diff available.
Propchange: oodt/trunk/workflow/src/testdata/testinstrepo/deletable
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: oodt/trunk/workflow/src/testdata/testinstrepo/segments
URL: http://svn.apache.org/viewvc/oodt/trunk/workflow/src/testdata/testinstrepo/segments?rev=1240665&view=auto
==============================================================================
Binary file - no diff available.
Propchange: oodt/trunk/workflow/src/testdata/testinstrepo/segments
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream