You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/07/15 23:06:52 UTC
svn commit: r1610852 - in /manifoldcf/trunk: ./ framework/
framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/
Author: kwright
Date: Tue Jul 15 21:06:52 2014
New Revision: 1610852
URL: http://svn.apache.org/r1610852
Log:
Fix for CONNECTORS-992
Added:
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java (with props)
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java (with props)
manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java (with props)
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/framework/build.xml
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1610852&r1=1610851&r2=1610852&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Tue Jul 15 21:06:52 2014
@@ -3,6 +3,10 @@ $Id$
======================= 1.7-dev =====================
+CONNECTORS-992: Add a test to exercise ServiceInterruption within
+a connector.
+(Karl Wright)
+
CONNECTORS-990: Revamp IRepositoryConnector API to no longer separate
getDocumentVersions() and processDocuments(). This modification basically
pushes responsibility to determine changes to the repository connector.
Modified: manifoldcf/trunk/framework/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/build.xml?rev=1610852&r1=1610851&r2=1610852&view=diff
==============================================================================
--- manifoldcf/trunk/framework/build.xml (original)
+++ manifoldcf/trunk/framework/build.xml Tue Jul 15 21:06:52 2014
@@ -1657,6 +1657,7 @@
<formatter type="brief" usefile="false"/>
<test name="org.apache.manifoldcf.crawler.tests.SchedulerHSQLDBTest" todir="test-output"/>
+ <test name="org.apache.manifoldcf.crawler.tests.InterruptionHSQLDBTest" todir="test-output"/>
</junit>
</target>
Added: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java?rev=1610852&view=auto
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java (added)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java Tue Jul 15 21:06:52 2014
@@ -0,0 +1,128 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.agents.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test of service interruptions */
+public class InterruptionHSQLDBTest extends ConnectorBaseHSQLDB
+{
+ protected final ManifoldCFInstance mcfInstance;
+ protected InterruptionTester tester;
+
+ public InterruptionHSQLDBTest()
+ {
+ super();
+ mcfInstance = new ManifoldCFInstance("A",false,false);
+ tester = new InterruptionTester(mcfInstance);
+ }
+
+ @Override
+ protected String[] getConnectorClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.crawler.tests.InterruptionRepositoryConnector"};
+ }
+
+ @Override
+ protected String[] getConnectorNames()
+ {
+ return new String[]{"InterruptionConnector"};
+ }
+
+ @Override
+ protected String[] getOutputClasses()
+ {
+ return new String[]{"org.apache.manifoldcf.agents.tests.TestingOutputConnector"};
+ }
+
+ @Override
+ protected String[] getOutputNames()
+ {
+ return new String[]{"NullOutput"};
+ }
+
+ @Test
+ public void interruptionTestRun()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+
+ @Before
+ public void setUp()
+ throws Exception
+ {
+ initializeSystem();
+ try
+ {
+ localReset();
+ }
+ catch (Exception e)
+ {
+ System.out.println("Warning: Preclean failed: "+e.getMessage());
+ }
+ try
+ {
+ localSetUp();
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @After
+ public void cleanUp()
+ throws Exception
+ {
+ Exception currentException = null;
+ // Last, shut down the web applications.
+ // If this is done too soon it closes the database before the rest of the cleanup happens.
+ try
+ {
+ mcfInstance.unload();
+ }
+ catch (Exception e)
+ {
+ if (currentException == null)
+ currentException = e;
+ }
+ try
+ {
+ localCleanUp();
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ throw e;
+ }
+ if (currentException != null)
+ throw currentException;
+ cleanupSystem();
+ }
+
+
+}
Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionHSQLDBTest.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java?rev=1610852&view=auto
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java (added)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java Tue Jul 15 21:06:52 2014
@@ -0,0 +1,103 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+
+/** Connector class to be used by general integration tests that need documents */
+public class InterruptionRepositoryConnector extends org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector
+{
+ public InterruptionRepositoryConnector()
+ {
+ }
+
+ @Override
+ public void addSeedDocuments(ISeedingActivity activities, DocumentSpecification spec,
+ long startTime, long endTime, int jobMode)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ String docCount = "10";
+ for (int i = 0; i < spec.getChildCount(); i++)
+ {
+ SpecificationNode sn = spec.getChild(i);
+ if (sn.getType().equals("documentcount"))
+ docCount = sn.getAttributeValue("count");
+ }
+ int count = Integer.parseInt(docCount);
+
+ for (int i = 0; i < count; i++)
+ {
+ String doc = "test"+i+".txt";
+ activities.addSeedDocument(doc,null);
+ }
+ }
+
+ @Override
+ public String[] getDocumentVersions(String[] documentIdentifiers, String[] oldVersions, IVersionActivity activities,
+ DocumentSpecification spec, int jobMode, boolean usesDefaultAuthority)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ String[] rval = new String[documentIdentifiers.length];
+ for (int i = 0; i < rval.length; i++)
+ {
+ rval[i] = "";
+ }
+ return rval;
+ }
+
+ @Override
+ public void processDocuments(String[] documentIdentifiers, String[] versions, IProcessActivity activities,
+ DocumentSpecification spec, boolean[] scanOnly, int jobMode)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ for (int i = 0; i < documentIdentifiers.length; i++)
+ {
+ String documentIdentifier = documentIdentifiers[i];
+ String version = versions[i];
+ if (!scanOnly[i])
+ {
+ if (documentIdentifier.equals("test0.txt"))
+ {
+ // This will emulate one particular document failing (and being skipped)
+ long currentTime = System.currentTimeMillis();
+ throw new ServiceInterruption("Pretending there's a service interruption",
+ null,currentTime+1000L,currentTime+5000L,10,false);
+ }
+ RepositoryDocument rd = new RepositoryDocument();
+ byte[] bytes = documentIdentifier.getBytes(StandardCharsets.UTF_8);
+ rd.setBinary(new ByteArrayInputStream(bytes),bytes.length);
+ try
+ {
+ activities.ingestDocumentWithException(documentIdentifier,version,"http://"+documentIdentifier,rd);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Shouldn't be seeing IOException from binary array input stream: "+e.getMessage(),e);
+ }
+ }
+ }
+ }
+
+}
Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionRepositoryConnector.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java?rev=1610852&view=auto
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java (added)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java Tue Jul 15 21:06:52 2014
@@ -0,0 +1,99 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+/** This is a test whether we can handle service interruptions */
+public class InterruptionTester
+{
+ protected final ManifoldCFInstance instance;
+
+ public InterruptionTester(ManifoldCFInstance instance)
+ {
+ this.instance = instance;
+ }
+
+ public void executeTest()
+ throws Exception
+ {
+ instance.start();
+
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("InterruptionTest Connection");
+ conn.setDescription("InterruptionTest Connection");
+ conn.setClassName("org.apache.manifoldcf.crawler.tests.InterruptionRepositoryConnector");
+ conn.setMaxConnections(100);
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Null Connection");
+ outputConn.setDescription("Null Connection");
+ outputConn.setClassName("org.apache.manifoldcf.agents.tests.TestingOutputConnector");
+ outputConn.setMaxConnections(100);
+ // Now, save
+ outputMgr.save(outputConn);
+
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("InterruptionTest Connection");
+ job.addPipelineStage(-1,true,"Null Connection","");
+ //job.setOutputConnectionName("Null Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Now, start the job, and wait until it is running.
+ jobManager.manualStart(job.getID());
+ instance.waitJobRunningNative(jobManager,job.getID(),30000L);
+
+ // Wait for the job to become inactive. The time should not exceed 10 seconds for the actual crawl.
+ instance.waitJobInactiveNative(jobManager,job.getID(),30000L);
+ // The document will be skipped in the end.
+ if (jobManager.getStatus(job.getID()).getDocumentsProcessed() != 9)
+ throw new Exception("Expected 9 documents, saw "+jobManager.getStatus(job.getID()).getDocumentsProcessed());
+
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ instance.waitJobDeletedNative(jobManager,job.getID(),30000L);
+
+ // Shut down instance2
+ instance.stop();
+ }
+}
Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/InterruptionTester.java
------------------------------------------------------------------------------
svn:keywords = Id