svn commit: r1653004 - in /manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests: ConnectionChangeHSQLDBTest.java ConnectionChangeRepositoryConnector.java ConnectionChangeTester.java

[kw...@apache.org]

Author: kwright
Date: Mon Jan 19 14:09:36 2015
New Revision: 1653004

URL: http://svn.apache.org/r1653004
Log:
Add cache change test

Added:
    manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeHSQLDBTest.java   (with props)
    manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeRepositoryConnector.java   (with props)
    manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeTester.java   (with props)

Added: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeHSQLDBTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeHSQLDBTest.java?rev=1653004&view=auto
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeHSQLDBTest.java (added)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeHSQLDBTest.java Mon Jan 19 14:09:36 2015
@@ -0,0 +1,128 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.agents.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** This is a test of connection caching */
+public class ConnectionChangeHSQLDBTest extends ConnectorBaseHSQLDB
+{
+  protected final ManifoldCFInstance mcfInstance;
+  protected ConnectionChangeTester tester;
+
+  public ConnectionChangeHSQLDBTest()
+  {
+    super();
+    mcfInstance = new ManifoldCFInstance("A",false,false);
+    tester = new ConnectionChangeTester(mcfInstance);
+  }
+  
+  @Override
+  protected String[] getConnectorClasses()
+  {
+    return new String[]{"org.apache.manifoldcf.crawler.tests.ConnectionChangeRepositoryConnector"};
+  }
+  
+  @Override
+  protected String[] getConnectorNames()
+  {
+    return new String[]{"ConnectionChangeConnector"};
+  }
+
+  @Override
+  protected String[] getOutputClasses()
+  {
+    return new String[]{"org.apache.manifoldcf.agents.tests.TestingOutputConnector"};
+  }
+  
+  @Override
+  protected String[] getOutputNames()
+  {
+    return new String[]{"NullOutput"};
+  }
+
+  @Test
+  public void interruptionTestRun()
+    throws Exception
+  {
+    tester.executeTest();
+  }
+  
+  @Before
+  public void setUp()
+    throws Exception
+  {
+    initializeSystem();
+    try
+    {
+      localReset();
+    }
+    catch (Exception e)
+    {
+      System.out.println("Warning: Preclean failed: "+e.getMessage());
+    }
+    try
+    {
+      localSetUp();
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @After
+  public void cleanUp()
+    throws Exception
+  {
+    Exception currentException = null;
+    // Last, shut down the web applications.
+    // If this is done too soon it closes the database before the rest of the cleanup happens.
+    try
+    {
+      mcfInstance.unload();
+    }
+    catch (Exception e)
+    {
+      if (currentException == null)
+        currentException = e;
+    }
+    try
+    {
+      localCleanUp();
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+    if (currentException != null)
+      throw currentException;
+    cleanupSystem();
+  }
+  
+
+}

Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeHSQLDBTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeHSQLDBTest.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeRepositoryConnector.java?rev=1653004&view=auto
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeRepositoryConnector.java (added)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeRepositoryConnector.java Mon Jan 19 14:09:36 2015
@@ -0,0 +1,103 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+
+/** Connector class to be used by integration test that insures caching is not stuck in some way */
+public class ConnectionChangeRepositoryConnector extends org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector
+{
+  public ConnectionChangeRepositoryConnector()
+  {
+  }
+
+  @Override
+  public String addSeedDocuments(ISeedingActivity activities, Specification spec,
+    String lastSeedVersion, long seedTime, int jobMode)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    String docCount = "10";
+    for (int i = 0; i < spec.getChildCount(); i++)
+    {
+      SpecificationNode sn = spec.getChild(i);
+      if (sn.getType().equals("documentcount"))
+        docCount = sn.getAttributeValue("count");
+    }
+    int count = Integer.parseInt(docCount);
+    
+    for (int i = 0; i < count; i++)
+    {
+      String doc = "test"+i+".txt";
+      activities.addSeedDocument(doc,null);
+    }
+    return "";
+  }
+  
+  /** Process a set of documents.
+  * This is the method that should cause each document to be fetched, processed, and the results either added
+  * to the queue of documents for the current job, and/or entered into the incremental ingestion manager.
+  * The document specification allows this class to filter what is done based on the job.
+  * The connector will be connected before this method can be called.
+  *@param documentIdentifiers is the set of document identifiers to process.
+  *@param statuses are the currently-stored document versions for each document in the set of document identifiers
+  * passed in above.
+  *@param activities is the interface this method should use to queue up new document references
+  * and ingest documents.
+  *@param jobMode is an integer describing how the job is being run, whether continuous or once-only.
+  *@param usesDefaultAuthority will be true only if the authority in use for these documents is the default one.
+  */
+  @Override
+  public void processDocuments(String[] documentIdentifiers, IExistingVersions statuses, Specification spec,
+    IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    String blockOnDocument = params.getParameter("proceed");
+    boolean block = (blockOnDocument==null||blockOnDocument.equals("false"));
+    
+    for (int i = 0; i < documentIdentifiers.length; i++)
+    {
+      String documentIdentifier = documentIdentifiers[i];
+      if (block && documentIdentifier.equals("test0.txt"))
+      {
+        // This will emulate one particular document failing (and being skipped)
+        long currentTime = System.currentTimeMillis();
+        throw new ServiceInterruption("Pretending there's a service interruption",
+          null,currentTime+1000L,currentTime+5000L,1000,true);
+      }
+      RepositoryDocument rd = new RepositoryDocument();
+      byte[] bytes = documentIdentifier.getBytes(StandardCharsets.UTF_8);
+      rd.setBinary(new ByteArrayInputStream(bytes),bytes.length);
+      try
+      {
+        activities.ingestDocumentWithException(documentIdentifier,"","http://"+documentIdentifier,rd);
+      }
+      catch (IOException e)
+      {
+        throw new RuntimeException("Shouldn't be seeing IOException from binary array input stream: "+e.getMessage(),e);
+      }
+    }
+  }
+
+}

Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeRepositoryConnector.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeRepositoryConnector.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeTester.java?rev=1653004&view=auto
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeTester.java (added)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeTester.java Mon Jan 19 14:09:36 2015
@@ -0,0 +1,105 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+/** This is a test whether we deal with changes to configuration properly */
+public class ConnectionChangeTester
+{
+  protected final ManifoldCFInstance instance;
+  
+  public ConnectionChangeTester(ManifoldCFInstance instance)
+  {
+    this.instance = instance;
+  }
+  
+  public void executeTest()
+    throws Exception
+  {
+    instance.start();
+    
+    // Hey, we were able to install the file system connector etc.
+    // Now, create a local test job and run it.
+    IThreadContext tc = ThreadContextFactory.make();
+      
+    // Create a basic file system connection, and save it.
+    IRepositoryConnectionManager mgr = RepositoryConnectionManagerFactory.make(tc);
+    IRepositoryConnection conn = mgr.create();
+    conn.setName("ConnectionChangeTest Connection");
+    conn.setDescription("ConnectionChangeTest Connection");
+    conn.setClassName("org.apache.manifoldcf.crawler.tests.ConnectionChangeRepositoryConnector");
+    conn.setMaxConnections(100);
+    // Now, save
+    mgr.save(conn);
+      
+    // Create a basic null output connection, and save it.
+    IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
+    IOutputConnection outputConn = outputMgr.create();
+    outputConn.setName("Null Connection");
+    outputConn.setDescription("Null Connection");
+    outputConn.setClassName("org.apache.manifoldcf.agents.tests.TestingOutputConnector");
+    outputConn.setMaxConnections(100);
+    // Now, save
+    outputMgr.save(outputConn);
+
+    // Create a job.
+    IJobManager jobManager = JobManagerFactory.make(tc);
+    IJobDescription job = jobManager.createJob();
+    job.setDescription("Test Job");
+    job.setConnectionName("InterruptionTest Connection");
+    job.addPipelineStage(-1,true,"Null Connection","");
+    //job.setOutputConnectionName("Null Connection");
+    job.setType(job.TYPE_SPECIFIED);
+    job.setStartMethod(job.START_DISABLE);
+    job.setHopcountMode(job.HOPCOUNT_ACCURATE);
+      
+    // Save the job.
+    jobManager.save(job);
+
+    // Now, start the job, and wait until it is running.
+    jobManager.manualStart(job.getID());
+    instance.waitJobRunningNative(jobManager,job.getID(),30000L);
+    
+    // Now, update the connection to allow the job to finish.
+    conn = mgr.load("ConnectionChangeTest Connection");
+    ConfigParams cp = conn.getConfigParams();
+    cp.setParameter("proceed","true");
+    mgr.save(conn);
+    
+    // Wait for the job to become inactive.  The time should not exceed 10 seconds for the actual crawl.
+    instance.waitJobInactiveNative(jobManager,job.getID(),30000L);
+    // The document will be skipped in the end.
+    if (jobManager.getStatus(job.getID()).getDocumentsProcessed() != 9)
+      throw new Exception("Expected 10 documents, saw "+jobManager.getStatus(job.getID()).getDocumentsProcessed());
+    
+    // Now, delete the job.
+    jobManager.deleteJob(job.getID());
+    instance.waitJobDeletedNative(jobManager,job.getID(),30000L);
+
+    // Shut down instance2
+    instance.stop();
+  }
+}

Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeTester.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ConnectionChangeTester.java
------------------------------------------------------------------------------
    svn:keywords = Id