You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2012/12/02 21:01:43 UTC
svn commit: r1416244 - in /manifoldcf/trunk: ./
connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/
framework/core/src/test/java/org/apache/manifoldcf/core/tests/
tests/webcrawler/src/test/java/org/apache/...
Author: kwright
Date: Sun Dec 2 20:01:41 2012
New Revision: 1416244
URL: http://svn.apache.org/viewvc?rev=1416244&view=rev
Log:
Finish up CONNECTORS-574.
Added:
manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/SessionLoginDerbyIT.java (with props)
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/build.xml
manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
manifoldcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseDerby.java
manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockSessionWebService.java
manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/SessionTester.java
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1416244&r1=1416243&r2=1416244&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sun Dec 2 20:01:41 2012
@@ -3,6 +3,10 @@ $Id$
======================= 1.1-dev =====================
+CONNECTORS-574: Add more Web connector session login stages, and
+a session login test.
+(Karl Wright)
+
CONNECTORS-573: Add the ability to specify path separator characters
in the LiveLink connector.
(David Morana, Karl Wright)
Modified: manifoldcf/trunk/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/build.xml?rev=1416244&r1=1416243&r2=1416244&view=diff
==============================================================================
--- manifoldcf/trunk/build.xml (original)
+++ manifoldcf/trunk/build.xml Sun Dec 2 20:01:41 2012
@@ -2282,6 +2282,10 @@
<ant dir="tests/rss" target="run-load-HSQLDB"/>
</target>
+ <target name="run-webcrawler-tests-derby" depends="build-tests-framework,build-tests-webcrawler-connector,build-tests-nulloutput-connector,calculate-webcrawler-tests-condition" if="webcrawler-tests.include">
+ <ant dir="tests/webcrawler" target="run-derby"/>
+ </target>
+
<target name="run-webcrawler-loadtests-postgresql" depends="build-tests-framework,build-tests-webcrawler-connector,build-tests-nulloutput-connector,calculate-webcrawler-tests-condition" if="webcrawler-tests.include">
<ant dir="tests/webcrawler" target="run-load-postgresql"/>
</target>
@@ -2474,7 +2478,7 @@
<target name="run-tests-derby-lgpl-connectors" depends="run-tests-derby-jcifs-connector"/>
<target name="run-tests-derby-proprietary-connectors" depends="run-tests-derby-documentum-connector,run-tests-derby-filenet-connector,run-tests-derby-livelink-connector,run-tests-derby-memex-connector,run-tests-derby-meridio-connector,run-tests-derby-sharepoint-connector"/>
- <target name="end-to-end-tests-derby" depends="run-filesystem-tests-derby,run-wiki-tests-derby,run-alfresco-tests-derby,run-cmis-tests-derby,run-sharepoint-tests-derby"/>
+ <target name="end-to-end-tests-derby" depends="run-filesystem-tests-derby,run-webcrawler-tests-derby,run-wiki-tests-derby,run-alfresco-tests-derby,run-cmis-tests-derby,run-sharepoint-tests-derby"/>
<target name="run-tests-postgresql-open-connectors" depends="run-tests-postgresql-activedirectory-connector,run-tests-postgresql-ldap-connector,run-tests-postgresql-alfresco-connector,run-tests-postgresql-cmis-connector,run-tests-postgresql-filesystem-connector,run-tests-postgresql-nullauthority-connector,run-tests-postgresql-nulloutput-connector,run-tests-postgresql-rss-connector,run-tests-postgresql-solr-connector,run-tests-postgresql-webcrawler-connector,run-tests-postgresql-wiki-connector,run-tests-postgresql-jdbc-connector,run-tests-postgresql-opensearchserver-connector,run-tests-postgresql-elasticsearch-connector"/>
<target name="run-tests-postgresql-lgpl-connectors" depends="run-tests-postgresql-jcifs-connector"/>
Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java?rev=1416244&r1=1416243&r2=1416244&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java (original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java Sun Dec 2 20:01:41 2012
@@ -626,6 +626,11 @@ public class WebcrawlerConnector extends
// later to detect redirection to login.
SequenceCredentials sessionCredential = getSequenceCredential(documentIdentifier);
+ if (Logging.connectors.isDebugEnabled())
+ {
+ Logging.connectors.debug("Web: For document identifier '"+documentIdentifier+"' found session credential key '"+sessionCredential.getSequenceKey()+"'");
+ }
+
// Set up the initial state and state variables.
int sessionState = SESSIONSTATE_NORMAL;
String currentURI = documentIdentifier;
Modified: manifoldcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseDerby.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseDerby.java?rev=1416244&r1=1416243&r2=1416244&view=diff
==============================================================================
--- manifoldcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseDerby.java (original)
+++ manifoldcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseDerby.java Sun Dec 2 20:01:41 2012
@@ -45,6 +45,7 @@ public class BaseDerby extends Base
" <property name=\"org.apache.manifoldcf.crawler.expirethreads\" value=\"10\"/>\n" +
" <property name=\"org.apache.manifoldcf.crawler.cleanupthreads\" value=\"10\"/>\n" +
" <property name=\"org.apache.manifoldcf.crawler.deletethreads\" value=\"10\"/>\n" +
+ //" <property name=\"org.apache.manifoldcf.connectors\" value=\"DEBUG\"/>\n" +
" <property name=\"org.apache.manifoldcf.database.maxhandles\" value=\"80\"/>\n"
);
}
Modified: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockSessionWebService.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockSessionWebService.java?rev=1416244&r1=1416243&r2=1416244&view=diff
==============================================================================
--- manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockSessionWebService.java (original)
+++ manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/MockSessionWebService.java Sun Dec 2 20:01:41 2012
@@ -43,9 +43,10 @@ public class MockSessionWebService
server.setThreadPool(new QueuedThreadPool(100));
servlet = new SessionWebServlet(numContentDocs,userName,password);
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
+ context.setInitParameter("org.eclipse.jetty.servlet.SessionIdPathParameterName","none");
context.setContextPath("/web");
server.setHandler(context);
- context.addServlet(new ServletHolder(servlet), "/");
+ context.addServlet(new ServletHolder(servlet), "/*");
}
public void start() throws Exception
@@ -203,7 +204,7 @@ public class MockSessionWebService
{
String redirectTarget;
if (returnID == null)
- redirectTarget = "/web/indexpage.html";
+ redirectTarget = "/web/index.html";
else
redirectTarget = "/web/protectedcontent.html?id="+returnID;
res.sendRedirect(redirectTarget);
Added: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/SessionLoginDerbyIT.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/SessionLoginDerbyIT.java?rev=1416244&view=auto
==============================================================================
--- manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/SessionLoginDerbyIT.java (added)
+++ manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/SessionLoginDerbyIT.java Sun Dec 2 20:01:41 2012
@@ -0,0 +1,61 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.webcrawler_tests;
+
+import java.io.*;
+import java.util.*;
+import org.junit.*;
+
+/** Web connector session login test */
+public class SessionLoginDerbyIT extends BaseDerby
+{
+
+ protected SessionTester tester;
+ protected MockSessionWebService webService = null;
+
+ public SessionLoginDerbyIT()
+ {
+ tester = new SessionTester(mcfInstance);
+ }
+
+ // Setup and teardown the mock wiki service
+
+ @Before
+ public void createWebService()
+ throws Exception
+ {
+ webService = new MockSessionWebService(100,"foo","bar");
+ webService.start();
+ }
+
+ @After
+ public void shutdownWebService()
+ throws Exception
+ {
+ if (webService != null)
+ webService.stop();
+ }
+
+ @Test
+ public void sessionCrawl()
+ throws Exception
+ {
+ tester.executeTest();
+ }
+}
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/SessionLoginDerbyIT.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/SessionLoginDerbyIT.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/SessionTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/SessionTester.java?rev=1416244&r1=1416243&r2=1416244&view=diff
==============================================================================
--- manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/SessionTester.java (original)
+++ manifoldcf/trunk/tests/webcrawler/src/test/java/org/apache/manifoldcf/webcrawler_tests/SessionTester.java Sun Dec 2 20:01:41 2012
@@ -83,7 +83,7 @@ public class SessionTester
ConfigurationNode loginPage = new ConfigurationNode(WebcrawlerConfig.NODE_AUTHPAGE);
loginPage.setAttribute(WebcrawlerConfig.ATTR_URLREGEXP,"/loginpage\\.html(\\?|$)");
loginPage.setAttribute(WebcrawlerConfig.ATTR_TYPE,WebcrawlerConfig.ATTRVALUE_FORM);
- loginPage.setAttribute(WebcrawlerConfig.ATTR_MATCHREGEXP,"/loginpage\\.html(\\?|$)");
+ loginPage.setAttribute(WebcrawlerConfig.ATTR_MATCHREGEXP,"");
// Set credentials
ConfigurationNode userParameter = new ConfigurationNode(WebcrawlerConfig.NODE_AUTHPARAMETER);
userParameter.setAttribute(WebcrawlerConfig.ATTR_NAMEREGEXP,"user");
@@ -113,7 +113,7 @@ public class SessionTester
// Now, save
mgr.save(conn);
-
+
// Create a basic null output connection, and save it.
IOutputConnectionManager outputMgr = OutputConnectionManagerFactory.make(tc);
IOutputConnection outputConn = outputMgr.create();