You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/02/24 19:54:40 UTC
svn commit: r1449526 [1/2] - in /manifoldcf/trunk: ./
framework/agents/src/main/java/org/apache/manifoldcf/agents/system/
framework/api-servlet/src/main/java/org/apache/manifoldcf/apiservlet/
framework/crawler-ui/src/main/webapp/ framework/pull-agent/s...
Author: kwright
Date: Sun Feb 24 18:54:40 2013
New Revision: 1449526
URL: http://svn.apache.org/r1449526
Log:
Add ability to support history and queue reports via API. Part of CONNECTORS-63.
Modified:
manifoldcf/trunk/ (props changed)
manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
manifoldcf/trunk/framework/api-servlet/src/main/java/org/apache/manifoldcf/apiservlet/APIServlet.java
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/maxactivityreport.jsp
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/maxbandwidthreport.jsp
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/resultreport.jsp
manifoldcf/trunk/framework/crawler-ui/src/main/webapp/simplereport.jsp
manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
Propchange: manifoldcf/trunk/
------------------------------------------------------------------------------
Merged /manifoldcf/branches/CONNECTORS-63:r1449286-1449525
Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java?rev=1449526&r1=1449525&r2=1449526&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java (original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java Sun Feb 24 18:54:40 2013
@@ -265,7 +265,17 @@ public class ManifoldCF extends org.apac
return null;
return node.getValue();
}
-
+
+ /** Create an error node with a general error message. */
+ public static void createErrorNode(Configuration output, String errorMessage)
+ throws ManifoldCFException
+ {
+ ConfigurationNode error = new ConfigurationNode(API_ERRORNODE);
+ error.setValue(errorMessage);
+ output.addChild(output.getChildCount(),error);
+ }
+
+
/** Handle an exception, by converting it to an error node. */
public static void createErrorNode(Configuration output, ManifoldCFException e)
throws ManifoldCFException
@@ -273,9 +283,7 @@ public class ManifoldCF extends org.apac
if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
throw e;
Logging.api.error(e.getMessage(),e);
- ConfigurationNode error = new ConfigurationNode(API_ERRORNODE);
- error.setValue(e.getMessage());
- output.addChild(output.getChildCount(),error);
+ createErrorNode(output,e.getMessage());
}
/** Handle a service interruption, by converting it to a serviceinterruption node. */
Modified: manifoldcf/trunk/framework/api-servlet/src/main/java/org/apache/manifoldcf/apiservlet/APIServlet.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/api-servlet/src/main/java/org/apache/manifoldcf/apiservlet/APIServlet.java?rev=1449526&r1=1449525&r2=1449526&view=diff
==============================================================================
--- manifoldcf/trunk/framework/api-servlet/src/main/java/org/apache/manifoldcf/apiservlet/APIServlet.java (original)
+++ manifoldcf/trunk/framework/api-servlet/src/main/java/org/apache/manifoldcf/apiservlet/APIServlet.java Sun Feb 24 18:54:40 2013
@@ -43,37 +43,12 @@ public class APIServlet extends HttpServ
throws ServletException
{
super.init(config);
-/*
- try
- {
- // Set up the environment
- ManifoldCF.initializeEnvironment();
- // Nothing more needs to be done at this point.
- }
- catch (ManifoldCFException e)
- {
- Logging.misc.error("Error starting API service: "+e.getMessage(),e);
- throw new ServletException("Error starting API service: "+e.getMessage(),e);
- }
-*/
}
/** The destroy method.
*/
public void destroy()
{
-/*
- try
- {
- // Set up the environment
- ManifoldCF.initializeEnvironment();
- // Nothing more needs to be done.
- }
- catch (ManifoldCFException e)
- {
- Logging.misc.error("Error shutting down API service: "+e.getMessage(),e);
- }
-*/
super.destroy();
}
@@ -84,14 +59,13 @@ public class APIServlet extends HttpServ
{
try
{
- // Set up the environment
- //ManifoldCF.initializeEnvironment();
-
// Mint a thread context
IThreadContext tc = ThreadContextFactory.make();
// Get the path info string. This will furnish the command.
String pathInfo = request.getPathInfo();
+ // Get query string. This is used by some GET operations.
+ String queryString = request.getQueryString();
if (pathInfo == null)
{
@@ -99,9 +73,8 @@ public class APIServlet extends HttpServ
return;
}
- // Perform the deletion
- executeRead(tc,response,pathInfo);
-
+ // Perform the get
+ executeRead(tc,response,pathInfo,queryString);
}
catch (ManifoldCFException e)
{
@@ -118,9 +91,6 @@ public class APIServlet extends HttpServ
{
try
{
- // Set up the environment
- //ManifoldCF.initializeEnvironment();
-
// Mint a thread context
IThreadContext tc = ThreadContextFactory.make();
@@ -161,9 +131,6 @@ public class APIServlet extends HttpServ
{
try
{
- // Set up the environment
- //ManifoldCF.initializeEnvironment();
-
// Mint a thread context
IThreadContext tc = ThreadContextFactory.make();
@@ -204,9 +171,6 @@ public class APIServlet extends HttpServ
{
try
{
- // Set up the environment
- //ManifoldCF.initializeEnvironment();
-
// Mint a thread context
IThreadContext tc = ThreadContextFactory.make();
@@ -235,7 +199,7 @@ public class APIServlet extends HttpServ
/** Perform a general "read" operation.
*/
- protected static void executeRead(IThreadContext tc, HttpServletResponse response, String pathInfo)
+ protected static void executeRead(IThreadContext tc, HttpServletResponse response, String pathInfo, String queryString)
throws ManifoldCFException, IOException
{
// Strip off leading "/"
@@ -256,13 +220,16 @@ public class APIServlet extends HttpServ
command = pathInfo.substring(index+1);
}
+ // If query string exists, parse it
+ Map<String,List<String>> queryParameters = parseQueryString(queryString);
+
// Execute the request.
// Since there are no input arguments, we can do this before we look at the protocol.
// There the only response distinction we have here is between exception and no exception.
Configuration output = new Configuration();
- boolean exists = ManifoldCF.executeReadCommand(tc,output,command);
-
+ int readResult = ManifoldCF.executeReadCommand(tc,output,command,queryParameters);
+
// Output
String outputText = null;
@@ -288,10 +255,12 @@ public class APIServlet extends HttpServ
response.sendError(response.SC_BAD_REQUEST,"Unknown API protocol: "+protocol);
return;
}
-
- if (!exists)
+
+ if (readResult == ManifoldCF.READRESULT_NOTFOUND)
response.setStatus(response.SC_NOT_FOUND);
-
+ else if (readResult == ManifoldCF.READRESULT_BADARGS)
+ response.setStatus(response.SC_BAD_REQUEST);
+
byte[] responseValue = outputText.getBytes("utf-8");
// Set response mime type
@@ -612,4 +581,33 @@ public class APIServlet extends HttpServ
// return code 200 assumed!
}
+ protected static Map<String,List<String>> parseQueryString(String queryString)
+ throws UnsupportedEncodingException
+ {
+ if (queryString == null)
+ return null;
+ Map<String,List<String>> rval = new HashMap<String,List<String>>();
+ String[] terms = queryString.split("&");
+ for (String term : terms)
+ {
+ int index = queryString.indexOf("=");
+ if (index == -1)
+ addValue(rval,URLDecoder.decode(term,"utf-8"),"");
+ else
+ addValue(rval,URLDecoder.decode(term.substring(0,index),"utf-8"),URLDecoder.decode(term.substring(index+1),"utf-8"));
+ }
+ return rval;
+ }
+
+ protected static void addValue(Map<String,List<String>> rval, String name, String value)
+ {
+ List<String> valueList = rval.get(name);
+ if (valueList == null)
+ {
+ valueList = new ArrayList<String>(1);
+ rval.put(name,valueList);
+ }
+ valueList.add(value);
+ }
+
}
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/maxactivityreport.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/maxactivityreport.jsp?rev=1449526&r1=1449525&r2=1449526&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/maxactivityreport.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/maxactivityreport.jsp Sun Feb 24 18:54:40 2013
@@ -343,54 +343,9 @@ if (maintenanceUnderway == false)
String[] activityList = null;
if (reportConnection.length() > 0)
{
- IRepositoryConnection thisConnection = connMgr.load(reportConnection);
- if (thisConnection == null)
+ activityList = org.apache.manifoldcf.crawler.system.ManifoldCF.getActivitiesList(threadContext,reportConnection);
+ if (activityList == null)
reportConnection = "";
- else
- {
- String[] outputActivityList = OutputConnectionManagerFactory.getAllOutputActivities(threadContext);
- String[] connectorActivityList = RepositoryConnectorFactory.getActivitiesList(threadContext,thisConnection.getClassName());
- String[] globalActivityList = IRepositoryConnectionManager.activitySet;
- activityList = new String[outputActivityList.length + ((connectorActivityList==null)?0:connectorActivityList.length) + globalActivityList.length];
- int k2 = 0;
- int j;
- if (outputActivityList != null)
- {
- j = 0;
- while (j < outputActivityList.length)
- {
- activityList[k2++] = outputActivityList[j++];
- }
- }
- if (connectorActivityList != null)
- {
- j = 0;
- while (j < connectorActivityList.length)
- {
- activityList[k2++] = connectorActivityList[j++];
- }
- }
- j = 0;
- while (j < globalActivityList.length)
- {
- activityList[k2++] = globalActivityList[j++];
- }
- java.util.Arrays.sort(activityList);
-
- selectedActivities = new HashMap();
- String[] activitiesToNote;
- j = 0;
- if (reportActivities == null)
- activitiesToNote = activityList;
- else
- activitiesToNote = reportActivities;
-
- while (j < activitiesToNote.length)
- {
- String activity = activitiesToNote[j++];
- selectedActivities.put(activity,activity);
- }
- }
}
%>
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/maxbandwidthreport.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/maxbandwidthreport.jsp?rev=1449526&r1=1449525&r2=1449526&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/maxbandwidthreport.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/maxbandwidthreport.jsp Sun Feb 24 18:54:40 2013
@@ -343,54 +343,9 @@ if (maintenanceUnderway == false)
String[] activityList = null;
if (reportConnection.length() > 0)
{
- IRepositoryConnection thisConnection = connMgr.load(reportConnection);
- if (thisConnection == null)
+ activityList = org.apache.manifoldcf.crawler.system.ManifoldCF.getActivitiesList(threadContext,reportConnection);
+ if (activityList == null)
reportConnection = "";
- else
- {
- String[] outputActivityList = OutputConnectionManagerFactory.getAllOutputActivities(threadContext);
- String[] connectorActivityList = RepositoryConnectorFactory.getActivitiesList(threadContext,thisConnection.getClassName());
- String[] globalActivityList = IRepositoryConnectionManager.activitySet;
- activityList = new String[outputActivityList.length + ((connectorActivityList==null)?0:connectorActivityList.length) + globalActivityList.length];
- int k2 = 0;
- int j;
- if (outputActivityList != null)
- {
- j = 0;
- while (j < outputActivityList.length)
- {
- activityList[k2++] = outputActivityList[j++];
- }
- }
- if (connectorActivityList != null)
- {
- j = 0;
- while (j < connectorActivityList.length)
- {
- activityList[k2++] = connectorActivityList[j++];
- }
- }
- j = 0;
- while (j < globalActivityList.length)
- {
- activityList[k2++] = globalActivityList[j++];
- }
- java.util.Arrays.sort(activityList);
-
- selectedActivities = new HashMap();
- String[] activitiesToNote;
- j = 0;
- if (reportActivities == null)
- activitiesToNote = activityList;
- else
- activitiesToNote = reportActivities;
-
- while (j < activitiesToNote.length)
- {
- String activity = activitiesToNote[j++];
- selectedActivities.put(activity,activity);
- }
- }
}
%>
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/resultreport.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/resultreport.jsp?rev=1449526&r1=1449525&r2=1449526&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/resultreport.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/resultreport.jsp Sun Feb 24 18:54:40 2013
@@ -370,54 +370,9 @@ if (maintenanceUnderway == false)
String[] activityList = null;
if (reportConnection.length() > 0)
{
- IRepositoryConnection thisConnection = connMgr.load(reportConnection);
- if (thisConnection == null)
+ activityList = org.apache.manifoldcf.crawler.system.ManifoldCF.getActivitiesList(threadContext,reportConnection);
+ if (activityList == null)
reportConnection = "";
- else
- {
- String[] outputActivityList = OutputConnectionManagerFactory.getAllOutputActivities(threadContext);
- String[] connectorActivityList = RepositoryConnectorFactory.getActivitiesList(threadContext,thisConnection.getClassName());
- String[] globalActivityList = IRepositoryConnectionManager.activitySet;
- activityList = new String[outputActivityList.length + ((connectorActivityList==null)?0:connectorActivityList.length) + globalActivityList.length];
- int k2 = 0;
- int j;
- if (outputActivityList != null)
- {
- j = 0;
- while (j < outputActivityList.length)
- {
- activityList[k2++] = outputActivityList[j++];
- }
- }
- if (connectorActivityList != null)
- {
- j = 0;
- while (j < connectorActivityList.length)
- {
- activityList[k2++] = connectorActivityList[j++];
- }
- }
- j = 0;
- while (j < globalActivityList.length)
- {
- activityList[k2++] = globalActivityList[j++];
- }
- java.util.Arrays.sort(activityList);
-
- selectedActivities = new HashMap();
- String[] activitiesToNote;
- j = 0;
- if (reportActivities == null)
- activitiesToNote = activityList;
- else
- activitiesToNote = reportActivities;
-
- while (j < activitiesToNote.length)
- {
- String activity = activitiesToNote[j++];
- selectedActivities.put(activity,activity);
- }
- }
}
%>
Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/simplereport.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/simplereport.jsp?rev=1449526&r1=1449525&r2=1449526&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/simplereport.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/simplereport.jsp Sun Feb 24 18:54:40 2013
@@ -298,54 +298,9 @@ if (maintenanceUnderway == false)
String[] activityList = null;
if (reportConnection.length() > 0)
{
- IRepositoryConnection thisConnection = connMgr.load(reportConnection);
- if (thisConnection == null)
+ activityList = org.apache.manifoldcf.crawler.system.ManifoldCF.getActivitiesList(threadContext,reportConnection);
+ if (activityList == null)
reportConnection = "";
- else
- {
- String[] outputActivityList = OutputConnectionManagerFactory.getAllOutputActivities(threadContext);
- String[] connectorActivityList = RepositoryConnectorFactory.getActivitiesList(threadContext,thisConnection.getClassName());
- String[] globalActivityList = IRepositoryConnectionManager.activitySet;
- activityList = new String[outputActivityList.length + ((connectorActivityList==null)?0:connectorActivityList.length) + globalActivityList.length];
- int k2 = 0;
- int j;
- if (outputActivityList != null)
- {
- j = 0;
- while (j < outputActivityList.length)
- {
- activityList[k2++] = outputActivityList[j++];
- }
- }
- if (connectorActivityList != null)
- {
- j = 0;
- while (j < connectorActivityList.length)
- {
- activityList[k2++] = connectorActivityList[j++];
- }
- }
- j = 0;
- while (j < globalActivityList.length)
- {
- activityList[k2++] = globalActivityList[j++];
- }
- java.util.Arrays.sort(activityList);
-
- selectedActivities = new HashMap();
- String[] activitiesToNote;
- j = 0;
- if (reportActivities == null)
- activitiesToNote = activityList;
- else
- activitiesToNote = reportActivities;
-
- while (j < activitiesToNote.length)
- {
- String activity = activitiesToNote[j++];
- selectedActivities.put(activity,activity);
- }
- }
}
%>