You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2013/06/14 21:43:42 UTC
svn commit: r1493226 - in /manifoldcf/trunk: ./
connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/
connectors/livelink/build-stub/src/main/java/com/opentext/api/
connectors/livelink/connector/src/main/j...
Author: kwright
Date: Fri Jun 14 19:43:42 2013
New Revision: 1493226
URL: http://svn.apache.org/r1493226
Log:
Fix for CONNECTORS-714. Revise LiveLink connector to allow document fetching via LAPI.
Added:
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadOutputStream.java
- copied unchanged from r1493217, manifoldcf/branches/CONNECTORS-714/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadOutputStream.java
Modified:
manifoldcf/trunk/ (props changed)
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
manifoldcf/trunk/connectors/livelink/build-stub/src/main/java/com/opentext/api/LAPI_DOCUMENTS.java
manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties
manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_ja_JP.properties
Propchange: manifoldcf/trunk/
------------------------------------------------------------------------------
Merged /manifoldcf/branches/CONNECTORS-714:r1492346-1493217
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1493226&r1=1493225&r2=1493226&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Fri Jun 14 19:43:42 2013
@@ -3,6 +3,9 @@ $Id$
======================= 1.3-dev =====================
+CONNECTORS-714: All LiveLink connector to use LAPI for document fetches.
+(David Morana, Karl Wright)
+
CONNECTORS-718: Alfresco Connector must throw exceptions with handler methods
(Piergiorgio Lucidi, Karl Wright)
Modified: manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java?rev=1493226&r1=1493225&r2=1493226&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java (original)
+++ manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java Fri Jun 14 19:43:42 2013
@@ -147,41 +147,4 @@ public class GoogleDriveSession {
}
}
- protected static class XThreadOutputStream extends OutputStream {
- protected final XThreadInputStream inputStream;
-
- byte[] byteBuffer = new byte[1];
-
- public XThreadOutputStream(XThreadInputStream inputStream) {
- this.inputStream = inputStream;
- }
-
- @Override
- public void write(int c)
- throws IOException {
- byteBuffer[0] = (byte)c;
- try {
- inputStream.stuffQueue(byteBuffer,0,1);
- } catch (InterruptedException e) {
- throw new InterruptedIOException(e.getMessage());
- }
- }
-
- @Override
- public void write(byte[] buffer, int pos, int amt)
- throws IOException {
- try {
- inputStream.stuffQueue(buffer,pos,amt);
- } catch (InterruptedException e) {
- throw new InterruptedIOException(e.getMessage());
- }
- }
-
- @Override
- public void close()
- throws IOException {
- inputStream.doneStuffingQueue();
- super.close();
- }
- }
}
Modified: manifoldcf/trunk/connectors/livelink/build-stub/src/main/java/com/opentext/api/LAPI_DOCUMENTS.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/build-stub/src/main/java/com/opentext/api/LAPI_DOCUMENTS.java?rev=1493226&r1=1493225&r2=1493226&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/livelink/build-stub/src/main/java/com/opentext/api/LAPI_DOCUMENTS.java (original)
+++ manifoldcf/trunk/connectors/livelink/build-stub/src/main/java/com/opentext/api/LAPI_DOCUMENTS.java Fri Jun 14 19:43:42 2013
@@ -53,6 +53,11 @@ public class LAPI_DOCUMENTS
return 0;
}
+ public int FetchVersion(int vol, int id, int revNumber, java.io.OutputStream output)
+ {
+ return 0;
+ }
+
public int GetObjectRights(int vol, int objID, LLValue objinfo)
{
return 0;
Modified: manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1493226&r1=1493225&r2=1493226&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java (original)
+++ manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java Fri Jun 14 19:43:42 2013
@@ -24,6 +24,7 @@ import org.apache.manifoldcf.crawler.int
import org.apache.manifoldcf.crawler.system.Logging;
import org.apache.manifoldcf.crawler.system.ManifoldCF;
import org.apache.manifoldcf.core.common.XThreadInputStream;
+import org.apache.manifoldcf.core.common.XThreadOutputStream;
import java.io.*;
import java.util.*;
@@ -334,21 +335,31 @@ public class LivelinkConnector extends o
serverHTTPNTLMPassword = params.getObfuscatedParameter(LiveLinkParameters.serverHTTPNTLMPassword);
if (ingestProtocol == null || ingestProtocol.length() == 0)
- ingestProtocol = "http";
+ ingestProtocol = null;
if (viewProtocol == null || viewProtocol.length() == 0)
- viewProtocol = ingestProtocol;
+ {
+ if (ingestProtocol == null)
+ viewProtocol = "http";
+ else
+ viewProtocol = ingestProtocol;
+ }
if (ingestPort == null || ingestPort.length() == 0)
{
- if (!ingestProtocol.equals("https"))
- ingestPort = "80";
+ if (ingestProtocol != null)
+ {
+ if (!ingestProtocol.equals("https"))
+ ingestPort = "80";
+ else
+ ingestPort = "443";
+ }
else
- ingestPort = "443";
+ ingestPort = null;
}
if (viewPort == null || viewPort.length() == 0)
{
- if (!viewProtocol.equals(ingestProtocol))
+ if (ingestProtocol == null || !viewProtocol.equals(ingestProtocol))
{
if (!viewProtocol.equals("https"))
viewPort = "80";
@@ -359,13 +370,16 @@ public class LivelinkConnector extends o
viewPort = ingestPort;
}
- try
+ if (ingestPort != null)
{
- ingestPortNumber = Integer.parseInt(ingestPort);
- }
- catch (NumberFormatException e)
- {
- throw new ManifoldCFException("Bad ingest port: "+e.getMessage(),e);
+ try
+ {
+ ingestPortNumber = Integer.parseInt(ingestPort);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new ManifoldCFException("Bad ingest port: "+e.getMessage(),e);
+ }
}
String viewPortString;
@@ -580,62 +594,67 @@ public class LivelinkConnector extends o
getSession();
// Now, set up trial of ingestion connection
- String contextMsg = "for document access";
- String ingestHttpAddress = ingestCgiPath;
-
- HttpClient client = getInitializedClient(contextMsg);
- HttpGet method = new HttpGet(getHost().toURI() + ingestHttpAddress);
- method.setHeader(new BasicHeader("Accept","*/*"));
- try
+ if (ingestProtocol != null)
{
- int statusCode = executeMethodViaThread(client,method);
- switch (statusCode)
+ String contextMsg = "for document access";
+ String ingestHttpAddress = ingestCgiPath;
+
+ HttpClient client = getInitializedClient(contextMsg);
+ HttpGet method = new HttpGet(getHost().toURI() + ingestHttpAddress);
+ method.setHeader(new BasicHeader("Accept","*/*"));
+ try
{
- case 502:
- return "Fetch test had transient 502 error response";
+ int statusCode = executeMethodViaThread(client,method);
+ switch (statusCode)
+ {
+ case 502:
+ return "Fetch test had transient 502 error response";
- case HttpStatus.SC_UNAUTHORIZED:
- return "Fetch test returned UNAUTHORIZED (401) response; check the security credentials and configuration";
+ case HttpStatus.SC_UNAUTHORIZED:
+ return "Fetch test returned UNAUTHORIZED (401) response; check the security credentials and configuration";
- case HttpStatus.SC_OK:
- return super.check();
+ case HttpStatus.SC_OK:
+ return super.check();
- default:
- return "Fetch test returned an unexpected response code of "+Integer.toString(statusCode);
+ default:
+ return "Fetch test returned an unexpected response code of "+Integer.toString(statusCode);
+ }
+ }
+ catch (InterruptedException e)
+ {
+ throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ catch (java.net.SocketTimeoutException e)
+ {
+ return "Fetch test timed out reading from the Livelink HTTP Server: "+e.getMessage();
+ }
+ catch (java.net.SocketException e)
+ {
+ return "Fetch test received a socket error reading from Livelink HTTP Server: "+e.getMessage();
+ }
+ catch (javax.net.ssl.SSLHandshakeException e)
+ {
+ return "Fetch test was unable to set up a SSL connection to Livelink HTTP Server: "+e.getMessage();
+ }
+ catch (ConnectTimeoutException e)
+ {
+ return "Fetch test connection timed out reading from Livelink HTTP Server: "+e.getMessage();
+ }
+ catch (InterruptedIOException e)
+ {
+ throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ catch (HttpException e)
+ {
+ return "Fetch test had an HTTP exception: "+e.getMessage();
+ }
+ catch (IOException e)
+ {
+ return "Fetch test had an IO failure: "+e.getMessage();
}
}
- catch (InterruptedException e)
- {
- throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (java.net.SocketTimeoutException e)
- {
- return "Fetch test timed out reading from the Livelink HTTP Server: "+e.getMessage();
- }
- catch (java.net.SocketException e)
- {
- return "Fetch test received a socket error reading from Livelink HTTP Server: "+e.getMessage();
- }
- catch (javax.net.ssl.SSLHandshakeException e)
- {
- return "Fetch test was unable to set up a SSL connection to Livelink HTTP Server: "+e.getMessage();
- }
- catch (ConnectTimeoutException e)
- {
- return "Fetch test connection timed out reading from Livelink HTTP Server: "+e.getMessage();
- }
- catch (InterruptedIOException e)
- {
- throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (HttpException e)
- {
- return "Fetch test had an HTTP exception: "+e.getMessage();
- }
- catch (IOException e)
- {
- return "Fetch test had an IO failure: "+e.getMessage();
- }
+ else
+ return super.check();
}
catch (ServiceInterruption e)
{
@@ -1686,16 +1705,23 @@ public class LivelinkConnector extends o
" editconnection.serverhttpcgipath.focus();\n"+
" return false;\n"+
" }\n"+
-" if (editconnection.ingestcgipath.value == \"\")\n"+
+" if (editconnection.viewprotocol.value == \"\" && editconnection.ingestprotocol.value == \"\")\n"+
" {\n"+
-" alert(\""+Messages.getBodyJavascriptString(locale,"LivelinkConnector.EnterTheCrawlCgiPathToLivelink")+"\");\n"+
-" SelectTab(\"" + Messages.getBodyJavascriptString(locale,"LivelinkConnector.DocumentAccess") + "\");\n"+
-" editconnection.ingestcgipath.focus();\n"+
+" alert(\""+Messages.getBodyJavascriptString(locale,"LivelinkConnector.SelectAViewProtocol")+"\");\n"+
+" SelectTab(\"" + Messages.getBodyJavascriptString(locale,"LivelinkConnector.DocumentView") + "\");\n"+
+" editconnection.viewprotocol.focus();\n"+
+" return false;\n"+
+" }\n"+
+" if (editconnection.viewcgipath.value == \"\" && editconnection.ingestcgipath.value == \"\")\n"+
+" {\n"+
+" alert(\""+Messages.getBodyJavascriptString(locale,"LivelinkConnector.EnterTheViewCgiPathToLivelink")+"\");\n"+
+" SelectTab(\"" + Messages.getBodyJavascriptString(locale,"LivelinkConnector.DocumentView") + "\");\n"+
+" editconnection.viewcgipath.focus();\n"+
" return false;\n"+
" }\n"+
-" if (editconnection.ingestcgipath.value.substring(0,1) != \"/\")\n"+
+" if (editconnection.ingestcgipath.value != \"\" && editconnection.ingestcgipath.value.substring(0,1) != \"/\")\n"+
" {\n"+
-" alert(\""+Messages.getBodyJavascriptString(locale,"LivelinkConnector.TheIngestCgiPathMustBeginWithACharacter")+"\");\n"+
+" alert(\""+Messages.getBodyJavascriptString(locale,"LivelinkConnector.TheIngestCgiPathMustBeBlankOrBeginWithACharacter")+"\");\n"+
" SelectTab(\"" + Messages.getBodyJavascriptString(locale,"LivelinkConnector.DocumentAccess") + "\");\n"+
" editconnection.ingestcgipath.focus();\n"+
" return false;\n"+
@@ -1768,13 +1794,13 @@ public class LivelinkConnector extends o
// Document access parameters
String ingestProtocol = parameters.getParameter(LiveLinkParameters.ingestProtocol);
if (ingestProtocol == null)
- ingestProtocol = "http";
+ ingestProtocol = "";
String ingestPort = parameters.getParameter(LiveLinkParameters.ingestPort);
if (ingestPort == null)
ingestPort = "";
String ingestCgiPath = parameters.getParameter(LiveLinkParameters.ingestCgiPath);
if (ingestCgiPath == null)
- ingestCgiPath = "/livelink/livelink.exe";
+ ingestCgiPath = "";
String ingestNtlmUsername = parameters.getParameter(LiveLinkParameters.ingestNtlmUsername);
if (ingestNtlmUsername == null)
ingestNtlmUsername = "";
@@ -1794,7 +1820,7 @@ public class LivelinkConnector extends o
// Document view parameters
String viewProtocol = parameters.getParameter(LiveLinkParameters.viewProtocol);
if (viewProtocol == null)
- viewProtocol = "";
+ viewProtocol = "http";
String viewServerName = parameters.getParameter(LiveLinkParameters.viewServerName);
if (viewServerName == null)
viewServerName = "";
@@ -1803,7 +1829,7 @@ public class LivelinkConnector extends o
viewPort = "";
String viewCgiPath = parameters.getParameter(LiveLinkParameters.viewCgiPath);
if (viewCgiPath == null)
- viewCgiPath = "";
+ viewCgiPath = "/livelink/livelink.exe";
// The "Server" tab
// Always pass the whole keystore as a hidden.
@@ -1951,7 +1977,8 @@ public class LivelinkConnector extends o
" <tr>\n"+
" <td class=\"description\">"+Messages.getBodyString(locale,"LivelinkConnector.DocumentFetchProtocol")+"</td>\n"+
" <td class=\"value\">\n"+
-" <select name=\"ingestprotocol\" size=\"2\">\n"+
+" <select name=\"ingestprotocol\" size=\"3\">\n"+
+" <option value=\"\" "+((ingestProtocol.equals(""))?"selected=\"selected\"":"")+">"+Messages.getBodyString(locale,"LivelinkConnector.UseLAPI")+"</option>\n"+
" <option value=\"http\" "+((ingestProtocol.equals("http"))?"selected=\"selected\"":"")+">http</option>\n"+
" <option value=\"https\" "+((ingestProtocol.equals("https"))?"selected=\"selected\"":"")+">https</option>\n"+
" </select>\n"+
@@ -4175,14 +4202,6 @@ public class LivelinkConnector extends o
String contextMsg = "for '"+documentIdentifier+"'";
- String ingestHttpAddress = convertToIngestURI(documentIdentifier);
- if (ingestHttpAddress == null)
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: No fetch URI "+contextMsg+" - not ingesting");
- return;
- }
-
String viewHttpAddress = convertToViewURI(documentIdentifier);
if (viewHttpAddress == null)
{
@@ -4191,13 +4210,17 @@ public class LivelinkConnector extends o
return;
}
- RepositoryDocument rd = new RepositoryDocument();
-
- int colonPos = documentIdentifier.indexOf(":",1);
-
+ // Fetch logging
+ long startTime = System.currentTimeMillis();
+ String resultCode = "FAILED";
+ String resultDescription = null;
+ Long readSize = null;
+ boolean wasInterrupted = false;
int objID;
int vol;
+ int colonPos = documentIdentifier.indexOf(":",1);
+
if (colonPos == -1)
{
objID = new Integer(documentIdentifier.substring(1)).intValue();
@@ -4208,356 +4231,504 @@ public class LivelinkConnector extends o
objID = new Integer(documentIdentifier.substring(colonPos+1)).intValue();
vol = new Integer(documentIdentifier.substring(1,colonPos)).intValue();
}
-
- // Add general metadata
- ObjectInformation objInfo = llc.getObjectInformation(vol,objID);
- VersionInformation versInfo = llc.getVersionInformation(vol,objID,0);
- if (!objInfo.exists())
- {
- Logging.connectors.debug("Livelink: No object "+contextMsg+": not ingesting");
- return;
- }
- if (!versInfo.exists())
- {
- Logging.connectors.debug("Livelink: No version data "+contextMsg+": not ingesting");
- return;
- }
- // Add general data we need for the output connector
- String mimeType = versInfo.getMimeType();
- if (mimeType != null)
- rd.setMimeType(mimeType);
- String fileName = versInfo.getFileName();
- if (fileName != null)
- rd.setFileName(fileName);
- Date creationDate = objInfo.getCreationDate();
- if (creationDate != null)
- rd.setCreatedDate(creationDate);
- Date modifyDate = versInfo.getModifyDate();
- if (modifyDate != null)
- rd.setModifiedDate(modifyDate);
-
- rd.addField(GENERAL_NAME_FIELD,objInfo.getName());
- rd.addField(GENERAL_DESCRIPTION_FIELD,objInfo.getComments());
- if (creationDate != null)
- rd.addField(GENERAL_CREATIONDATE_FIELD,creationDate.toString());
- if (modifyDate != null)
- rd.addField(GENERAL_MODIFYDATE_FIELD,modifyDate.toString());
- UserInformation owner = llc.getUserInformation(objInfo.getOwnerId().intValue());
- UserInformation creator = llc.getUserInformation(objInfo.getCreatorId().intValue());
- UserInformation modifier = llc.getUserInformation(versInfo.getOwnerId().intValue());
- if (owner != null)
- rd.addField(GENERAL_OWNER,owner.getName());
- if (creator != null)
- rd.addField(GENERAL_CREATOR,creator.getName());
- if (modifier != null)
- rd.addField(GENERAL_MODIFIER,modifier.getName());
-
- // Iterate over the metadata items. These are organized by category
- // for speed of lookup.
-
- // Unpack version string
- int startPos = 0;
-
- // Metadata items first
- ArrayList metadataItems = new ArrayList();
- startPos = unpackList(metadataItems,version,startPos,'+');
- Iterator catIter = desc.getItems(metadataItems);
- while (catIter.hasNext())
- {
- MetadataItem item = (MetadataItem)catIter.next();
- MetadataPathItem pathItem = item.getPathItem();
- if (pathItem != null)
- {
- int catID = pathItem.getCatID();
- // grab the associated catversion
- LLValue catVersion = getCatVersion(objID,catID);
- if (catVersion != null)
- {
- // Go through attributes now
- Iterator attrIter = item.getAttributeNames();
- while (attrIter.hasNext())
- {
- String attrName = (String)attrIter.next();
- // Create a unique metadata name
- String metadataName = pathItem.getCatName()+":"+attrName;
- // Fetch the metadata and stuff it into the RepositoryData structure
- String[] metadataValue = getAttributeValue(catVersion,attrName);
- if (metadataValue != null)
- rd.addField(metadataName,metadataValue);
- else
- Logging.connectors.warn("Livelink: Metadata attribute '"+metadataName+"' does not seem to exist; please correct the job");
- }
- }
-
- }
- }
-
- // Unpack acls (conditionally)
- if (startPos < version.length())
+ // Try/finally for fetch logging
+ try
{
- char x = version.charAt(startPos++);
- if (x == '+')
+ // Check URL first
+ if (activities.checkURLIndexable(viewHttpAddress))
{
- ArrayList acls = new ArrayList();
- startPos = unpackList(acls,version,startPos,'+');
- // Turn into acls and add into description
- String[] aclArray = new String[acls.size()];
- int j = 0;
- while (j < aclArray.length)
+
+ // Add general metadata
+ ObjectInformation objInfo = llc.getObjectInformation(vol,objID);
+ VersionInformation versInfo = llc.getVersionInformation(vol,objID,0);
+ if (!objInfo.exists())
{
- aclArray[j] = (String)acls.get(j);
- j++;
+ resultCode = "OBJECTNOTFOUND";
+ Logging.connectors.debug("Livelink: No object "+contextMsg+": not ingesting");
+ return;
+ }
+ if (!versInfo.exists())
+ {
+ resultCode = "VERSIONNOTFOUND";
+ Logging.connectors.debug("Livelink: No version data "+contextMsg+": not ingesting");
+ return;
}
- rd.setACL(aclArray);
- StringBuilder denyBuffer = new StringBuilder();
- startPos = unpack(denyBuffer,version,startPos,'+');
- String denyAcl = denyBuffer.toString();
- String[] denyAclArray = new String[1];
- denyAclArray[0] = denyAcl;
- rd.setDenyACL(denyAclArray);
- }
- }
+ String mimeType = versInfo.getMimeType();
+ if (activities.checkMimeTypeIndexable(mimeType))
+ {
+ Long dataSize = versInfo.getDataSize();
+ if (dataSize != null && activities.checkLengthIndexable(dataSize.longValue()))
+ {
+ String fileName = versInfo.getFileName();
+ Date creationDate = objInfo.getCreationDate();
+ Date modifyDate = versInfo.getModifyDate();
+ RepositoryDocument rd = new RepositoryDocument();
+
+
+ // Add general data we need for the output connector
+ if (mimeType != null)
+ rd.setMimeType(mimeType);
+ if (fileName != null)
+ rd.setFileName(fileName);
+ if (creationDate != null)
+ rd.setCreatedDate(creationDate);
+ if (modifyDate != null)
+ rd.setModifiedDate(modifyDate);
+
+ rd.addField(GENERAL_NAME_FIELD,objInfo.getName());
+ rd.addField(GENERAL_DESCRIPTION_FIELD,objInfo.getComments());
+ if (creationDate != null)
+ rd.addField(GENERAL_CREATIONDATE_FIELD,creationDate.toString());
+ if (modifyDate != null)
+ rd.addField(GENERAL_MODIFYDATE_FIELD,modifyDate.toString());
+ UserInformation owner = llc.getUserInformation(objInfo.getOwnerId().intValue());
+ UserInformation creator = llc.getUserInformation(objInfo.getCreatorId().intValue());
+ UserInformation modifier = llc.getUserInformation(versInfo.getOwnerId().intValue());
+ if (owner != null)
+ rd.addField(GENERAL_OWNER,owner.getName());
+ if (creator != null)
+ rd.addField(GENERAL_CREATOR,creator.getName());
+ if (modifier != null)
+ rd.addField(GENERAL_MODIFIER,modifier.getName());
+
+ // Iterate over the metadata items. These are organized by category
+ // for speed of lookup.
+
+ // Unpack version string
+ int startPos = 0;
+
+ // Metadata items first
+ ArrayList metadataItems = new ArrayList();
+ startPos = unpackList(metadataItems,version,startPos,'+');
+ Iterator catIter = desc.getItems(metadataItems);
+ while (catIter.hasNext())
+ {
+ MetadataItem item = (MetadataItem)catIter.next();
+ MetadataPathItem pathItem = item.getPathItem();
+ if (pathItem != null)
+ {
+ int catID = pathItem.getCatID();
+ // grab the associated catversion
+ LLValue catVersion = getCatVersion(objID,catID);
+ if (catVersion != null)
+ {
+ // Go through attributes now
+ Iterator attrIter = item.getAttributeNames();
+ while (attrIter.hasNext())
+ {
+ String attrName = (String)attrIter.next();
+ // Create a unique metadata name
+ String metadataName = pathItem.getCatName()+":"+attrName;
+ // Fetch the metadata and stuff it into the RepositoryData structure
+ String[] metadataValue = getAttributeValue(catVersion,attrName);
+ if (metadataValue != null)
+ rd.addField(metadataName,metadataValue);
+ else
+ Logging.connectors.warn("Livelink: Metadata attribute '"+metadataName+"' does not seem to exist; please correct the job");
+ }
+ }
- // Add the path metadata item into the mix, if enabled
- String pathAttributeName = sDesc.getPathAttributeName();
- if (pathAttributeName != null && pathAttributeName.length() > 0)
- {
- String pathString = sDesc.getPathAttributeValue(documentIdentifier);
- if (pathString != null)
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Path attribute name is '"+pathAttributeName+"'"+contextMsg+", value is '"+pathString+"'");
- rd.addField(pathAttributeName,pathString);
- }
- }
+ }
+ }
- // Set up connection
- HttpClient client = getInitializedClient(contextMsg);
+ // Unpack acls (conditionally)
+ if (startPos < version.length())
+ {
+ char x = version.charAt(startPos++);
+ if (x == '+')
+ {
+ ArrayList acls = new ArrayList();
+ startPos = unpackList(acls,version,startPos,'+');
+ // Turn into acls and add into description
+ String[] aclArray = new String[acls.size()];
+ int j = 0;
+ while (j < aclArray.length)
+ {
+ aclArray[j] = (String)acls.get(j);
+ j++;
+ }
+ rd.setACL(aclArray);
- long currentTime;
+ StringBuilder denyBuffer = new StringBuilder();
+ startPos = unpack(denyBuffer,version,startPos,'+');
+ String denyAcl = denyBuffer.toString();
+ String[] denyAclArray = new String[1];
+ denyAclArray[0] = denyAcl;
+ rd.setDenyACL(denyAclArray);
+ }
+ }
- if (Logging.connectors.isInfoEnabled())
- Logging.connectors.info("Livelink: " + ingestHttpAddress);
+ // Add the path metadata item into the mix, if enabled
+ String pathAttributeName = sDesc.getPathAttributeName();
+ if (pathAttributeName != null && pathAttributeName.length() > 0)
+ {
+ String pathString = sDesc.getPathAttributeValue(documentIdentifier);
+ if (pathString != null)
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Path attribute name is '"+pathAttributeName+"'"+contextMsg+", value is '"+pathString+"'");
+ rd.addField(pathAttributeName,pathString);
+ }
+ }
- long startTime = System.currentTimeMillis();
- String resultCode = "OK";
- String resultDescription = null;
- Long readSize = null;
+ if (viewProtocol != null)
+ {
+ // Use HTTP to fetch document!
+ String ingestHttpAddress = convertToIngestURI(documentIdentifier);
+ if (ingestHttpAddress != null)
+ {
- HttpGet method = new HttpGet(getHost().toURI() + ingestHttpAddress);
- method.setHeader(new BasicHeader("Accept","*/*"));
+ // Set up connection
+ HttpClient client = getInitializedClient(contextMsg);
- ExecuteMethodThread methodThread = new ExecuteMethodThread(client,method);
- methodThread.start();
- try
- {
+ long currentTime;
- int statusCode = methodThread.getResponseCode();
- switch (statusCode)
- {
- case 500:
- case 502:
- Logging.connectors.warn("Livelink: Service interruption during fetch "+contextMsg+" with Livelink HTTP Server, retrying...");
- throw new ServiceInterruption("Service interruption during fetch",new ManifoldCFException(Integer.toString(statusCode)+" error while fetching"),System.currentTimeMillis()+60000L,
- System.currentTimeMillis()+600000L,-1,true);
+ if (Logging.connectors.isInfoEnabled())
+ Logging.connectors.info("Livelink: " + ingestHttpAddress);
- case HttpStatus.SC_UNAUTHORIZED:
- Logging.connectors.warn("Livelink: Document fetch unauthorized for "+ingestHttpAddress+" ("+contextMsg+")");
- // Since we logged in, we should fail here if the ingestion user doesn't have access to the
- // the document, but if we do, don't fail hard.
- resultCode = "UNAUTHORIZED";
- activities.deleteDocument(documentIdentifier,version);
- return;
- case HttpStatus.SC_OK:
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Created http document connection to Livelink "+contextMsg);
- long dataSize = methodThread.getResponseContentLength();
- // The above replaces this, which required another access:
- // long dataSize = (long)value.toInteger("DataSize");
- // A non-existent content length will cause a value of -1 to be returned. This seems to indicate that the session login did not work right.
- if (dataSize >= 0)
- {
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Content length from livelink server "+contextMsg+"' = "+new Long(dataSize).toString());
- if (activities.checkLengthIndexable(dataSize))
- {
- try
- {
- InputStream is = methodThread.getSafeInputStream();
- try
- {
- rd.setBinary(is,dataSize);
-
- activities.ingestDocument(documentIdentifier,version,viewHttpAddress,rd);
+ HttpGet method = new HttpGet(getHost().toURI() + ingestHttpAddress);
+ method.setHeader(new BasicHeader("Accept","*/*"));
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("Livelink: Ingesting done "+contextMsg);
+ ExecuteMethodThread methodThread = new ExecuteMethodThread(client,method);
+ methodThread.start();
+ try
+ {
+ int statusCode = methodThread.getResponseCode();
+ switch (statusCode)
+ {
+ case 500:
+ case 502:
+ Logging.connectors.warn("Livelink: Service interruption during fetch "+contextMsg+" with Livelink HTTP Server, retrying...");
+ throw new ServiceInterruption("Service interruption during fetch",new ManifoldCFException(Integer.toString(statusCode)+" error while fetching"),System.currentTimeMillis()+60000L,
+ System.currentTimeMillis()+600000L,-1,true);
+
+ case HttpStatus.SC_UNAUTHORIZED:
+ Logging.connectors.warn("Livelink: Document fetch unauthorized for "+ingestHttpAddress+" ("+contextMsg+")");
+ // Since we logged in, we should fail here if the ingestion user doesn't have access to the
+ // the document, but if we do, don't fail hard.
+ resultCode = "UNAUTHORIZED";
+ activities.deleteDocument(documentIdentifier,version);
+ return;
+
+ case HttpStatus.SC_OK:
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Created http document connection to Livelink "+contextMsg);
+ // A non-existent content length will cause a value of -1 to be returned. This seems to indicate that the session login did not work right.
+ if (methodThread.getResponseContentLength() >= 0)
+ {
+ try
+ {
+ InputStream is = methodThread.getSafeInputStream();
+ try
+ {
+ rd.setBinary(is,dataSize);
+
+ activities.ingestDocument(documentIdentifier,version,viewHttpAddress,rd);
+
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Ingesting done "+contextMsg);
+
+ }
+ finally
+ {
+ // Close stream via thread, since otherwise this can hang
+ is.close();
+ }
+ }
+ catch (java.net.SocketTimeoutException e)
+ {
+ resultCode = "DATATIMEOUT";
+ resultDescription = e.getMessage();
+ currentTime = System.currentTimeMillis();
+ Logging.connectors.warn("Livelink: Livelink socket timed out ingesting from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+ throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+ }
+ catch (java.net.SocketException e)
+ {
+ resultCode = "DATASOCKETERROR";
+ resultDescription = e.getMessage();
+ currentTime = System.currentTimeMillis();
+ Logging.connectors.warn("Livelink: Livelink socket error ingesting from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+ throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+ }
+ catch (javax.net.ssl.SSLHandshakeException e)
+ {
+ resultCode = "DATASSLHANDSHAKEERROR";
+ resultDescription = e.getMessage();
+ currentTime = System.currentTimeMillis();
+ Logging.connectors.warn("Livelink: SSL handshake failed authenticating "+contextMsg+": "+e.getMessage(),e);
+ throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
+ }
+ catch (ConnectTimeoutException e)
+ {
+ resultCode = "CONNECTTIMEOUT";
+ resultDescription = e.getMessage();
+ currentTime = System.currentTimeMillis();
+ Logging.connectors.warn("Livelink: Livelink socket timed out connecting to the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+ throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+ }
+ catch (InterruptedException e)
+ {
+ wasInterrupted = true;
+ throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ catch (InterruptedIOException e)
+ {
+ wasInterrupted = true;
+ throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ catch (HttpException e)
+ {
+ resultCode = "HTTPEXCEPTION";
+ resultDescription = e.getMessage();
+ // Treat unknown error ingesting data as a transient condition
+ currentTime = System.currentTimeMillis();
+ Logging.connectors.warn("Livelink: HTTP exception ingesting "+contextMsg+": "+e.getMessage(),e);
+ throw new ServiceInterruption("HTTP exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+ }
+ catch (IOException e)
+ {
+ resultCode = "DATAEXCEPTION";
+ resultDescription = e.getMessage();
+ // Treat unknown error ingesting data as a transient condition
+ currentTime = System.currentTimeMillis();
+ Logging.connectors.warn("Livelink: IO exception ingesting "+contextMsg+": "+e.getMessage(),e);
+ throw new ServiceInterruption("IO exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+ }
+ readSize = dataSize;
+ }
+ else
+ {
+ resultCode = "SESSIONLOGINFAILED";
+ activities.deleteDocument(documentIdentifier,version);
+ }
+ break;
+ case HttpStatus.SC_BAD_REQUEST:
+ case HttpStatus.SC_USE_PROXY:
+ case HttpStatus.SC_GONE:
+ resultCode = "ERROR "+Integer.toString(statusCode);
+ throw new ManifoldCFException("Unrecoverable request failure; error = "+Integer.toString(statusCode));
+ default:
+ resultCode = "UNKNOWN";
+ Logging.connectors.warn("Livelink: Attempt to retrieve document from '"+ingestHttpAddress+"' received a response of "+Integer.toString(statusCode)+"; retrying in one minute");
+ currentTime = System.currentTimeMillis();
+ throw new ServiceInterruption("Fetch failed; retrying in 1 minute",new ManifoldCFException("Fetch failed with unknown code "+Integer.toString(statusCode)),
+ currentTime+60000L,currentTime+600000L,-1,true);
+ }
+ }
+ catch (InterruptedException e)
+ {
+ // Drop the connection on the floor
+ methodThread.interrupt();
+ methodThread = null;
+ throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ catch (java.net.SocketTimeoutException e)
+ {
+ Logging.connectors.warn("Livelink: Socket timed out reading from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+ resultCode = "TIMEOUT";
+ resultDescription = e.getMessage();
+ currentTime = System.currentTimeMillis();
+ throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+ }
+ catch (java.net.SocketException e)
+ {
+ Logging.connectors.warn("Livelink: Socket error reading from Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+ resultCode = "SOCKETERROR";
+ resultDescription = e.getMessage();
+ currentTime = System.currentTimeMillis();
+ throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+ }
+ catch (javax.net.ssl.SSLHandshakeException e)
+ {
+ currentTime = System.currentTimeMillis();
+ Logging.connectors.warn("Livelink: SSL handshake failed "+contextMsg+": "+e.getMessage(),e);
+ resultCode = "SSLHANDSHAKEERROR";
+ resultDescription = e.getMessage();
+ throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
+ }
+ catch (ConnectTimeoutException e)
+ {
+ Logging.connectors.warn("Livelink: Connect timed out reading from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
+ resultCode = "CONNECTTIMEOUT";
+ resultDescription = e.getMessage();
+ currentTime = System.currentTimeMillis();
+ throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+ }
+ catch (InterruptedIOException e)
+ {
+ methodThread.interrupt();
+ throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ catch (HttpException e)
+ {
+ resultCode = "EXCEPTION";
+ resultDescription = e.getMessage();
+ throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
+ }
+ catch (IOException e)
+ {
+ resultCode = "EXCEPTION";
+ resultDescription = e.getMessage();
+ throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
+ }
+ finally
+ {
+ if (methodThread != null)
+ {
+ methodThread.abort();
+ if (!wasInterrupted)
+ {
+ try
+ {
+ methodThread.finishUp();
+ }
+ catch (InterruptedException e)
+ {
+ wasInterrupted = true;
+ throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ }
+ }
+ }
}
- finally
+ else
{
- // Close stream via thread, since otherwise this can hang
- is.close();
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: No fetch URI "+contextMsg+" - not ingesting");
+ resultCode = "NOURI";
+ return;
}
}
- catch (java.net.SocketTimeoutException e)
- {
- resultCode = "DATATIMEOUT";
- resultDescription = e.getMessage();
- currentTime = System.currentTimeMillis();
- Logging.connectors.warn("Livelink: Livelink socket timed out ingesting from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
- throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
- }
- catch (java.net.SocketException e)
- {
- resultCode = "DATASOCKETERROR";
- resultDescription = e.getMessage();
- currentTime = System.currentTimeMillis();
- Logging.connectors.warn("Livelink: Livelink socket error ingesting from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
- throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
- }
- catch (javax.net.ssl.SSLHandshakeException e)
- {
- resultCode = "DATASSLHANDSHAKEERROR";
- resultDescription = e.getMessage();
- currentTime = System.currentTimeMillis();
- Logging.connectors.warn("Livelink: SSL handshake failed authenticating "+contextMsg+": "+e.getMessage(),e);
- throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
- }
- catch (ConnectTimeoutException e)
- {
- resultCode = "CONNECTTIMEOUT";
- resultDescription = e.getMessage();
- currentTime = System.currentTimeMillis();
- Logging.connectors.warn("Livelink: Livelink socket timed out connecting to the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
- throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
- }
- catch (InterruptedException e)
- {
- throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (InterruptedIOException e)
- {
- throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (HttpException e)
- {
- resultCode = "HTTPEXCEPTION";
- resultDescription = e.getMessage();
- // Treat unknown error ingesting data as a transient condition
- currentTime = System.currentTimeMillis();
- Logging.connectors.warn("Livelink: HTTP exception ingesting "+contextMsg+": "+e.getMessage(),e);
- throw new ServiceInterruption("HTTP exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
- }
- catch (IOException e)
+ else
{
- resultCode = "DATAEXCEPTION";
- resultDescription = e.getMessage();
- // Treat unknown error ingesting data as a transient condition
- currentTime = System.currentTimeMillis();
- Logging.connectors.warn("Livelink: IO exception ingesting "+contextMsg+": "+e.getMessage(),e);
- throw new ServiceInterruption("IO exception ingesting "+contextMsg+": "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,false);
+ // Use FetchVersion instead
+ long currentTime;
+
+ // Fire up the document reading thread
+ DocumentReadingThread t = new DocumentReadingThread(vol,objID,0);
+ try
+ {
+ t.start();
+ try
+ {
+ InputStream is = t.getSafeInputStream();
+ try
+ {
+ // Can only index while background thread is running!
+ rd.setBinary(is, dataSize);
+ activities.ingestDocument(documentIdentifier, version, viewHttpAddress, rd);
+ }
+ finally
+ {
+ is.close();
+ }
+ }
+ catch (ManifoldCFException e)
+ {
+ if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+ wasInterrupted = true;
+ throw e;
+ }
+ catch (java.net.SocketTimeoutException e)
+ {
+ throw e;
+ }
+ catch (InterruptedIOException e)
+ {
+ wasInterrupted = true;
+ throw e;
+ }
+ finally
+ {
+ if (!wasInterrupted)
+ t.finishUp();
+ }
+
+ // No errors. Record the fact that we made it.
+ resultCode = "OK";
+ readSize = dataSize;
+ }
+ catch (InterruptedException e)
+ {
+ t.interrupt();
+ throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
+ ManifoldCFException.INTERRUPTED);
+ }
+ catch (ConnectTimeoutException e)
+ {
+ Logging.connectors.warn("Livelink: Connect timed out "+contextMsg+": "+e.getMessage(), e);
+ resultCode = "CONNECTTIMEOUT";
+ resultDescription = e.getMessage();
+ currentTime = System.currentTimeMillis();
+ throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
+ }
+ catch (InterruptedIOException e)
+ {
+ t.interrupt();
+ throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+ }
+ catch (IOException e)
+ {
+ resultCode = "EXCEPTION";
+ resultDescription = e.getMessage();
+ throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
+ }
+ catch (ManifoldCFException e)
+ {
+ if (e.getErrorCode() != ManifoldCFException.INTERRUPTED)
+ {
+ resultCode = "EXCEPTION";
+ resultDescription = e.getMessage();
+ }
+ throw e;
+ }
+ catch (RuntimeException e)
+ {
+ resultCode = "EXCEPTION";
+ resultDescription = e.getMessage();
+ handleLivelinkRuntimeException(e,0,true);
+ }
}
- readSize = new Long(dataSize);
}
else
{
+ // Document not indexable because of its length
+ resultDescription = "Document length ("+dataSize+") was rejected by output connector";
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because its length ("+dataSize+") was rejected by output connector");
resultCode = "DOCUMENTTOOLONG";
activities.deleteDocument(documentIdentifier,version);
}
}
else
{
- resultCode = "SESSIONLOGINFAILED";
+ // Document not indexable because of its mime type
+ resultDescription = "Mime type ("+mimeType+") was rejected by output connector";
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because its mime type ("+mimeType+") was rejected by output connector");
+ resultCode = "MIMETYPEEXCLUSION";
activities.deleteDocument(documentIdentifier,version);
}
- break;
- case HttpStatus.SC_BAD_REQUEST:
- case HttpStatus.SC_USE_PROXY:
- case HttpStatus.SC_GONE:
- resultCode = "ERROR "+Integer.toString(statusCode);
- throw new ManifoldCFException("Unrecoverable request failure; error = "+Integer.toString(statusCode));
- default:
- resultCode = "UNKNOWN";
- Logging.connectors.warn("Livelink: Attempt to retrieve document from '"+ingestHttpAddress+"' received a response of "+Integer.toString(statusCode)+"; retrying in one minute");
- currentTime = System.currentTimeMillis();
- throw new ServiceInterruption("Fetch failed; retrying in 1 minute",new ManifoldCFException("Fetch failed with unknown code "+Integer.toString(statusCode)),
- currentTime+60000L,currentTime+600000L,-1,true);
}
- }
- catch (InterruptedException e)
- {
- // Drop the connection on the floor
- methodThread.interrupt();
- methodThread = null;
- throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (java.net.SocketTimeoutException e)
- {
- Logging.connectors.warn("Livelink: Socket timed out reading from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
- resultCode = "TIMEOUT";
- resultDescription = e.getMessage();
- currentTime = System.currentTimeMillis();
- throw new ServiceInterruption("Socket timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
- }
- catch (java.net.SocketException e)
- {
- Logging.connectors.warn("Livelink: Socket error reading from Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
- resultCode = "SOCKETERROR";
- resultDescription = e.getMessage();
- currentTime = System.currentTimeMillis();
- throw new ServiceInterruption("Socket error: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
- }
- catch (javax.net.ssl.SSLHandshakeException e)
- {
- currentTime = System.currentTimeMillis();
- Logging.connectors.warn("Livelink: SSL handshake failed "+contextMsg+": "+e.getMessage(),e);
- resultCode = "SSLHANDSHAKEERROR";
- resultDescription = e.getMessage();
- throw new ServiceInterruption("SSL handshake error: "+e.getMessage(),e,currentTime+60000L,currentTime+300000L,-1,true);
- }
- catch (ConnectTimeoutException e)
- {
- Logging.connectors.warn("Livelink: Connect timed out reading from the Livelink HTTP Server "+contextMsg+": "+e.getMessage(), e);
- resultCode = "CONNECTTIMEOUT";
- resultDescription = e.getMessage();
- currentTime = System.currentTimeMillis();
- throw new ServiceInterruption("Connect timed out: "+e.getMessage(),e,currentTime+300000L,currentTime+6*3600000L,-1,true);
- }
- catch (InterruptedIOException e)
- {
- throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (HttpException e)
- {
- resultCode = "EXCEPTION";
- resultDescription = e.getMessage();
- throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
- }
- catch (IOException e)
- {
- resultCode = "EXCEPTION";
- resultDescription = e.getMessage();
- throw new ManifoldCFException("Exception getting response "+contextMsg+": "+e.getMessage(), e);
+ else
+ {
+ // Document not ingestable due to URL
+ resultDescription = "URL ("+viewHttpAddress+") was rejected by output connector";
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("Livelink: Excluding document "+documentIdentifier+" because its URL ("+viewHttpAddress+") was rejected by output connector");
+ resultCode = "URLEXCLUSION";
+ activities.deleteDocument(documentIdentifier,version);
+ }
}
finally
{
- if (methodThread != null)
- {
- methodThread.abort();
- activities.recordActivity(new Long(startTime),ACTIVITY_FETCH,readSize,Integer.toString(objID),resultCode,resultDescription,null);
- try
- {
- methodThread.finishUp();
- }
- catch (InterruptedException e)
- {
- throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- }
+ if (!wasInterrupted)
+ activities.recordActivity(new Long(startTime),ACTIVITY_FETCH,readSize,vol+":"+objID,resultCode,resultDescription,null);
}
}
@@ -5430,7 +5601,18 @@ public class LivelinkConnector extends o
{
return getVersionValue() != null;
}
-
+
+ /** Get data size.
+ */
+ public Long getDataSize()
+ throws ServiceInterruption, ManifoldCFException
+ {
+ LLValue elem = getVersionValue();
+ if (elem == null)
+ return null;
+ return new Long(elem.toInteger("FILEDATASIZE"));
+ }
+
/** Get file name.
*/
public String getFileName()
@@ -5587,7 +5769,6 @@ public class LivelinkConnector extends o
return "(Volume: "+volumeID+", Object: "+objectID+")";
}
-
/**
* Returns the object ID specified by the path name.
* @param startPath is the folder name (a string with dots as separators)
@@ -7275,6 +7456,79 @@ public class LivelinkConnector extends o
}
+ /** This thread performs a LAPI FetchVersion command, streaming the resulting
+ * document back through a XThreadInputStream to the invoking thread.
+ */
+ protected class DocumentReadingThread extends Thread
+ {
+
+ protected Throwable exception = null;
+ protected final int volumeID;
+ protected final int docID;
+ protected final int versionNumber;
+ protected final XThreadInputStream stream;
+
+ public DocumentReadingThread(int volumeID, int docID, int versionNumber)
+ {
+ super();
+ this.volumeID = volumeID;
+ this.docID = docID;
+ this.versionNumber = versionNumber;
+ this.stream = new XThreadInputStream();
+ setDaemon(true);
+ }
+
+ @Override
+ public void run()
+ {
+ try
+ {
+ XThreadOutputStream outputStream = new XThreadOutputStream(stream);
+ try
+ {
+ int status = LLDocs.FetchVersion(volumeID, docID, versionNumber, outputStream);
+ if (status != 0)
+ {
+ throw new ManifoldCFException("Error retrieving contents of document "+Integer.toString(volumeID)+":"+Integer.toString(docID)+" revision "+versionNumber+" : Status="+Integer.toString(status)+" ("+llServer.getErrors()+")");
+ }
+ }
+ finally
+ {
+ outputStream.close();
+ }
+ } catch (Throwable e) {
+ this.exception = e;
+ }
+ }
+
+ public InputStream getSafeInputStream() {
+ return stream;
+ }
+
+ public void finishUp()
+ throws InterruptedException, ManifoldCFException
+ {
+ // This will be called during the finally
+ // block in the case where all is well (and
+ // the stream completed) and in the case where
+ // there were exceptions.
+ stream.abort();
+ join();
+ Throwable thr = exception;
+ if (thr != null) {
+ if (thr instanceof ManifoldCFException)
+ throw (ManifoldCFException) thr;
+ else if (thr instanceof RuntimeException)
+ throw (RuntimeException) thr;
+ else if (thr instanceof Error)
+ throw (Error) thr;
+ else
+ throw new RuntimeException("Unhandled exception of type: "+thr.getClass().getName(),thr);
+ }
+ }
+
+ }
+
/** This thread does the actual socket communication with the server.
* It's set up so that it can be abandoned at shutdown time.
*
Modified: manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties?rev=1493226&r1=1493225&r2=1493226&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties (original)
+++ manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_en_US.properties Fri Jun 14 19:43:42 2013
@@ -144,3 +144,8 @@ LivelinkConnector.TheServerCgiPathMustBe
LivelinkConnector.Delete=Delete
LivelinkConnector.Add=Add
LivelinkConnector.CrawlUserWorkspaces=Crawl user workspaces?
+
+LivelinkConnector.EnterTheViewCgiPathToLivelink=Enter the view CGI path to LiveLink
+LivelinkConnector.TheIngestCgiPathMustBeBlankOrBeginWithACharacter=The ingestion CGI path must be blank or begin with a '/' character
+LivelinkConnector.UseLAPI=Use LAPI
+LivelinkConnector.SelectAViewProtocol=Select a view protocol
Modified: manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_ja_JP.properties?rev=1493226&r1=1493225&r2=1493226&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_ja_JP.properties (original)
+++ manifoldcf/trunk/connectors/livelink/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/livelink/common_ja_JP.properties Fri Jun 14 19:43:42 2013
@@ -144,3 +144,8 @@ LivelinkConnector.TheServerCgiPathMustBe
LivelinkConnector.Delete=åé¤
LivelinkConnector.Add=追å
LivelinkConnector.CrawlUserWorkspaces=Crawl user workspaces?
+
+LivelinkConnector.EnterTheViewCgiPathToLivelink=Enter the view CGI path to LiveLink
+LivelinkConnector.TheIngestCgiPathMustBeBlankOrBeginWithACharacter=The ingestion CGI path must be blank or begin with a '/' character
+LivelinkConnector.UseLAPI=Use LAPI
+LivelinkConnector.SelectAViewProtocol=Select a view protocol