You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/10/04 14:12:44 UTC
svn commit: r1629377 - in /manifoldcf/branches/CONNECTORS-1037:
connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/
connectors/alfresco-webscript/connector/src/main/java/org/apache/mani...
Author: kwright
Date: Sat Oct 4 12:12:43 2014
New Revision: 1629377
URL: http://svn.apache.org/r1629377
Log:
Fix tests, url generation, document filtering, and obfuscation of password values
Modified:
manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/AlfrescoAuthorityConnector.java
manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/ConfigurationHandler.java
manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnector.java
manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/ConfigurationHandler.java
manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnectorTest.java
manifoldcf/branches/CONNECTORS-1037/framework/core/src/main/java/org/apache/manifoldcf/core/common/DateParser.java
manifoldcf/branches/CONNECTORS-1037/framework/core/src/test/java/org/apache/manifoldcf/core/common/DateTest.java
Modified: manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/AlfrescoAuthorityConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/AlfrescoAuthorityConnector.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/AlfrescoAuthorityConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/AlfrescoAuthorityConnector.java Sat Oct 4 12:12:43 2014
@@ -48,7 +48,7 @@ public class AlfrescoAuthorityConnector
String hostname = getConfig(config, "hostname", "localhost");
String endpoint = getConfig(config, "endpoint", "/alfresco/service");
String username = getConfig(config, "username", null);
- String password = getConfig(config, "password", null);
+ String password = getObfuscatedConfig(config, "password", null);
alfrescoClient = new WebScriptsAlfrescoClient(protocol, hostname, endpoint,
null, null, username, password);
@@ -64,6 +64,16 @@ public class AlfrescoAuthorityConnector
return protocol;
}
+ private static String getObfuscatedConfig(ConfigParams config,
+ String parameter,
+ String defaultValue) {
+ final String protocol = config.getObfuscatedParameter(parameter);
+ if (protocol == null) {
+ return defaultValue;
+ }
+ return protocol;
+ }
+
/*
* (non-Javadoc)
* @see org.apache.manifoldcf.core.connector.BaseConnector#check()
Modified: manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/ConfigurationHandler.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/ConfigurationHandler.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/ConfigurationHandler.java (original)
+++ manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/alfrescowebscript/ConfigurationHandler.java Sat Oct 4 12:12:43 2014
@@ -59,19 +59,28 @@ public class ConfigurationHandler {
List<String> tabsArray) throws ManifoldCFException, IOException {
tabsArray.add("Server");
Map<String, Object> paramMap = new HashMap<String, Object>();
- fillInParameters(paramMap, parameters);
+ fillInParameters(paramMap, out, parameters);
Messages.outputResourceWithVelocity(out, locale, EDIT_CONFIG_HEADER, paramMap);
}
private static void fillInParameters(Map<String, Object> paramMap,
- ConfigParams parameters) {
+ IHTTPOutput out, ConfigParams parameters) {
for (Map.Entry<String, String> parameter : DEFAULT_CONFIGURATION_PARAMETERS
.entrySet()) {
- String paramValue = parameters.getParameter(parameter.getKey());
- if (paramValue == null) {
- paramValue = parameter.getValue();
+ String paramName = parameter.getKey();
+ if (paramName.endsWith("password")) {
+ String paramValue = parameters.getObfuscatedParameter(paramName);
+ if (paramValue == null) {
+ paramValue = parameter.getValue();
+ }
+ paramMap.put(paramName, out.mapPasswordToKey(paramValue));
+ } else {
+ String paramValue = parameters.getParameter(paramName);
+ if (paramValue == null) {
+ paramValue = parameter.getValue();
+ }
+ paramMap.put(paramName, paramValue);
}
- paramMap.put(parameter.getKey(), paramValue);
}
}
@@ -81,7 +90,7 @@ public class ConfigurationHandler {
throws ManifoldCFException, IOException {
Map<String, Object> paramMap = new HashMap<String, Object>();
paramMap.put("tabName", tabName);
- fillInParameters(paramMap, parameters);
+ fillInParameters(paramMap, out, parameters);
Messages.outputResourceWithVelocity(out, locale, EDIT_CONFIG_SERVER, paramMap);
}
@@ -92,7 +101,10 @@ public class ConfigurationHandler {
for (String paramName : DEFAULT_CONFIGURATION_PARAMETERS.keySet()) {
String paramValue = variableContext.getParameter(paramName);
if (paramValue != null) {
- parameters.setParameter(paramName, paramValue);
+ if (paramName.endsWith("password"))
+ parameters.setObfuscatedParameter(paramName,variableContext.mapKeyToPassword(paramValue));
+ else
+ parameters.setParameter(paramName, paramValue);
}
}
return null;
@@ -103,7 +115,7 @@ public class ConfigurationHandler {
ConfigParams parameters)
throws ManifoldCFException, IOException {
Map<String, Object> paramMap = new HashMap<String, Object>();
- fillInParameters(paramMap, parameters);
+ fillInParameters(paramMap, out, parameters);
Messages.outputResourceWithVelocity(out, locale, VIEW_CONFIG, paramMap);
}
}
Modified: manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnector.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnector.java Sat Oct 4 12:12:43 2014
@@ -23,6 +23,7 @@ import org.alfresco.consulting.indexer.c
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.core.common.DateParser;
import org.apache.manifoldcf.crawler.connectors.BaseRepositoryConnector;
import org.apache.manifoldcf.crawler.interfaces.IExistingVersions;
import org.apache.manifoldcf.crawler.interfaces.IProcessActivity;
@@ -31,6 +32,7 @@ import org.apache.manifoldcf.crawler.sys
import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.io.InputStream;
import java.text.MessageFormat;
import java.text.ParseException;
@@ -76,7 +78,7 @@ public class AlfrescoConnector extends B
String storeProtocol = getConfig(config, "storeprotocol", "workspace");
String storeId = getConfig(config, "storeid", "SpacesStore");
String username = getConfig(config, "username", null);
- String password = getConfig(config, "password", null);
+ String password = getObfuscatedConfig(config, "password", null);
this.enableDocumentProcessing = new Boolean(getConfig(config, "enabledocumentprocessing", "false"));
alfrescoClient = new WebScriptsAlfrescoClient(protocol, hostname, endpoint,
@@ -93,9 +95,28 @@ public class AlfrescoConnector extends B
return protocol;
}
+ private static String getObfuscatedConfig(ConfigParams config,
+ String parameter,
+ String defaultValue) {
+ final String protocol = config.getObfuscatedParameter(parameter);
+ if (protocol == null) {
+ return defaultValue;
+ }
+ return protocol;
+ }
+
@Override
public String check() throws ManifoldCFException {
- return super.check();
+ try {
+ // We really want to do something more like fetching a document here...
+ alfrescoClient.fetchUserAuthorities("admin");
+ return super.check();
+ } catch (AlfrescoDownException e) {
+ if (Logging.connectors != null) {
+ Logging.connectors.warn(e.getMessage(), e);
+ }
+ return "Connection failed: " + e.getMessage();
+ }
}
@Override
@@ -164,7 +185,8 @@ public class AlfrescoConnector extends B
Logging.connectors.debug(MessageFormat.format("Recording {0} as last transaction id and {1} as last changeset id", new Object[]{lastTransactionId, lastAclChangesetId}));
return lastTransactionId + "|" + lastAclChangesetId;
} catch (AlfrescoDownException e) {
- throw new ManifoldCFException(e);
+ handleAlfrescoDownException(e,"seeding");
+ return null;
}
}
@@ -173,121 +195,128 @@ public class AlfrescoConnector extends B
IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
throws ManifoldCFException, ServiceInterruption {
- for (String doc : documentIdentifiers) {
+ try {
+ for (String doc : documentIdentifiers) {
- String nextVersion = statuses.getIndexedVersionString(doc);
-
- // Calling again Alfresco API because Document's actions are lost from seeding method
- AlfrescoResponse response = alfrescoClient.fetchNode(doc);
- if(response.getDocumentList().isEmpty()){ // Not found seeded document. Could reflect an error in Alfresco
- if (Logging.connectors != null)
- Logging.connectors.warn(MessageFormat.format("Invalid Seeded Document from Alfresco with ID {0}", new Object[]{doc}));
- activities.deleteDocument(doc);
- continue;
- }
- Map<String, Object> map = response.getDocumentList().get(0); // Should be only one
- if ((Boolean) map.get("deleted")) {
- activities.deleteDocument(doc);
- continue;
- }
+ String nextVersion = statuses.getIndexedVersionString(doc);
+
+ // Calling again Alfresco API because Document's actions are lost from seeding method
+ AlfrescoResponse response = alfrescoClient.fetchNode(doc);
+ if(response.getDocumentList().isEmpty()){ // Not found seeded document. Could reflect an error in Alfresco
+ if (Logging.connectors != null)
+ Logging.connectors.warn(MessageFormat.format("Invalid Seeded Document from Alfresco with ID {0}", new Object[]{doc}));
+ activities.deleteDocument(doc);
+ continue;
+ }
+ Map<String, Object> map = response.getDocumentList().get(0); // Should be only one
+ if ((Boolean) map.get("deleted")) {
+ activities.deleteDocument(doc);
+ continue;
+ }
- // From the map, get the things we know about
- String uuid = map.containsKey(FIELD_UUID) ? map.get(FIELD_UUID).toString() : doc;
- String nodeRef = map.containsKey(FIELD_NODEREF) ? map.get(FIELD_NODEREF).toString() : "";
- String type = map.containsKey(FIELD_TYPE) ? map.get(FIELD_TYPE).toString() : "";
- String name = map.containsKey(FIELD_NAME) ? map.get(FIELD_NAME).toString() : "";
-
- // Fetch document metadata
- Map<String,Object> properties = alfrescoClient.fetchMetadata(uuid);
-
- // Process various special fields
- Object mdObject;
-
- // Size
- Long lSize = null;
- mdObject = properties.get(SIZE_PROPERTY);
- if (mdObject != null) {
- String size = mdObject.toString();
- lSize = new Long(size);
- }
-
- // Modified Date
- Date modifiedDate = null;
- mdObject = properties.get(MODIFIED_DATE_PROPERTY);
- if (mdObject != null) {
- SimpleDateFormat f = new SimpleDateFormat("YYYY-MM-DDThh:mm:ss.sTZD");
- try {
- modifiedDate = f.parse(mdObject.toString());
- } catch (ParseException e) {
- // Nothing to do
+ // From the map, get the things we know about
+ String uuid = doc;
+ String nodeRef = map.containsKey(FIELD_NODEREF) ? map.get(FIELD_NODEREF).toString() : "";
+ String type = map.containsKey(FIELD_TYPE) ? map.get(FIELD_TYPE).toString() : "";
+ String name = map.containsKey(FIELD_NAME) ? map.get(FIELD_NAME).toString() : "";
+
+ // Fetch document metadata
+ Map<String,Object> properties = alfrescoClient.fetchMetadata(uuid);
+
+ // Process various special fields
+ Object mdObject;
+
+ // Size
+ Long lSize = null;
+ mdObject = properties.get(SIZE_PROPERTY);
+ if (mdObject != null) {
+ String size = mdObject.toString();
+ lSize = new Long(size);
+ }
+
+ // Modified Date
+ Date modifiedDate = null;
+ mdObject = properties.get(MODIFIED_DATE_PROPERTY);
+ if (mdObject != null) {
+ modifiedDate = DateParser.parseISO8601Date(mdObject.toString());
}
- }
- // Created Date
- Date createdDate = null;
- mdObject = properties.get(CREATED_DATE_PROPERTY);
- if (mdObject != null) {
- SimpleDateFormat f = new SimpleDateFormat("YYYY-MM-DDThh:mm:ss.sTZD");
- try {
- createdDate = f.parse(mdObject.toString());
- } catch (ParseException e) {
- // Nothing to do
+ // Created Date
+ Date createdDate = null;
+ mdObject = properties.get(CREATED_DATE_PROPERTY);
+ if (mdObject != null) {
+ createdDate = DateParser.parseISO8601Date(mdObject.toString());
}
- }
- // Establish the document version.
- if (modifiedDate == null) {
- activities.deleteDocument(doc);
- continue;
- }
-
- String documentVersion = modifiedDate.toString();
+ // Establish the document version.
+ if (modifiedDate == null) {
+ activities.deleteDocument(doc);
+ continue;
+ }
+
+ String documentVersion = new Long(modifiedDate.getTime()).toString();
- if(!activities.checkDocumentNeedsReindexing(doc, documentVersion))
- continue;
+ if(!activities.checkDocumentNeedsReindexing(doc, documentVersion))
+ continue;
- RepositoryDocument rd = new RepositoryDocument();
- rd.addField(FIELD_NODEREF, nodeRef);
- rd.addField(FIELD_TYPE, type);
- rd.setFileName(name);
-
- if (modifiedDate != null)
- rd.setModifiedDate(modifiedDate);
+ String mimeType = null;
+ Object mimetypeObject = properties.get(MIMETYPE_PROPERTY);
+ if (mimetypeObject != null) {
+ mimeType = mimetypeObject.toString();
+ }
- if (createdDate != null)
- rd.setCreatedDate(createdDate);
+ if (lSize != null && !activities.checkLengthIndexable(lSize.longValue())) {
+ activities.noDocument(doc, documentVersion);
+ continue;
+ }
+
+ if (mimeType != null && !activities.checkMimeTypeIndexable(mimeType)) {
+ activities.noDocument(doc, documentVersion);
+ continue;
+ }
- for(String property : properties.keySet()) {
- Object propertyValue = properties.get(property);
- rd.addField(property,propertyValue.toString());
- }
+ RepositoryDocument rd = new RepositoryDocument();
+ rd.addField(FIELD_NODEREF, nodeRef);
+ rd.addField(FIELD_TYPE, type);
+ rd.setFileName(name);
+
+ if (modifiedDate != null)
+ rd.setModifiedDate(modifiedDate);
- Object mimetypeObject = properties.get(MIMETYPE_PROPERTY);
- if (mimetypeObject != null) {
- String mimetype = mimetypeObject.toString();
- if (mimetype != null && !mimetype.isEmpty())
- rd.setMimeType(mimetype);
- }
+ if (createdDate != null)
+ rd.setCreatedDate(createdDate);
- // Indexing Permissions
- @SuppressWarnings("unchecked")
- List<String> permissions = (List<String>) properties.remove(AUTHORITIES_PROPERTY);
- if(permissions != null){
- rd.setSecurityACL(RepositoryDocument.SECURITY_TYPE_DOCUMENT,
- permissions.toArray(new String[permissions.size()]));
- }
+ for(String property : properties.keySet()) {
+ Object propertyValue = properties.get(property);
+ rd.addField(property,propertyValue.toString());
+ }
+
+ if (mimeType != null && !mimeType.isEmpty())
+ rd.setMimeType(mimeType);
+
+ // Indexing Permissions
+ @SuppressWarnings("unchecked")
+ List<String> permissions = (List<String>) properties.remove(AUTHORITIES_PROPERTY);
+ if(permissions != null){
+ rd.setSecurityACL(RepositoryDocument.SECURITY_TYPE_DOCUMENT,
+ permissions.toArray(new String[permissions.size()]));
+ }
- // Document Binary Content
- InputStream stream;
- long length;
- String contentUrlPath = (String) properties.get(CONTENT_URL_PROPERTY);
- if (contentUrlPath != null && !contentUrlPath.isEmpty()) {
+ // Document Binary Content
+ InputStream stream;
+ long length;
byte[] empty = new byte[0];
- if (this.enableDocumentProcessing) {
- if (lSize != null) {
- stream = alfrescoClient.fetchContent(contentUrlPath);
- length = lSize.longValue();
+ String contentUrlPath = (String) properties.get(CONTENT_URL_PROPERTY);
+ if (contentUrlPath != null && !contentUrlPath.isEmpty()) {
+ if (this.enableDocumentProcessing) {
+ if (lSize != null) {
+ stream = alfrescoClient.fetchContent(contentUrlPath);
+ length = lSize.longValue();
+ } else {
+ stream = new ByteArrayInputStream(empty);
+ length = 0L;
+ }
} else {
stream = new ByteArrayInputStream(empty);
length = 0L;
@@ -301,27 +330,122 @@ public class AlfrescoConnector extends B
rd.setBinary(stream, length);
if (Logging.connectors != null && Logging.connectors.isDebugEnabled())
Logging.connectors.debug(MessageFormat.format("Ingesting with id: {0}, URI {1} and rd {2}", new Object[]{uuid, nodeRef, rd.getFileName()}));
- activities.ingestDocumentWithException(doc, documentVersion, uuid, rd);
+ activities.ingestDocumentWithException(doc, documentVersion, nodeRef/*was uuid*/, rd);
} catch (IOException e) {
- handleIOException(e);
+ handleIOException(e,"reading stream");
} finally {
try {
stream.close();
} catch (IOException e) {
- handleIOException(e);
+ handleIOException(e,"closing stream");
}
}
- }
+ }
+ } catch (AlfrescoDownException e) {
+ handleAlfrescoDownException(e,"processing");
}
}
+
+ protected final static long interruptionRetryTime = 5L*60L*1000L;
- protected static void handleIOException(IOException e)
+ protected static void handleAlfrescoDownException(AlfrescoDownException e, String context)
throws ManifoldCFException, ServiceInterruption {
- // MHL
- throw new ManifoldCFException(
- "Error Ingesting Document:" +e.getMessage(), e);
+ long currentTime = System.currentTimeMillis();
+
+ // Server doesn't appear to by up. Try for a brief time then give up.
+ String message = "Server appears down during "+context+": "+e.getMessage();
+ Logging.connectors.warn(message,e);
+ throw new ServiceInterruption(message,
+ e,
+ currentTime + interruptionRetryTime,
+ -1L,
+ 3,
+ true);
+ }
+
+ protected static void handleIOException(IOException e, String context)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ if ((e instanceof InterruptedIOException) && (!(e instanceof java.net.SocketTimeoutException)))
+ throw new ManifoldCFException(e.getMessage(), ManifoldCFException.INTERRUPTED);
+
+ long currentTime = System.currentTimeMillis();
+
+ if (e instanceof java.net.ConnectException)
+ {
+ // Server isn't up at all. Try for a brief time then give up.
+ String message = "Server could not be contacted during "+context+": "+e.getMessage();
+ Logging.connectors.warn(message,e);
+ throw new ServiceInterruption(message,
+ e,
+ currentTime + interruptionRetryTime,
+ -1L,
+ 3,
+ true);
+ }
+
+ if (e instanceof java.net.SocketTimeoutException)
+ {
+ String message2 = "Socket timeout exception during "+context+": "+e.getMessage();
+ Logging.connectors.warn(message2,e);
+ throw new ServiceInterruption(message2,
+ e,
+ currentTime + interruptionRetryTime,
+ currentTime + 20L * 60000L,
+ -1,
+ false);
+ }
+
+ if (e.getClass().getName().equals("java.net.SocketException"))
+ {
+ // In the past we would have treated this as a straight document rejection, and
+ // treated it in the same manner as a 400. The reasoning is that the server can
+ // perfectly legally send out a 400 and drop the connection immediately thereafter,
+ // this a race condition.
+ // However, Solr 4.0 (or the Jetty version that the example runs on) seems
+ // to have a bug where it drops the connection when two simultaneous documents come in
+ // at the same time. This is the final version of Solr 4.0 so we need to deal with
+ // this.
+ if (e.getMessage().toLowerCase(Locale.ROOT).indexOf("broken pipe") != -1 ||
+ e.getMessage().toLowerCase(Locale.ROOT).indexOf("connection reset") != -1 ||
+ e.getMessage().toLowerCase(Locale.ROOT).indexOf("target server failed to respond") != -1)
+ {
+ // Treat it as a service interruption, but with a limited number of retries.
+ // In that way we won't burden the user with a huge retry interval; it should
+ // give up fairly quickly, and yet NOT give up if the error was merely transient
+ String message = "Server dropped connection during "+context+": "+e.getMessage();
+ Logging.connectors.warn(message,e);
+ throw new ServiceInterruption(message,
+ e,
+ currentTime + interruptionRetryTime,
+ -1L,
+ 3,
+ false);
+ }
+
+ // Other socket exceptions are service interruptions - but if we keep getting them, it means
+ // that a socket timeout is probably set too low to accept this particular document. So
+ // we retry for a while, then skip the document.
+ String message2 = "Socket exception during "+context+": "+e.getMessage();
+ Logging.connectors.warn(message2,e);
+ throw new ServiceInterruption(message2,
+ e,
+ currentTime + interruptionRetryTime,
+ currentTime + 20L * 60000L,
+ -1,
+ false);
+ }
+ // Otherwise, no idea what the trouble is, so presume that retries might fix it.
+ String message3 = "IO exception during "+context+": "+e.getMessage();
+ Logging.connectors.warn(message3,e);
+ throw new ServiceInterruption(message3,
+ e,
+ currentTime + interruptionRetryTime,
+ currentTime + 2L * 60L * 60000L,
+ -1,
+ true);
}
@Override
Modified: manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/ConfigurationHandler.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/ConfigurationHandler.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/ConfigurationHandler.java (original)
+++ manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/ConfigurationHandler.java Sat Oct 4 12:12:43 2014
@@ -112,19 +112,28 @@ public class ConfigurationHandler {
List<String> tabsArray) throws ManifoldCFException, IOException {
tabsArray.add("Server");
Map<String, Object> paramMap = new HashMap<String, Object>();
- fillInParameters(paramMap, parameters);
+ fillInParameters(paramMap, out, parameters);
Messages.outputResourceWithVelocity(out, locale, EDIT_CONFIG_HEADER, paramMap);
}
private static void fillInParameters(Map<String, Object> paramMap,
- ConfigParams parameters) {
+ IHTTPOutput out, ConfigParams parameters) {
for (Map.Entry<String, String> parameter : DEFAULT_CONFIGURATION_PARAMETERS
.entrySet()) {
- String paramValue = parameters.getParameter(parameter.getKey());
- if (paramValue == null) {
- paramValue = parameter.getValue();
+ String paramName = parameter.getKey();
+ if (paramName.endsWith("password")) {
+ String paramValue = parameters.getObfuscatedParameter(paramName);
+ if (paramValue == null) {
+ paramValue = parameter.getValue();
+ }
+ paramMap.put(paramName, out.mapPasswordToKey(paramValue));
+ } else {
+ String paramValue = parameters.getParameter(paramName);
+ if (paramValue == null) {
+ paramValue = parameter.getValue();
+ }
+ paramMap.put(paramName, paramValue);
}
- paramMap.put(parameter.getKey(), paramValue);
}
}
@@ -133,7 +142,7 @@ public class ConfigurationHandler {
throws ManifoldCFException, IOException {
Map<String, Object> paramMap = new HashMap<String, Object>();
paramMap.put("tabName", tabName);
- fillInParameters(paramMap, parameters);
+ fillInParameters(paramMap, out, parameters);
Messages.outputResourceWithVelocity(out, locale, EDIT_CONFIG_SERVER, paramMap);
}
@@ -143,7 +152,10 @@ public class ConfigurationHandler {
for (String paramName : DEFAULT_CONFIGURATION_PARAMETERS.keySet()) {
String paramValue = variableContext.getParameter(paramName);
if (paramValue != null) {
- parameters.setParameter(paramName, paramValue);
+ if (paramName.endsWith("password"))
+ parameters.setObfuscatedParameter(paramName,variableContext.mapKeyToPassword(paramValue));
+ else
+ parameters.setParameter(paramName, paramValue);
}
}
return null;
@@ -153,7 +165,7 @@ public class ConfigurationHandler {
IHTTPOutput out, Locale locale, ConfigParams parameters)
throws ManifoldCFException, IOException {
Map<String, Object> paramMap = new HashMap<String, Object>();
- fillInParameters(paramMap, parameters);
+ fillInParameters(paramMap, out, parameters);
Messages.outputResourceWithVelocity(out, locale, VIEW_CONFIG, paramMap);
}
Modified: manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnectorTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnectorTest.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnectorTest.java (original)
+++ manifoldcf/branches/CONNECTORS-1037/connectors/alfresco-webscript/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/alfrescowebscript/AlfrescoConnectorTest.java Sat Oct 4 12:12:43 2014
@@ -18,6 +18,7 @@ package org.apache.manifoldcf.crawler.co
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
@@ -116,12 +117,24 @@ public class AlfrescoConnectorTest {
@Test
public void whenProcessingDocumentsNodeRefsAreUsedAsDocumentURI() throws Exception {
TestDocument testDocument = new TestDocument();
+ testDocument.put("cm:modified","2014-10-02T16:15:25.124Z");
+ testDocument.put("size","115");
+ testDocument.put("mimetype","text/plain");
IProcessActivity activities = mock(IProcessActivity.class);
+ when(activities.checkDocumentNeedsReindexing(anyString(),anyString()))
+ .thenReturn(true);
+ when(activities.checkLengthIndexable(anyLong()))
+ .thenReturn(true);
+ when(activities.checkMimeTypeIndexable(anyString()))
+ .thenReturn(true);
IExistingVersions statuses = mock(IExistingVersions.class);
when(client.fetchNode(anyString()))
.thenReturn(new AlfrescoResponse(0, 0, "", "",
Arrays.<Map<String, Object>>asList(testDocument)));
+
+ when(client.fetchMetadata(anyString()))
+ .thenReturn(testDocument);
// processDocuments(String[] documentIdentifiers, IExistingVersions statuses, Specification spec,
// IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
@@ -129,13 +142,24 @@ public class AlfrescoConnectorTest {
ArgumentCaptor<RepositoryDocument> rd = ArgumentCaptor.forClass(RepositoryDocument.class);
verify(activities)
+ .checkDocumentNeedsReindexing(eq(TestDocument.uuid), eq("1412266525124"));
+ verify(activities)
+ .checkLengthIndexable(eq(115L));
+ verify(activities)
+ .checkMimeTypeIndexable(eq("text/plain"));
+ verify(activities)
.ingestDocumentWithException(eq(TestDocument.uuid), anyString(),
- eq(TestDocument.uuid), rd.capture());
-
+ eq(TestDocument.nodeRef), rd.capture());
+
Iterator<String> i = rd.getValue().getFields();
while(i.hasNext()) {
String fieldName = i.next();
Object value1 = rd.getValue().getField(fieldName)[0];
+ Object[] values = testDocument.getRepositoryDocument().getField(fieldName);
+ if (values == null) {
+ System.out.println("field "+fieldName+"has no value in testDocument");
+ continue;
+ }
Object value2 = testDocument.getRepositoryDocument().getField(fieldName)[0];
assert value1.equals(value2);
}
@@ -147,8 +171,8 @@ public class AlfrescoConnectorTest {
testDocument.setDeleted(true);
when(client.fetchNode(anyString()))
- .thenReturn(new AlfrescoResponse(0, 0, "", "",
- Arrays.<Map<String, Object>>asList(testDocument)));
+ .thenReturn(new AlfrescoResponse(0, 0, "", "",
+ Arrays.<Map<String, Object>>asList(testDocument)));
IProcessActivity activities = mock(IProcessActivity.class);
IExistingVersions statuses = mock(IExistingVersions.class);
Modified: manifoldcf/branches/CONNECTORS-1037/framework/core/src/main/java/org/apache/manifoldcf/core/common/DateParser.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/framework/core/src/main/java/org/apache/manifoldcf/core/common/DateParser.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/framework/core/src/main/java/org/apache/manifoldcf/core/common/DateParser.java (original)
+++ manifoldcf/branches/CONNECTORS-1037/framework/core/src/main/java/org/apache/manifoldcf/core/common/DateParser.java Sat Oct 4 12:12:43 2014
@@ -51,7 +51,10 @@ public class DateParser
if (isoDateValue.indexOf(".") != -1)
isoFormatString.append(".SSS");
if (isoDateValue.endsWith("Z"))
- isoFormatString.append("'Z'");
+ {
+ isoDateValue = isoDateValue.substring(0,isoDateValue.length()-1) + "-0000";
+ isoFormatString.append("Z");
+ }
else
{
// We need to be able to parse either "-08:00" or "-0800". The 'Z' specifier only handles
Modified: manifoldcf/branches/CONNECTORS-1037/framework/core/src/test/java/org/apache/manifoldcf/core/common/DateTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1037/framework/core/src/test/java/org/apache/manifoldcf/core/common/DateTest.java?rev=1629377&r1=1629376&r2=1629377&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1037/framework/core/src/test/java/org/apache/manifoldcf/core/common/DateTest.java (original)
+++ manifoldcf/branches/CONNECTORS-1037/framework/core/src/test/java/org/apache/manifoldcf/core/common/DateTest.java Sat Oct 4 12:12:43 2014
@@ -31,8 +31,9 @@ public class DateTest
{
Date d = DateParser.parseISO8601Date("96-11-15T01:32:33.344GMT");
assertNotNull(d);
- d = DateParser.parseISO8601Date("2012-11-15T01:32:33.344Z");
- assertNotNull(d);
+ Date d2 = DateParser.parseISO8601Date("1996-11-15T01:32:33.344Z");
+ assertNotNull(d2);
+ assertEquals(d,d2);
d = DateParser.parseISO8601Date("2012-11-15T01:32:33Z");
assertNotNull(d);
d = DateParser.parseISO8601Date("2012-11-15T01:32:33+0100");