You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2015/01/05 18:49:12 UTC
svn commit: r1649598 - in
/manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src:
main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ test/
test/java/ test/java/org/ test/java/org/apache/ test/java/org/apache/man...
Author: kwright
Date: Mon Jan 5 17:49:11 2015
New Revision: 1649598
URL: http://svn.apache.org/r1649598
Log:
Add tests of expression evaluation, and make some methods public for that reason
Added:
manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/
manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/
manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/
manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/
manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/
manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/
manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/
manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java (with props)
Modified:
manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
Modified: manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java?rev=1649598&r1=1649597&r2=1649598&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java Mon Jan 5 17:49:11 2015
@@ -114,7 +114,12 @@ public class ForcedMetadataConnector ext
}
} else if (sp.keepAllMetadata()) {
// Copy ALL current fields from old document, but go through FieldDataFactory
- // MHL
+ Iterator<String> fields = document.getFields();
+ while (fields.hasNext())
+ {
+ String field = fields.next();
+ moveData(docCopy,field,fdf,field,false);
+ }
}
// Iterate through the expressions
@@ -624,7 +629,7 @@ public class ForcedMetadataConnector ext
return input;
}
- protected interface IDataSource {
+ public interface IDataSource {
public int getSize() throws IOException, ManifoldCFException;
public Object[] getRawForm() throws IOException, ManifoldCFException;
public String[] getStringForm() throws IOException, ManifoldCFException;
@@ -712,7 +717,7 @@ public class ForcedMetadataConnector ext
return new StringSource(rval);
}
- protected static IDataSource processExpression(String expression, FieldDataFactory sourceDocument)
+ public static IDataSource processExpression(String expression, FieldDataFactory sourceDocument)
throws IOException, ManifoldCFException {
int index = 0;
IDataSource input = null;
@@ -874,7 +879,7 @@ public class ForcedMetadataConnector ext
* a CharacterInput implementation, thus making a temporary file copy. So it is imperative
* that this object is closed when it is no longer needed.
*/
- protected static class FieldDataFactory {
+ public static class FieldDataFactory {
protected final RepositoryDocument sourceDocument;
Added: manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java?rev=1649598&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java (added)
+++ manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java Mon Jan 5 17:49:11 2015
@@ -0,0 +1,89 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.connectors.webcrawler;
+
+import org.apache.manifoldcf.agents.transformation.forcedmetadata.*;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import java.io.*;
+import java.util.*;
+
+import org.junit.*;
+import static org.junit.Assert.*;
+
+public class ExpressionTest
+{
+
+ @Test
+ public void simpleExpressions()
+ throws Exception {
+ RepositoryDocument inputDoc = new RepositoryDocument();
+ inputDoc.addField("stringfield",new String[]{"stringa","stringb","stringc"});
+ inputDoc.addField("readerfield",new Reader[]{new StringReader("readera"),new StringReader("readerb")});
+ inputDoc.addField("datefield",new Date[]{new Date(0L), new Date(100000000L)});
+ ForcedMetadataConnector.FieldDataFactory fdf = new ForcedMetadataConnector.FieldDataFactory(inputDoc);
+ try {
+ arrayEquals(new String[]{"stringa","stringb","stringc"}, (String[])(ForcedMetadataConnector.processExpression("${stringfield}", fdf).getRawForm()));
+ arrayEquals(new String[]{"prefixstringapostfix","prefixstringbpostfix","prefixstringcpostfix"}, (String[])(ForcedMetadataConnector.processExpression("prefix${stringfield}postfix", fdf).getRawForm()));
+ arrayEquals(new Reader[]{new StringReader("readera"),new StringReader("readerb")}, (Reader[])(ForcedMetadataConnector.processExpression("${readerfield}", fdf).getRawForm()));
+ // Second access of reader fields, without prior string conversion, also must work
+ arrayEquals(new Reader[]{new StringReader("readera"),new StringReader("readerb")}, (Reader[])(ForcedMetadataConnector.processExpression("${readerfield}", fdf).getRawForm()));
+ arrayEquals(new String[]{"prefixreaderapostfix","prefixreaderbpostfix"}, (String[])(ForcedMetadataConnector.processExpression("prefix${readerfield}postfix", fdf).getRawForm()));
+ } finally {
+ fdf.close();
+ }
+ }
+
+ protected static void arrayEquals(Reader[] expected, Reader[] actual)
+ throws Exception {
+ assertEquals(expected.length,actual.length);
+ Set<String> expectedSet = new HashSet<String>();
+ for (Reader expectedValue : expected) {
+ expectedSet.add(readData(expectedValue));
+ }
+ for (Reader actualValue : actual) {
+ assertEquals(true,expectedSet.contains(readData(actualValue)));
+ }
+
+ }
+
+ protected static String readData(Reader r)
+ throws IOException {
+ StringBuilder sb = new StringBuilder();
+ char[] buffer = new char[65536];
+ while (true) {
+ int amt = r.read(buffer);
+ if (amt == -1)
+ break;
+ sb.append(buffer,0,amt);
+ }
+ return sb.toString();
+ }
+
+ protected static void arrayEquals(String[] expected, String[] actual)
+ throws Exception {
+ assertEquals(expected.length,actual.length);
+ Set<String> expectedSet = new HashSet<String>();
+ for (String expectedValue : expected) {
+ expectedSet.add(expectedValue);
+ }
+ for (String actualValue : actual) {
+ assertEquals(true,expectedSet.contains(actualValue));
+ }
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-1134/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java
------------------------------------------------------------------------------
svn:keywords = Id