You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/10/10 14:49:18 UTC
svn commit: r1630812 - in
/manifoldcf/trunk/connectors/forcedmetadata/connector/src/main:
java/org/apache/manifoldcf/agents/transformation/forcedmetadata/
native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/
resources/org/apache/man...
Author: kwright
Date: Fri Oct 10 12:49:17 2014
New Revision: 1630812
URL: http://svn.apache.org/r1630812
Log:
Part of CONNECTORS-1066. Add empty metadata value filtering to Metadata Adjuster transformer.
Modified:
manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties
manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties
manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties
manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_FieldMapping.html
manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html
Modified: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java?rev=1630812&r1=1630811&r2=1630812&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java (original)
+++ manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java Fri Oct 10 12:49:17 2014
@@ -38,6 +38,7 @@ public class ForcedMetadataConnector ext
public static final String ATTRIBUTE_PARAMETER = "parameter";
public static final String NODE_FIELDMAP = "fieldmap";
public static final String NODE_KEEPMETADATA = "keepAllMetadata";
+ public static final String NODE_FILTEREMPTY = "filterEmpty";
public static final String ATTRIBUTE_SOURCE = "source";
public static final String ATTRIBUTE_TARGET = "target";
public static final String ATTRIBUTE_VALUE = "value";
@@ -98,27 +99,16 @@ public class ForcedMetadataConnector ext
while (fields.hasNext())
{
String field = fields.next();
- Object[] fieldData = document.getField(field);
String target = sp.getMapping(field);
if (target != null)
{
- if (fieldData instanceof Date[])
- docCopy.addField(target,(Date[])fieldData);
- else if (fieldData instanceof Reader[])
- docCopy.addField(target,(Reader[])fieldData);
- else if (fieldData instanceof String[])
- docCopy.addField(target,(String[])fieldData);
+ moveData(docCopy,target,document,field,sp.filterEmpty());
}
else
{
if (sp.keepAllMetadata())
{
- if (fieldData instanceof Date[])
- docCopy.addField(field,(Date[])fieldData);
- else if (fieldData instanceof Reader[])
- docCopy.addField(field,(Reader[])fieldData);
- else if (fieldData instanceof String[])
- docCopy.addField(field,(String[])fieldData);
+ moveData(docCopy,field,document,field,sp.filterEmpty());
}
}
}
@@ -135,6 +125,76 @@ public class ForcedMetadataConnector ext
return activities.sendDocument(documentURI,docCopy);
}
+ protected static void moveData(RepositoryDocument docCopy, String target, RepositoryDocument document, String field, boolean filterEmpty)
+ throws ManifoldCFException, IOException
+ {
+ Object[] fieldData = document.getField(field);
+ if (fieldData instanceof Date[])
+ docCopy.addField(target,(Date[])conditionallyRemoveNulls(fieldData,filterEmpty));
+ else if (fieldData instanceof Reader[])
+ {
+ // To strip out empty fields, we will need to convert readers to strings
+ if (filterEmpty)
+ docCopy.addField(target,removeEmpties(document.getFieldAsStrings(field)));
+ else
+ docCopy.addField(target,(Reader[])fieldData);
+ }
+ else if (fieldData instanceof String[])
+ {
+ String[] processedFieldData;
+ if (filterEmpty)
+ processedFieldData = removeEmpties((String[])fieldData);
+ else
+ processedFieldData = (String[])fieldData;
+ docCopy.addField(target,processedFieldData);
+ }
+ }
+
+ protected static String[] removeEmpties(String[] input)
+ {
+ int count = 0;
+ for (String s : input)
+ {
+ if (s != null && s.length() > 0)
+ count++;
+ }
+ if (count == input.length)
+ return input;
+
+ String[] rval = new String[count];
+ count = 0;
+ for (String s : input)
+ {
+ if (s != null && s.length() > 0)
+ rval[count++] = s;
+ }
+ return rval;
+
+ }
+
+ protected static Object[] conditionallyRemoveNulls(Object[] input, boolean filterEmpty)
+ {
+ if (!filterEmpty)
+ return input;
+ int count = 0;
+ for (Object o : input)
+ {
+ if (o != null)
+ count++;
+ }
+ if (count == input.length)
+ return input;
+
+ Object[] rval = new Object[count];
+ count = 0;
+ for (Object o : input)
+ {
+ if (o != null)
+ rval[count++] = o;
+ }
+ return rval;
+ }
+
// UI support methods.
//
// These support methods come in two varieties. The first bunch (inherited from IConnector) is involved in setting up connection configuration information.
@@ -323,18 +383,52 @@ public class ForcedMetadataConnector ext
node.setAttribute(ATTRIBUTE_TARGET,target);
os.addChild(os.getChildCount(),node);
}
-
- // Gather the keep all metadata parameter to be the last one
- SpecificationNode node = new SpecificationNode(NODE_KEEPMETADATA);
+ }
+
+ x = variableContext.getParameter(seqPrefix+"keepallmetadata_present");
+ if (x != null && x.length() > 0)
+ {
String keepAll = variableContext.getParameter(seqPrefix+"keepallmetadata");
- if (keepAll != null)
+ if (keepAll == null)
+ keepAll = "false";
+ // About to gather the fieldmapping nodes, so get rid of the old ones.
+ int i = 0;
+ while (i < os.getChildCount())
{
- node.setAttribute(ATTRIBUTE_VALUE, keepAll);
+ SpecificationNode node = os.getChild(i);
+ if (node.getType().equals(NODE_KEEPMETADATA))
+ os.removeChild(i);
+ else
+ i++;
}
- else
+
+ // Gather the keep all metadata parameter to be the last one
+ SpecificationNode node = new SpecificationNode(NODE_KEEPMETADATA);
+ node.setAttribute(ATTRIBUTE_VALUE, keepAll);
+ // Add the new keepallmetadata config parameter
+ os.addChild(os.getChildCount(), node);
+ }
+
+ x = variableContext.getParameter(seqPrefix+"filterempty_present");
+ if (x != null && x.length() > 0)
+ {
+ String filterEmpty = variableContext.getParameter(seqPrefix+"filterempty");
+ if (filterEmpty == null)
+ filterEmpty = "false";
+ // About to gather the fieldmapping nodes, so get rid of the old ones.
+ int i = 0;
+ while (i < os.getChildCount())
{
- node.setAttribute(ATTRIBUTE_VALUE, "false");
+ SpecificationNode node = os.getChild(i);
+ if (node.getType().equals(NODE_KEEPMETADATA))
+ os.removeChild(i);
+ else
+ i++;
}
+
+ // Gather the keep all metadata parameter to be the last one
+ SpecificationNode node = new SpecificationNode(NODE_FILTEREMPTY);
+ node.setAttribute(ATTRIBUTE_VALUE, filterEmpty);
// Add the new keepallmetadata config parameter
os.addChild(os.getChildCount(), node);
}
@@ -371,6 +465,7 @@ public class ForcedMetadataConnector ext
// Prep for field mappings
List<Map<String,String>> fieldMappings = new ArrayList<Map<String,String>>();
String keepAllMetadataValue = "true";
+ String filterEmptyValue = "true";
for (int i = 0; i < os.getChildCount(); i++)
{
SpecificationNode sn = os.getChild(i);
@@ -395,9 +490,14 @@ public class ForcedMetadataConnector ext
{
keepAllMetadataValue = sn.getAttributeValue(ATTRIBUTE_VALUE);
}
+ else if (sn.getType().equals(NODE_FILTEREMPTY))
+ {
+ filterEmptyValue = sn.getAttributeValue(ATTRIBUTE_VALUE);
+ }
}
paramMap.put("FIELDMAPPINGS",fieldMappings);
paramMap.put("KEEPALLMETADATA",keepAllMetadataValue);
+ paramMap.put("FILTEREMPTY",filterEmptyValue);
}
protected static void fillInForcedMetadataTab(Map<String,Object> paramMap, Specification os)
@@ -460,16 +560,21 @@ public class ForcedMetadataConnector ext
private final Map<String,String> sourceTargets = new HashMap<String,String>();
private final boolean keepAllMetadata;
+ private final boolean filterEmpty;
private final Map<String,Set<String>> parameters = new HashMap<String,Set<String>>();
public SpecPacker(Specification os) {
boolean keepAllMetadata = true;
+ boolean filterEmpty = true;
for (int i = 0; i < os.getChildCount(); i++) {
SpecificationNode sn = os.getChild(i);
if(sn.getType().equals(NODE_KEEPMETADATA)) {
String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
keepAllMetadata = Boolean.parseBoolean(value);
+ } else if (sn.getType().equals(NODE_FILTEREMPTY)) {
+ String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
+ filterEmpty = Boolean.parseBoolean(value);
} else if (sn.getType().equals(NODE_FIELDMAP)) {
String source = sn.getAttributeValue(ATTRIBUTE_SOURCE);
String target = sn.getAttributeValue(ATTRIBUTE_TARGET);
@@ -493,6 +598,7 @@ public class ForcedMetadataConnector ext
}
}
this.keepAllMetadata = keepAllMetadata;
+ this.filterEmpty = filterEmpty;
}
public String toPackedString() {
@@ -549,6 +655,12 @@ public class ForcedMetadataConnector ext
packList(sb,valueArray,'+');
}
+ // Filter empty
+ if (filterEmpty)
+ sb.append('+');
+ else
+ sb.append('-');
+
return sb.toString();
}
@@ -560,6 +672,10 @@ public class ForcedMetadataConnector ext
return keepAllMetadata;
}
+ public boolean filterEmpty() {
+ return filterEmpty;
+ }
+
public Iterator<String> getParameterKeys()
{
return parameters.keySet().iterator();
Modified: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties?rev=1630812&r1=1630811&r2=1630812&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties (original)
+++ manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties Fri Oct 10 12:49:17 2014
@@ -30,6 +30,7 @@ ForcedMetadata.MetadataFieldName=Incomin
ForcedMetadata.FinalFieldName=Final metadata name
ForcedMetadata.NoFieldMappingSpecified=No metadata mapping specified
ForcedMetadata.KeepAllMetadata=Keep all incoming metadata
+ForcedMetadata.FilterEmpty=Remove empty metadata values
ForcedMetadata.Add=Add
ForcedMetadata.AddFieldMapping=Add metadata mapping
ForcedMetadata.Delete=Delete
Modified: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties?rev=1630812&r1=1630811&r2=1630812&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties (original)
+++ manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties Fri Oct 10 12:49:17 2014
@@ -30,6 +30,7 @@ ForcedMetadata.MetadataFieldName=�
ForcedMetadata.FinalFieldName=æå¾ã®ãã£ã¼ã«ãå
ForcedMetadata.NoFieldMappingSpecified=ãã£ã¼ã«ããããã³ã°ãæå®ãã¦ãã ãã
ForcedMetadata.KeepAllMetadata=å
¨ã¡ã¿ãã¼ã¿ãä¿å
+ForcedMetadata.FilterEmpty=Remove empty metadata values
ForcedMetadata.Add=追å
ForcedMetadata.AddFieldMapping=ãã£ã¼ã«ããããã³ã°ã追å
ForcedMetadata.Delete=åé¤
Modified: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties?rev=1630812&r1=1630811&r2=1630812&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties (original)
+++ manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties Fri Oct 10 12:49:17 2014
@@ -30,6 +30,7 @@ ForcedMetadata.MetadataFieldName=å
�
ForcedMetadata.FinalFieldName=æç»å段å
ForcedMetadata.NoFieldMappingSpecified=æªæå®å段æ å°
ForcedMetadata.KeepAllMetadata=ä¿æææå
æ°æ®
+ForcedMetadata.FilterEmpty=Remove empty metadata values
ForcedMetadata.Add=æ·»å
ForcedMetadata.AddFieldMapping=æ·»å å段æ å°
ForcedMetadata.Delete=å é¤
Modified: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_FieldMapping.html
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_FieldMapping.html?rev=1630812&r1=1630811&r2=1630812&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_FieldMapping.html (original)
+++ manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_FieldMapping.html Fri Oct 10 12:49:17 2014
@@ -84,6 +84,7 @@
<tr>
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.KeepAllMetadata'))</nobr></td>
<td class="value">
+ <input type="hidden" name="s${SEQNUM}_keepallmetadata_present" value="true"/>
#if($KEEPALLMETADATA == 'true')
<input type="checkbox" checked="true" name="s${SEQNUM}_keepallmetadata" value="true"/>
#else
@@ -91,6 +92,19 @@
#end
</td>
</tr>
+
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.FilterEmpty'))</nobr></td>
+ <td class="value">
+ <input type="hidden" name="s${SEQNUM}_filterempty_present" value="true"/>
+ #if($FILTEREMPTY == 'true')
+ <input type="checkbox" checked="true" name="s${SEQNUM}_filterempty" value="true"/>
+ #else
+ <input type="checkbox" name="s${SEQNUM}_filterempty" value="true"/>
+ #end
+ </td>
+ </tr>
+
</table>
#else
@@ -102,6 +116,9 @@
#set($fieldcounter = $fieldcounter + 1)
#end
<input type="hidden" name="s${SEQNUM}_fieldmapping_count" value="$fieldcounter"/>
-<input type="hidden" name="s${SEQNUM}_keepallmetadata" value="$Encoder.bodyEscape($KEEPALLMETADATA)"/>
+<input type="hidden" name="s${SEQNUM}_keepallmetadata_present" value="true"/>
+<input type="hidden" name="s${SEQNUM}_keepallmetadata" value="$Encoder.bodyEscape($FILTEREMPTY)"/>
+<input type="hidden" name="s${SEQNUM}_filterempty_present" value="true"/>
+<input type="hidden" name="s${SEQNUM}_filterempty" value="$Encoder.bodyEscape($FILTEREMPTY)"/>
#end
\ No newline at end of file
Modified: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html?rev=1630812&r1=1630811&r2=1630812&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html (original)
+++ manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html Fri Oct 10 12:49:17 2014
@@ -51,6 +51,10 @@
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.KeepAllMetadata'))</nobr></td>
<td class="value"><nobr>$Encoder.bodyEscape($KEEPALLMETADATA)</nobr></td>
</tr>
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.FilterEmpty'))</nobr></td>
+ <td class="value"><nobr>$Encoder.bodyEscape($FILTEREMPTY)</nobr></td>
+ </tr>
<tr>
<td class="separator" colspan="4"><hr/></td>