You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2015/01/05 19:40:19 UTC
svn commit: r1649608 - in /manifoldcf/branches/dev_1x: ./
connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/
connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agen...
Author: kwright
Date: Mon Jan 5 18:40:18 2015
New Revision: 1649608
URL: http://svn.apache.org/r1649608
Log:
Pull up fix for CONNECTORS-1134 from trunk.
Added:
manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_Expressions.html
- copied unchanged from r1649605, manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_Expressions.html
manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/test/
- copied from r1649605, manifoldcf/trunk/connectors/forcedmetadata/connector/src/test/
Removed:
manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_FieldMapping.html
manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification_ForcedMetadata.html
Modified:
manifoldcf/branches/dev_1x/ (props changed)
manifoldcf/branches/dev_1x/CHANGES.txt
manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties
manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties
manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties
manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification.js
manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html
manifoldcf/branches/dev_1x/framework/ (props changed)
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/RepositoryDocumentFactory.java
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jan 5 18:40:18 2015
@@ -46,6 +46,7 @@
/manifoldcf/branches/CONNECTORS-1118:1644108-1644398
/manifoldcf/branches/CONNECTORS-1119:1645497-1647584
/manifoldcf/branches/CONNECTORS-1130:1648217-1648685
+/manifoldcf/branches/CONNECTORS-1134:1649449-1649604
/manifoldcf/branches/CONNECTORS-120:1406712-1407974,1407982-1411043,1411049-1416451
/manifoldcf/branches/CONNECTORS-120-1:1416450-1417056
/manifoldcf/branches/CONNECTORS-13:1525862-1527182,1539324-1541634
@@ -120,4 +121,4 @@
/manifoldcf/branches/CONNECTORS-981:1605049-1605773
/manifoldcf/branches/CONNECTORS-989:1611600-1612101
/manifoldcf/branches/CONNECTORS-990:1610284-1610707
-/manifoldcf/trunk:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1631750,1631953,1632013,1632225,1632289,1632562,1632844,1632847,1632854,1633062-1633063,1633108,1633193,1633202,1633282,1633284,1633295,1633336,1633339,1633345,1633348,1633364,1633378,1633383,1633432,1633546,1633590,1633634,1633668,1633727,1633760,1633764,1633786,1633910,1633923,1634021,1634028,1634067,1634132,1634145,1634148,163
4155,1634188,1634202,1634264,1634373,1634530,1634688,1634850,1634857,1635103,1635116,1635421,1635438,1635478,1635481,1635484,1635490,1635809,1635939,1636146,1636167,1636180,1636207,1636215,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1637310,1637350,1637364,1637373,1637378,1639259,1639593,1639600,1640018,1640101,1640199,1640204,1640208,1640314,1640319,1640749,1640772,1640805,1640888,1640925,1640941-1640942,1641222,1641328,1641557,1641559,1641629,1641633,1641724,1641754,1641911,1642163,1642255,1642318,1642531,1642650,1642658,1642673,1642716,1644197,1644399,1644538,1644920,1644931,1646317,1646397,1646403,1646408,1646640,1646947,1647574,1647585,1647608,1648686,1648976,1649201,1649203,1649529
+/manifoldcf/trunk:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1631750,1631953,1632013,1632225,1632289,1632562,1632844,1632847,1632854,1633062-1633063,1633108,1633193,1633202,1633282,1633284,1633295,1633336,1633339,1633345,1633348,1633364,1633378,1633383,1633432,1633546,1633590,1633634,1633668,1633727,1633760,1633764,1633786,1633910,1633923,1634021,1634028,1634067,1634132,1634145,1634148,163
4155,1634188,1634202,1634264,1634373,1634530,1634688,1634850,1634857,1635103,1635116,1635421,1635438,1635478,1635481,1635484,1635490,1635809,1635939,1636146,1636167,1636180,1636207,1636215,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1637310,1637350,1637364,1637373,1637378,1639259,1639593,1639600,1640018,1640101,1640199,1640204,1640208,1640314,1640319,1640749,1640772,1640805,1640888,1640925,1640941-1640942,1641222,1641328,1641557,1641559,1641629,1641633,1641724,1641754,1641911,1642163,1642255,1642318,1642531,1642650,1642658,1642673,1642716,1644197,1644399,1644538,1644920,1644931,1646317,1646397,1646403,1646408,1646640,1646947,1647574,1647585,1647608,1648686,1648976,1649201,1649203,1649529,1649605
Modified: manifoldcf/branches/dev_1x/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/CHANGES.txt?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/CHANGES.txt (original)
+++ manifoldcf/branches/dev_1x/CHANGES.txt Mon Jan 5 18:40:18 2015
@@ -3,6 +3,12 @@ $Id$
======================= 1.9-dev =====================
+CONNECTORS-1134: Revamp metadata adjuster transformer to
+handle arbitrary combination expressions across multiple fields.
+This replaces the "move" and "add" paradigm with a more general
+approach.
+(Hemant Jain, Karl Wright)
+
CONNECTORS-1135: Update Japanese translations for JCIFS connector.
(KOIZUMI Satoru)
Modified: manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java (original)
+++ manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java Mon Jan 5 18:40:18 2015
@@ -34,6 +34,7 @@ public class ForcedMetadataConnector ext
// Nodes and attributes representing parameters and values.
// There will be node for every parameter/value pair.
+ public static final String NODE_EXPRESSION = "expression";
public static final String NODE_PAIR = "pair";
public static final String ATTRIBUTE_PARAMETER = "parameter";
public static final String NODE_FIELDMAP = "fieldmap";
@@ -47,8 +48,7 @@ public class ForcedMetadataConnector ext
private static final String VIEW_SPEC = "viewSpecification.html";
private static final String EDIT_SPEC_HEADER = "editSpecification.js";
- private static final String EDIT_SPEC_FORCED_METADATA = "editSpecification_ForcedMetadata.html";
- private static final String EDIT_SPEC_FIELDMAPPING = "editSpecification_FieldMapping.html";
+ private static final String EDIT_SPEC_EXPRESSIONS = "editSpecification_Expressions.html";
/** Get a pipeline version string, given a pipeline specification object. The version string is used to
* uniquely describe the pertinent details of the specification and the configuration, to allow the Connector
@@ -91,41 +91,135 @@ public class ForcedMetadataConnector ext
{
// Unpack the forced metadata
SpecPacker sp = new SpecPacker(pipelineDescription.getSpecification());
- // We have to create a copy of the Repository Document, since we might be rearranging things
- RepositoryDocument docCopy = document.duplicate();
- docCopy.clearFields();
- // Do the mapping first!!
- Iterator<String> fields = document.getFields();
- while (fields.hasNext())
- {
- String field = fields.next();
- String target = sp.getMapping(field);
- if (target != null)
- {
- moveData(docCopy,target,document,field,sp.filterEmpty());
- }
- else
- {
- if (sp.keepAllMetadata())
+
+ // Create a structure that will allow us access to fields without sharing Reader objects
+ FieldDataFactory fdf = new FieldDataFactory(document);
+ try {
+ // We have to create a copy of the Repository Document, since we might be rearranging things
+ RepositoryDocument docCopy = document.duplicate();
+ // We must explicitly copy all fields, since we can't share references to Reader objects and
+ // expect anything to work right
+ docCopy.clearFields();
+
+ // Clear fields, unless we're supposed to keep what we don't specify
+ if (sp.filterEmpty()) {
+ if (sp.keepAllMetadata()) {
+ // Loop through fields and copy them, filtering empties
+ Iterator<String> fields = document.getFields();
+ while (fields.hasNext())
+ {
+ String field = fields.next();
+ moveData(docCopy,field,fdf,field,true);
+ }
+ }
+ } else if (sp.keepAllMetadata()) {
+ // Copy ALL current fields from old document, but go through FieldDataFactory
+ Iterator<String> fields = document.getFields();
+ while (fields.hasNext())
{
- moveData(docCopy,field,document,field,sp.filterEmpty());
+ String field = fields.next();
+ moveData(docCopy,field,fdf,field,false);
}
}
+
+ // Iterate through the expressions
+ Iterator<String> expressionKeys = sp.getExpressionKeys();
+ while (expressionKeys.hasNext()) {
+ String expressionKey = expressionKeys.next();
+ // Get the set of expressions for the key
+ Set<String> values = sp.getExpressionValues(expressionKey);
+ IDataSource[] dataSources = new IDataSource[values.size()];
+ int k = 0;
+ for (String expression : values) {
+ dataSources[k++] = processExpression(expression, fdf);
+ }
+ int totalSize = 0;
+ for (IDataSource dataSource : dataSources) {
+ if (dataSource != null)
+ totalSize += dataSource.getSize();
+ }
+ if (totalSize == 0) {
+ docCopy.removeField(expressionKey);
+ } else {
+ // Each IDataSource will contribute zero or more results to the final array. But here's the tricky part:
+ // the results all must be of the same type. If there are any differences, then we have to bash them all to
+ // strings first.
+ Object[] allValues;
+ k = 0;
+ if (allDates(dataSources)) {
+ allValues = new Date[totalSize];
+ for (IDataSource dataSource : dataSources) {
+ if (dataSource != null) {
+ for (Object o : dataSource.getRawForm()) {
+ allValues[k++] = o;
+ }
+ }
+ }
+ docCopy.addField(expressionKey,(Date[])conditionallyRemoveNulls(allValues,sp.filterEmpty()));
+ } else if (allReaders(dataSources)) {
+ if (sp.filterEmpty())
+ allValues = new String[totalSize];
+ else
+ allValues = new Reader[totalSize];
+ for (IDataSource dataSource : dataSources) {
+ if (dataSource != null) {
+ Object[] sources = sp.filterEmpty()?dataSource.getStringForm():dataSource.getRawForm();
+ for (Object o : sources) {
+ allValues[k++] = o;
+ }
+ }
+ }
+ if (sp.filterEmpty())
+ docCopy.addField(expressionKey,removeEmpties((String[])allValues));
+ else
+ docCopy.addField(expressionKey,(Reader[])allValues);
+ } else {
+ allValues = new String[totalSize];
+ // Convert to strings throughout
+ for (IDataSource dataSource : dataSources) {
+ if (dataSource != null) {
+ for (Object o : dataSource.getStringForm()) {
+ allValues[k++] = o;
+ }
+ }
+ }
+ if (sp.filterEmpty())
+ docCopy.addField(expressionKey,removeEmpties((String[])allValues));
+ else
+ docCopy.addField(expressionKey,(String[])allValues);
+ }
+ }
+ }
+
+ // Finally, send the modified repository document onward to the next pipeline stage.
+ // If we'd done anything to the stream, we'd have needed to create a new RepositoryDocument object and copied the
+ // data into it, and closed the new stream after sendDocument() was called.
+ return activities.sendDocument(documentURI,docCopy);
+
+ } finally {
+ fdf.close();
}
+ }
- Iterator<String> keys = sp.getParameterKeys();
- while (keys.hasNext())
- {
- String key = keys.next();
- docCopy.addField(key,sp.getParameterValues(key));
+ protected static boolean allDates(IDataSource[] dataSources)
+ throws IOException, ManifoldCFException {
+ for (IDataSource ds : dataSources) {
+ if (ds != null && !(ds.getRawForm() instanceof Date[]))
+ return false;
}
- // Finally, send the modified repository document onward to the next pipeline stage.
- // If we'd done anything to the stream, we'd have needed to create a new RepositoryDocument object and copied the
- // data into it, and closed the new stream after sendDocument() was called.
- return activities.sendDocument(documentURI,docCopy);
+ return true;
}
- protected static void moveData(RepositoryDocument docCopy, String target, RepositoryDocument document, String field, boolean filterEmpty)
+ protected static boolean allReaders(IDataSource[] dataSources)
+ throws IOException, ManifoldCFException {
+ for (IDataSource ds : dataSources) {
+ if (ds != null && !(ds.getRawForm() instanceof Reader[]))
+ return false;
+ }
+ return true;
+ }
+
+ protected static void moveData(RepositoryDocument docCopy, String target, FieldDataFactory document, String field, boolean filterEmpty)
throws ManifoldCFException, IOException
{
Object[] fieldData = document.getField(field);
@@ -240,9 +334,7 @@ public class ForcedMetadataConnector ext
{
// Output specification header
- // Add Forced Metadata to tab array
- tabsArray.add(Messages.getString(locale, "ForcedMetadata.FieldMappingTabName"));
- tabsArray.add(Messages.getString(locale, "ForcedMetadata.ForcedMetadata"));
+ tabsArray.add(Messages.getString(locale, "ForcedMetadata.Expressions"));
Map<String, Object> paramMap = new HashMap<String, Object>();
paramMap.put("SEQNUM",Integer.toString(connectionSequenceNumber));
@@ -272,11 +364,9 @@ public class ForcedMetadataConnector ext
paramMap.put("SEQNUM",Integer.toString(connectionSequenceNumber));
paramMap.put("SELECTEDNUM",Integer.toString(actualSequenceNumber));
- fillInForcedMetadataTab(paramMap, os);
- fillInFieldMappingSpecificationMap(paramMap, os);
+ fillInExpressionsTab(paramMap, os);
- Messages.outputResourceWithVelocity(out,locale,EDIT_SPEC_FORCED_METADATA,paramMap);
- Messages.outputResourceWithVelocity(out,locale,EDIT_SPEC_FIELDMAPPING,paramMap);
+ Messages.outputResourceWithVelocity(out,locale,EDIT_SPEC_EXPRESSIONS,paramMap);
}
/** Process a specification post.
@@ -296,96 +386,53 @@ public class ForcedMetadataConnector ext
{
// Process specification post
String seqPrefix = "s"+connectionSequenceNumber+"_";
- String forcedCount = variableContext.getParameter(seqPrefix+"forcedmetadata_count");
- if (forcedCount != null)
+ String expressionCount = variableContext.getParameter(seqPrefix+"expression_count");
+ if (expressionCount != null)
{
- int count = Integer.parseInt(forcedCount);
- // Delete old spec data
+ int count = Integer.parseInt(expressionCount);
+ // Delete old spec data, including legacy node types we no longer use
int i = 0;
while (i < os.getChildCount())
{
SpecificationNode cn = os.getChild(i);
- if (cn.getType().equals(NODE_PAIR))
+ if (cn.getType().equals(NODE_EXPRESSION) || cn.getType().equals(NODE_PAIR) || cn.getType().equals(NODE_FIELDMAP))
os.removeChild(i);
else
i++;
}
+
// Now, go through form data
for (int j = 0; j < count; j++)
{
- String op = variableContext.getParameter(seqPrefix+"forcedmetadata_"+j+"_op");
+ String op = variableContext.getParameter(seqPrefix+"expression_"+j+"_op");
if (op != null && op.equals("Delete"))
continue;
- String paramName = variableContext.getParameter(seqPrefix+"forcedmetadata_"+j+"_name");
- String paramValue = variableContext.getParameter(seqPrefix+"forcedmetadata_"+j+"_value");
- SpecificationNode sn = new SpecificationNode(NODE_PAIR);
+ String paramName = variableContext.getParameter(seqPrefix+"expression_"+j+"_name");
+ String paramRemove = variableContext.getParameter(seqPrefix+"expression_"+j+"_remove");
+ String paramValue = variableContext.getParameter(seqPrefix+"expression_"+j+"_value");
+ SpecificationNode sn = new SpecificationNode(NODE_EXPRESSION);
sn.setAttribute(ATTRIBUTE_PARAMETER,paramName);
- sn.setAttribute(ATTRIBUTE_VALUE,paramValue);
+ if (!(paramRemove != null && paramRemove.equals("true")))
+ sn.setAttribute(ATTRIBUTE_VALUE,paramValue);
os.addChild(os.getChildCount(),sn);
}
// Look for add operation
- String addOp = variableContext.getParameter(seqPrefix+"forcedmetadata_op");
+ String addOp = variableContext.getParameter(seqPrefix+"expression_op");
if (addOp != null && addOp.equals("Add"))
{
- String paramName = variableContext.getParameter(seqPrefix+"forcedmetadata_name");
- String paramValue = variableContext.getParameter(seqPrefix+"forcedmetadata_value");
- SpecificationNode sn = new SpecificationNode(NODE_PAIR);
+ String paramName = variableContext.getParameter(seqPrefix+"expression_name");
+ String paramRemove = variableContext.getParameter(seqPrefix+"expression_remove");
+ String paramValue = variableContext.getParameter(seqPrefix+"expression_value");
+ SpecificationNode sn = new SpecificationNode(NODE_EXPRESSION);
sn.setAttribute(ATTRIBUTE_PARAMETER,paramName);
- sn.setAttribute(ATTRIBUTE_VALUE,paramValue);
+ if (!(paramRemove != null && paramRemove.equals("true")))
+ sn.setAttribute(ATTRIBUTE_VALUE,paramValue);
os.addChild(os.getChildCount(),sn);
}
+
}
- String x = variableContext.getParameter(seqPrefix+"fieldmapping_count");
- if (x != null && x.length() > 0)
- {
- // About to gather the fieldmapping nodes, so get rid of the old ones.
- int i = 0;
- while (i < os.getChildCount())
- {
- SpecificationNode node = os.getChild(i);
- if (node.getType().equals(NODE_FIELDMAP) || node.getType().equals(NODE_KEEPMETADATA))
- os.removeChild(i);
- else
- i++;
- }
- int count = Integer.parseInt(x);
- i = 0;
- while (i < count)
- {
- String prefix = seqPrefix+"fieldmapping_";
- String suffix = "_"+Integer.toString(i);
- String op = variableContext.getParameter(prefix+"op"+suffix);
- if (op == null || !op.equals("Delete"))
- {
- // Gather the fieldmap etc.
- String source = variableContext.getParameter(prefix+"source"+suffix);
- String target = variableContext.getParameter(prefix+"target"+suffix);
- if (target == null)
- target = "";
- SpecificationNode node = new SpecificationNode(NODE_FIELDMAP);
- node.setAttribute(ATTRIBUTE_SOURCE,source);
- node.setAttribute(ATTRIBUTE_TARGET,target);
- os.addChild(os.getChildCount(),node);
- }
- i++;
- }
-
- String addop = variableContext.getParameter(seqPrefix+"fieldmapping_op");
- if (addop != null && addop.equals("Add"))
- {
- String source = variableContext.getParameter(seqPrefix+"fieldmapping_source");
- String target = variableContext.getParameter(seqPrefix+"fieldmapping_target");
- if (target == null)
- target = "";
- SpecificationNode node = new SpecificationNode(NODE_FIELDMAP);
- node.setAttribute(ATTRIBUTE_SOURCE,source);
- node.setAttribute(ATTRIBUTE_TARGET,target);
- os.addChild(os.getChildCount(),node);
- }
- }
-
- x = variableContext.getParameter(seqPrefix+"keepallmetadata_present");
+ String x = variableContext.getParameter(seqPrefix+"keepallmetadata_present");
if (x != null && x.length() > 0)
{
String keepAll = variableContext.getParameter(seqPrefix+"keepallmetadata");
@@ -454,16 +501,17 @@ public class ForcedMetadataConnector ext
paramMap.put("SEQNUM",Integer.toString(connectionSequenceNumber));
// Fill in the map with data from all tabs
- fillInForcedMetadataTab(paramMap, os);
- fillInFieldMappingSpecificationMap(paramMap, os);
+ fillInExpressionsTab(paramMap, os);
Messages.outputResourceWithVelocity(out,locale,VIEW_SPEC,paramMap);
}
- protected static void fillInFieldMappingSpecificationMap(Map<String,Object> paramMap, Specification os)
+ protected static void fillInExpressionsTab(Map<String,Object> paramMap, Specification os)
{
- // Prep for field mappings
- List<Map<String,String>> fieldMappings = new ArrayList<Map<String,String>>();
+ final Map<String,Set<String>> expressions = new HashMap<String,Set<String>>();
+ final Map<String,Set<String>> expressionAdditions = new HashMap<String,Set<String>>();
+ final Map<String,Set<String>> additions = new HashMap<String,Set<String>>();
+
String keepAllMetadataValue = "true";
String filterEmptyValue = "false";
for (int i = 0; i < os.getChildCount(); i++)
@@ -473,18 +521,44 @@ public class ForcedMetadataConnector ext
String source = sn.getAttributeValue(ATTRIBUTE_SOURCE);
String target = sn.getAttributeValue(ATTRIBUTE_TARGET);
String targetDisplay;
- if (target == null)
+
+ expressions.put(source,new HashSet<String>());
+ if (target != null)
{
- target = "";
- targetDisplay = "(remove)";
+ Set<String> sources = new HashSet<String>();
+ sources.add("${"+source+"}");
+ expressions.put(target,sources);
+ }
+ }
+ else if (sn.getType().equals(NODE_PAIR))
+ {
+ String parameter = sn.getAttributeValue(ATTRIBUTE_PARAMETER);
+ String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
+ // Since the same target is completely superceded by a NODE_PAIR, but NODE_PAIRs
+ // are cumulative, I have to build these completely and then post-process them.
+ Set<String> addition = additions.get(parameter);
+ if (addition == null)
+ {
+ addition = new HashSet<String>();
+ additions.put(parameter,addition);
+ }
+ addition.add(expressionEscape(value));
+ }
+ else if (sn.getType().equals(NODE_EXPRESSION))
+ {
+ String parameter = sn.getAttributeValue(ATTRIBUTE_PARAMETER);
+ String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
+ if (value == null) {
+ expressionAdditions.put(parameter,new HashSet<String>());
+ } else {
+ Set<String> expressionAddition = expressionAdditions.get(parameter);
+ if (expressionAddition == null)
+ {
+ expressionAddition = new HashSet<String>();
+ expressionAdditions.put(parameter,expressionAddition);
+ }
+ expressionAddition.add(value);
}
- else
- targetDisplay = target;
- Map<String,String> fieldMapping = new HashMap<String,String>();
- fieldMapping.put("SOURCE",source);
- fieldMapping.put("TARGET",target);
- fieldMapping.put("TARGETDISPLAY",targetDisplay);
- fieldMappings.add(fieldMapping);
}
else if (sn.getType().equals(NODE_KEEPMETADATA))
{
@@ -495,77 +569,193 @@ public class ForcedMetadataConnector ext
filterEmptyValue = sn.getAttributeValue(ATTRIBUTE_VALUE);
}
}
- paramMap.put("FIELDMAPPINGS",fieldMappings);
- paramMap.put("KEEPALLMETADATA",keepAllMetadataValue);
- paramMap.put("FILTEREMPTY",filterEmptyValue);
- }
-
- protected static void fillInForcedMetadataTab(Map<String,Object> paramMap, Specification os)
- {
- // First, sort everything
- Map<String,Set<String>> params = new HashMap<String,Set<String>>();
- for (int i = 0; i < os.getChildCount(); i++)
+
+ // Postprocessing.
+ // Override the moves with the additions
+ for (String parameter : additions.keySet())
{
- SpecificationNode sn = os.getChild(i);
- if (sn.getType().equals(NODE_PAIR))
- {
- String parameter = sn.getAttributeValue(ATTRIBUTE_PARAMETER);
- String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
- Set<String> values = params.get(parameter);
- if (values == null)
- {
- values = new HashSet<String>();
- params.put(parameter,values);
- }
- values.add(value);
- }
+ expressions.put(parameter,additions.get(parameter));
}
-
- // We construct a list of maps from the parameters
- List<Map<String,String>> pObject = new ArrayList<Map<String,String>>();
- String[] keys = new String[params.size()];
- int j = 0;
- for (String key : params.keySet())
+ // Override all with expression additions
+ for (String parameter : expressionAdditions.keySet())
{
- keys[j++] = key;
+ expressions.put(parameter,expressionAdditions.get(parameter));
}
+
+ // Problem: how to display case where we want a null source??
+ // A: Special value
+ List<Map<String,String>> pObject = new ArrayList<Map<String,String>>();
+ String[] keys = expressions.keySet().toArray(new String[0]);
java.util.Arrays.sort(keys);
// Now, build map
for (String key : keys)
{
- Set<String> values = params.get(key);
- String[] valueArray = new String[values.size()];
- j = 0;
- for (String value : values)
- {
- valueArray[j++] = value;
- }
- java.util.Arrays.sort(valueArray);
-
- for (String value : valueArray)
- {
+ Set<String> values = expressions.get(key);
+ if (values.size() == 0) {
Map<String,String> record = new HashMap<String,String>();
record.put("parameter",key);
- record.put("value",value);
+ record.put("value","");
+ record.put("isnull","true");
pObject.add(record);
+ } else {
+ String[] valueArray = values.toArray(new String[0]);
+ java.util.Arrays.sort(valueArray);
+
+ for (String value : valueArray)
+ {
+ Map<String,String> record = new HashMap<String,String>();
+ record.put("parameter",key);
+ record.put("value",value);
+ record.put("isnull","false");
+ pObject.add(record);
+ }
}
}
- paramMap.put("PARAMETERS",pObject);
+ paramMap.put("EXPRESSIONS",pObject);
+ paramMap.put("KEEPALLMETADATA",keepAllMetadataValue);
+ paramMap.put("FILTEREMPTY",filterEmptyValue);
+ }
+
+ protected static String expressionEscape(String input) {
+ // Not doing any escaping yet
+ return input;
}
+ protected static String expressionUnescape(String input) {
+ // Not doing any escaping yet
+ return input;
+ }
+
+ public interface IDataSource {
+ public int getSize() throws IOException, ManifoldCFException;
+ public Object[] getRawForm() throws IOException, ManifoldCFException;
+ public String[] getStringForm() throws IOException, ManifoldCFException;
+ }
+
+ protected static class StringSource implements IDataSource {
+ protected final String[] data;
+
+ public StringSource(String data) {
+ this.data = new String[]{data};
+ }
+
+ public StringSource(String[] data) {
+ this.data = data;
+ }
+
+ @Override
+ public int getSize() {
+ return data.length;
+ }
+
+ @Override
+ public Object[] getRawForm() {
+ return data;
+ }
+
+ @Override
+ public String[] getStringForm() {
+ return data;
+ }
+ }
+
+ protected static class FieldSource implements IDataSource {
+
+ protected final FieldDataFactory rd;
+ protected final String fieldName;
+
+ public FieldSource(FieldDataFactory rd, String fieldName) {
+ this.rd = rd;
+ this.fieldName = fieldName;
+ }
+
+ @Override
+ public int getSize()
+ throws IOException, ManifoldCFException {
+ Object[] rawForm = getRawForm();
+ if (rawForm == null)
+ return 0;
+ return rawForm.length;
+ }
+
+ @Override
+ public Object[] getRawForm()
+ throws IOException, ManifoldCFException {
+ return rd.getField(fieldName);
+ }
+
+ @Override
+ public String[] getStringForm()
+ throws IOException, ManifoldCFException {
+ return rd.getFieldAsStrings(fieldName);
+ }
+ }
+
+ protected static IDataSource append(IDataSource currentValues, IDataSource data)
+ throws IOException, ManifoldCFException {
+ // currentValues and data can either be:
+ // Date[], String[], or Reader[].
+ // We want to preserve the type in as high a form as possible when we compute the combinations.
+ if (currentValues == null)
+ return data;
+ if (currentValues.getSize() == 0)
+ return currentValues;
+ // Any combination causes conversion to a string, so if we get here, we can read the inputs all
+ // as strings safely.
+ String[] currentStrings = currentValues.getStringForm();
+ String[] dataStrings = data.getStringForm();
+ String[] rval = new String[currentStrings.length * dataStrings.length];
+ int rvalIndex = 0;
+ for (String currentString : currentStrings) {
+ for (String dataString : dataStrings) {
+ rval[rvalIndex++] = currentString + dataString;
+ }
+ }
+ return new StringSource(rval);
+ }
+
+ public static IDataSource processExpression(String expression, FieldDataFactory sourceDocument)
+ throws IOException, ManifoldCFException {
+ int index = 0;
+ IDataSource input = null;
+ while (true) {
+ // If we're at the end, return the input
+ if (index == expression.length())
+ return input;
+ // Look for next field specification
+ int field = expression.indexOf("${",index);
+ if (field == -1)
+ return append(input, new StringSource(expressionUnescape(expression.substring(index))));
+ if (field > 0)
+ input = append(input, new StringSource(expressionUnescape(expression.substring(index,field))));
+ // Get the field name
+ int fieldEnd = expression.indexOf("}",field);
+ String fieldName;
+ if (fieldEnd == -1) {
+ fieldName = expression.substring(field+2);
+ return append(input, new FieldSource(sourceDocument, fieldName));
+ } else {
+ fieldName = expression.substring(field+2,fieldEnd);
+ input = append(input, new FieldSource(sourceDocument, fieldName));
+ index = fieldEnd+1;
+ }
+ }
+ }
+
protected static class SpecPacker {
- private final Map<String,String> sourceTargets = new HashMap<String,String>();
private final boolean keepAllMetadata;
private final boolean filterEmpty;
- private final Map<String,Set<String>> parameters = new HashMap<String,Set<String>>();
+ private final Map<String,Set<String>> expressions = new HashMap<String,Set<String>>();
public SpecPacker(Specification os) {
boolean keepAllMetadata = true;
boolean filterEmpty = false;
+ final Map<String,Set<String>> additions = new HashMap<String,Set<String>>();
+ final Map<String,Set<String>> expressionAdditions = new HashMap<String,Set<String>>();
for (int i = 0; i < os.getChildCount(); i++) {
SpecificationNode sn = os.getChild(i);
@@ -579,24 +769,58 @@ public class ForcedMetadataConnector ext
String source = sn.getAttributeValue(ATTRIBUTE_SOURCE);
String target = sn.getAttributeValue(ATTRIBUTE_TARGET);
- if (target == null) {
- target = "";
+ expressions.put(source,new HashSet<String>());
+ // Null target means to remove the *source* from the document.
+ if (target != null) {
+ Set<String> sources = new HashSet<String>();
+ sources.add("${"+source+"}");
+ expressions.put(target,sources);
}
- sourceTargets.put(source, target);
}
else if (sn.getType().equals(NODE_PAIR))
{
String parameter = sn.getAttributeValue(ATTRIBUTE_PARAMETER);
String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
- Set<String> params = parameters.get(parameter);
- if (params == null)
+ // Since the same target is completely superceded by a NODE_PAIR, but NODE_PAIRs
+ // are cumulative, I have to build these completely and then post-process them.
+ Set<String> addition = additions.get(parameter);
+ if (addition == null)
{
- params = new HashSet<String>();
- parameters.put(parameter,params);
+ addition = new HashSet<String>();
+ additions.put(parameter,addition);
}
- params.add(value);
+ addition.add(expressionEscape(value));
}
+ else if (sn.getType().equals(NODE_EXPRESSION))
+ {
+ String parameter = sn.getAttributeValue(ATTRIBUTE_PARAMETER);
+ String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
+ if (value == null) {
+ expressionAdditions.put(parameter,new HashSet<String>());
+ } else {
+ Set<String> expressionAddition = expressionAdditions.get(parameter);
+ if (expressionAddition == null)
+ {
+ expressionAddition = new HashSet<String>();
+ expressionAdditions.put(parameter,expressionAddition);
+ }
+ expressionAddition.add(value);
+ }
+ }
+ }
+
+ // Override the moves with the additions
+ for (String parameter : additions.keySet())
+ {
+ expressions.put(parameter,additions.get(parameter));
+ }
+
+ // Override all with expression additions
+ for (String parameter : expressionAdditions.keySet())
+ {
+ expressions.put(parameter,expressionAdditions.get(parameter));
}
+
this.keepAllMetadata = keepAllMetadata;
this.filterEmpty = filterEmpty;
}
@@ -605,25 +829,16 @@ public class ForcedMetadataConnector ext
StringBuilder sb = new StringBuilder();
int i;
- // Mappings
- final String[] sortArray = new String[sourceTargets.size()];
- i = 0;
- for (String source : sourceTargets.keySet()) {
- sortArray[i++] = source;
- }
+ final String[] sortArray = expressions.keySet().toArray(new String[0]);
java.util.Arrays.sort(sortArray);
-
- List<String> packedMappings = new ArrayList<String>();
- String[] fixedList = new String[2];
- for (String source : sortArray) {
- String target = sourceTargets.get(source);
- StringBuilder localBuffer = new StringBuilder();
- fixedList[0] = source;
- fixedList[1] = target;
- packFixedList(localBuffer,fixedList,':');
- packedMappings.add(localBuffer.toString());
+ // Pack the list of keys
+ packList(sb,sortArray,'+');
+ for (String key : sortArray) {
+ Set<String> values = expressions.get(key);
+ String[] valueArray = values.toArray(new String[0]);
+ java.util.Arrays.sort(valueArray);
+ packList(sb,valueArray,'+');
}
- packList(sb,packedMappings,'+');
// Keep all metadata
if (keepAllMetadata)
@@ -631,30 +846,6 @@ public class ForcedMetadataConnector ext
else
sb.append('-');
- // Get the keys and sort them
- final String[] keys = new String[parameters.size()];
- int j = 0;
- for (String key : parameters.keySet())
- {
- keys[j++] = key;
- }
- java.util.Arrays.sort(keys);
- // Pack the list of keys
- packList(sb,keys,'+');
- // Now, go through each key and individually pack the values
- for (String key : keys)
- {
- Set<String> values = parameters.get(key);
- String[] valueArray = new String[values.size()];
- j = 0;
- for (String value : values)
- {
- valueArray[j++] = value;
- }
- java.util.Arrays.sort(valueArray);
- packList(sb,valueArray,'+');
- }
-
// Filter empty
if (filterEmpty)
sb.append('+');
@@ -664,8 +855,14 @@ public class ForcedMetadataConnector ext
return sb.toString();
}
- public String getMapping(String source) {
- return sourceTargets.get(source);
+ public Iterator<String> getExpressionKeys()
+ {
+ return expressions.keySet().iterator();
+ }
+
+ public Set<String> getExpressionValues(String key)
+ {
+ return expressions.get(key);
}
public boolean keepAllMetadata() {
@@ -675,27 +872,119 @@ public class ForcedMetadataConnector ext
public boolean filterEmpty() {
return filterEmpty;
}
+ }
+
+ /** This class provides unique Reader and other field instances, when requested, based
+ * on an input RepositoryDocument. It does this by pulling the values of the field into
+ * a CharacterInput implementation, thus making a temporary file copy. So it is imperative
+ * that this object is closed when it is no longer needed.
+ */
+ public static class FieldDataFactory {
- public Iterator<String> getParameterKeys()
- {
- return parameters.keySet().iterator();
+ protected final RepositoryDocument sourceDocument;
+
+ // Readers (organized by metadata)
+ protected final Map<String,CharacterInput[]> metadataReaders = new HashMap<String,CharacterInput[]>();
+
+ public FieldDataFactory(RepositoryDocument sourceDocument) {
+ this.sourceDocument = sourceDocument;
}
- public String[] getParameterValues(String key)
+ public void close()
+ throws ManifoldCFException
{
- Set<String> values = parameters.get(key);
- if (values == null)
- return null;
- String[] rval = new String[values.size()];
- int i = 0;
- for (String value : values)
+ for (String key : metadataReaders.keySet())
+ {
+ CharacterInput[] rt = metadataReaders.get(key);
+ for (CharacterInput r : rt)
+ {
+ r.discard();
+ }
+ }
+ }
+
+ public Object[] getField(String fieldName)
+ throws IOException, ManifoldCFException {
+ CharacterInput[] inputs = metadataReaders.get(fieldName);
+ if (inputs == null) {
+ // Either never seen the field before, or it's not a Reader
+ Object[] fieldValues = sourceDocument.getField(fieldName);
+ if (fieldValues == null)
+ return fieldValues;
+ if (fieldValues instanceof Reader[]) {
+ // Create a copy
+ CharacterInput[] newValues = new CharacterInput[fieldValues.length];
+ try {
+ // Populate newValues
+ for (int i = 0; i < newValues.length; i++)
+ {
+ newValues[i] = new TempFileCharacterInput((Reader)fieldValues[i]);
+ }
+ metadataReaders.put(fieldName,newValues);
+ inputs = newValues;
+ } catch (Throwable e) {
+ for (CharacterInput r : newValues)
+ {
+ if (r != null)
+ r.discard();
+ }
+ if (e instanceof IOException)
+ throw (IOException)e;
+ else if (e instanceof RuntimeException)
+ throw (RuntimeException)e;
+ else if (e instanceof Error)
+ throw (Error)e;
+ else
+ throw new RuntimeException("Unknown exception type: "+e.getClass().getName()+": "+e.getMessage(),e);
+ }
+ } else {
+ return fieldValues;
+ }
+ }
+
+ Reader[] newReaders = new Reader[inputs.length];
+ for (int i = 0; i < inputs.length; i++)
{
- rval[i++] = value;
+ inputs[i].doneWithStream();
+ newReaders[i] = inputs[i].getStream();
+ }
+ return newReaders;
+ }
+
+ public String[] getFieldAsStrings(String fieldName)
+ throws IOException, ManifoldCFException {
+ CharacterInput[] cilist = metadataReaders.get(fieldName);
+ if (cilist == null)
+ return sourceDocument.getFieldAsStrings(fieldName);
+
+ // We've created a local array of CharacterInputs from this field. We'll need to convert these
+ // to strings.
+ char[] buffer = new char[65536];
+ String[] rval = new String[cilist.length];
+ for (int i = 0; i < rval.length; i++) {
+ CharacterInput ci = cilist[i];
+ ci.doneWithStream();
+ Reader r = ci.getStream();
+ // Read into a buffer
+ StringBuilder newValue = new StringBuilder();
+ while (true)
+ {
+ int amt = r.read(buffer);
+ if (amt == -1)
+ break;
+ newValue.append(buffer,0,amt);
+ }
+ rval[i] = newValue.toString();
+ }
+ sourceDocument.addField(fieldName,rval);
+ metadataReaders.remove(fieldName);
+ for (CharacterInput ci : cilist) {
+ ci.discard();
}
return rval;
}
}
-
+
}
Modified: manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties (original)
+++ manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_en_US.properties Mon Jan 5 18:40:18 2015
@@ -13,26 +13,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-ForcedMetadata.ForcedMetadata=Add metadata
-ForcedMetadata.FieldMappingTabName=Move metadata
-
-ForcedMetadata.ForcedMetadataNameMustNotBeNull=Added metadata name must not be null
-ForcedMetadata.ForcedMetadataColon=Added metadata:
+ForcedMetadata.Expressions=Metadata expressions
+ForcedMetadata.RemoveQ=Remove this parameter?
+ForcedMetadata.ParameterValueExpression=Expression ("${fieldname}" references a field)
+ForcedMetadata.MetadataExpressionsColon=Metadata expressions:
+ForcedMetadata.NoExpressionsSpecified=No expressions provided
+ForcedMetadata.Deleteexpressionnumber=Delete expression #
+ForcedMetadata.Addexpression=Add expression
+ForcedMetadata.True=true
+ForcedMetadata.False=false
+ForcedMetadata.ExpressionMetadataNameMustNotBeNull=Expression metadata name must not be null
ForcedMetadata.ParameterName=Parameter name
-ForcedMetadata.ParameterValue=Parameter value
ForcedMetadata.Delete=Delete
-ForcedMetadata.Deleteforcedmetadatanumber=Delete added metadata #
ForcedMetadata.Add=Add
-ForcedMetadata.Addforcedmetadata=Add metadata item
-ForcedMetadata.NoForcedMetadataSpecified=No added metadata specified
-ForcedMetadata.FieldMappings=Metadata mappings:
-ForcedMetadata.MetadataFieldName=Incoming metadata name
-ForcedMetadata.FinalFieldName=Final metadata name
-ForcedMetadata.NoFieldMappingSpecified=No metadata mapping specified
ForcedMetadata.KeepAllMetadata=Keep all incoming metadata
ForcedMetadata.FilterEmpty=Remove empty metadata values
-ForcedMetadata.Add=Add
-ForcedMetadata.AddFieldMapping=Add metadata mapping
-ForcedMetadata.Delete=Delete
-ForcedMetadata.DeleteFieldMapping=Delete metadata mapping #
ForcedMetadata.NoFieldNameSpecified=Please specify a metadata name
Modified: manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties (original)
+++ manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_ja_JP.properties Mon Jan 5 18:40:18 2015
@@ -13,26 +13,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-ForcedMetadata.ForcedMetadata=å¼·å¶ã¡ã¿ãã¼ã¿
-ForcedMetadata.FieldMappingTabName=ãã£ã¼ã«ããããã³ã°
-
-ForcedMetadata.ForcedMetadataNameMustNotBeNull=å¼·å¶ã¡ã¿ãã¼ã¿åãå¿
ãå
¥åãã¦ãã ãã
-ForcedMetadata.ForcedMetadataColon=å¼·å¶ã¡ã¿ãã¼ã¿:
+ForcedMetadata.Expressions=Metadata expressions
+ForcedMetadata.RemoveQ=Remove this parameter?
+ForcedMetadata.ParameterValueExpression=Expression ("${fieldname}" references a field)
+ForcedMetadata.MetadataExpressionsColon=Metadata expressions:
+ForcedMetadata.NoExpressionsSpecified=No expressions provided
+ForcedMetadata.Deleteexpressionnumber=Delete expression #
+ForcedMetadata.Addexpression=Add expression
+ForcedMetadata.True=true
+ForcedMetadata.False=false
+ForcedMetadata.ExpressionMetadataNameMustNotBeNull=Expression metadata name must not be null
ForcedMetadata.ParameterName=ãã©ã¡ã¼ã¿å
-ForcedMetadata.ParameterValue=ãã©ã¡ã¼ã¿å¤
ForcedMetadata.Delete=åé¤
-ForcedMetadata.Deleteforcedmetadatanumber=å¼·å¶ã¡ã¿ãã¼ã¿ãåé¤ #
ForcedMetadata.Add=追å
-ForcedMetadata.Addforcedmetadata=å¼·å¶ã¡ã¿ãã¼ã¿ã追å ãã¦ãã ãã
-ForcedMetadata.NoForcedMetadataSpecified=å¼·å¶ã¡ã¿ãã¼ã¿ãæå®ãã¦ãã ãã
-ForcedMetadata.FieldMappings=ãã£ã¼ã«ããããã³ã°:
-ForcedMetadata.MetadataFieldName=ã¡ã¿ãã¼ã¿ãã£ã¼ã«ãå
-ForcedMetadata.FinalFieldName=æå¾ã®ãã£ã¼ã«ãå
-ForcedMetadata.NoFieldMappingSpecified=ãã£ã¼ã«ããããã³ã°ãæå®ãã¦ãã ãã
ForcedMetadata.KeepAllMetadata=å
¨ã¡ã¿ãã¼ã¿ãä¿å
ForcedMetadata.FilterEmpty=Remove empty metadata values
-ForcedMetadata.Add=追å
-ForcedMetadata.AddFieldMapping=ãã£ã¼ã«ããããã³ã°ã追å
-ForcedMetadata.Delete=åé¤
-ForcedMetadata.DeleteFieldMapping=ãã£ã¼ã«ããããã³ã°ãåé¤ #
ForcedMetadata.NoFieldNameSpecified=ãã£ã¼ã«ãåãå
¥åãã¦ãã ãã
Modified: manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties (original)
+++ manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/forcedmetadata/common_zh_CN.properties Mon Jan 5 18:40:18 2015
@@ -13,26 +13,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-ForcedMetadata.ForcedMetadata=强å¶æ§å
æ°æ®
-ForcedMetadata.FieldMappingTabName=å段æ å°
-
-ForcedMetadata.ForcedMetadataNameMustNotBeNull=强å¶æ§å
æ°æ®åä¸è½ä¸ºç©º
-ForcedMetadata.ForcedMetadataColon=强å¶æ§å
æ°æ®:
+ForcedMetadata.Expressions=Metadata expressions
+ForcedMetadata.RemoveQ=Remove this parameter?
+ForcedMetadata.ParameterValueExpression=Expression ("${fieldname}" references a field)
+ForcedMetadata.MetadataExpressionsColon=Metadata expressions:
+ForcedMetadata.NoExpressionsSpecified=No expressions provided
+ForcedMetadata.Deleteexpressionnumber=Delete expression #
+ForcedMetadata.Addexpression=Add expression
+ForcedMetadata.True=true
+ForcedMetadata.False=false
+ForcedMetadata.ExpressionMetadataNameMustNotBeNull=Expression metadata name must not be null
ForcedMetadata.ParameterName=åæ°å
-ForcedMetadata.ParameterValue=åæ°å¼
ForcedMetadata.Delete=å é¤
-ForcedMetadata.Deleteforcedmetadatanumber=å é¤å¼ºå¶æ§å
æ°æ® #
ForcedMetadata.Add=æ·»å
-ForcedMetadata.Addforcedmetadata=æ·»å 强å¶æ§å
æ°æ®
-ForcedMetadata.NoForcedMetadataSpecified=æªæå®å¼ºå¶æ§å
æ°æ®
-ForcedMetadata.FieldMappings=å段æ å°:
-ForcedMetadata.MetadataFieldName=å
æ°æ®å段å
-ForcedMetadata.FinalFieldName=æç»å段å
-ForcedMetadata.NoFieldMappingSpecified=æªæå®å段æ å°
ForcedMetadata.KeepAllMetadata=ä¿æææå
æ°æ®
ForcedMetadata.FilterEmpty=Remove empty metadata values
-ForcedMetadata.Add=æ·»å
-ForcedMetadata.AddFieldMapping=æ·»å å段æ å°
-ForcedMetadata.Delete=å é¤
-ForcedMetadata.DeleteFieldMapping=å é¤å段æ å° #
ForcedMetadata.NoFieldNameSpecified=请æå®å段å
Modified: manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification.js
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification.js?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification.js (original)
+++ manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/editSpecification.js Mon Jan 5 18:40:18 2015
@@ -18,6 +18,27 @@
<script type="text/javascript">
<!--
+function s${SEQNUM}_AddExpression()
+{
+ if (editjob.s${SEQNUM}_expression_name.value == "")
+ {
+ alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('ForcedMetadata.ExpressionMetadataNameMustNotBeNull'))");
+ editjob.s${SEQNUM}_expression_name.focus();
+ return;
+ }
+ document.editjob.s${SEQNUM}_expression_op.value="Add";
+ postFormSetAnchor("s${SEQNUM}_expression_tag");
+}
+
+function s${SEQNUM}_DeleteExpression(n)
+{
+ eval("document.editjob.s${SEQNUM}_expression_"+n+"_op.value = 'Delete'");
+ if (n == 0)
+ postFormSetAnchor("s${SEQNUM}_expression_tag");
+ else
+ postFormSetAnchor("s${SEQNUM}_expression_"+(n-1)+"_tag");
+}
+
function s${SEQNUM}_AddForcedMetadata()
{
if (editjob.s${SEQNUM}_forcedmetadata_name.value == "")
Modified: manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html (original)
+++ manifoldcf/branches/dev_1x/connectors/forcedmetadata/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/forcedmetadata/viewSpecification.html Mon Jan 5 18:40:18 2015
@@ -17,58 +17,16 @@
<table class="displaytable">
<tr>
- <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.FieldMappings'))</nobr></td>
- <td class="boxcell">
- <table class="formtable">
- <tr class="formheaderrow">
- <td class="formcolumnheader"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.MetadataFieldName'))</nobr></td>
- <td class="formcolumnheader"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.FinalFieldName'))</nobr></td>
- </tr>
-#set($fieldcounter = 0)
-#foreach($fieldmapping in $FIELDMAPPINGS)
- #if(($fieldcounter % 2) == 0)
- <tr class="evenformrow">
- #else
- <tr class="oddformrow">
- #end
- <td class="formcolumncell">
- <nobr>$Encoder.bodyEscape($fieldmapping.get('SOURCE'))</nobr>
- </td>
- <td class="formcolumncell">
- <nobr>$Encoder.bodyEscape($fieldmapping.get('TARGETDISPLAY'))</nobr>
- </td>
- </tr>
- #set($fieldcounter = $fieldcounter + 1)
-#end
-#if($fieldcounter == 0)
- <tr class="formrow"><td class="formmessage" colspan="2">$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.NoFieldMappingSpecified'))</td></tr>
-#end
- </table>
- </td>
- </tr>
- <tr><td class="separator" colspan="2"><hr/></td></tr>
- <tr>
- <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.KeepAllMetadata'))</nobr></td>
- <td class="value"><nobr>$Encoder.bodyEscape($KEEPALLMETADATA)</nobr></td>
- </tr>
- <tr>
- <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.FilterEmpty'))</nobr></td>
- <td class="value"><nobr>$Encoder.bodyEscape($FILTEREMPTY)</nobr></td>
- </tr>
-
- <tr>
- <td class="separator" colspan="4"><hr/></td>
- </tr>
- <tr>
- <td class="description" colspan="1"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.ForcedMetadataColon'))</nobr></td>
+ <td class="description" colspan="1"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.MetadataExpressionsColon'))</nobr></td>
<td class="boxcell" colspan="3">
<table class="formtable">
<tr class="formheaderrow">
<td class="formcolumnheader"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.ParameterName'))</nobr></td>
- <td class="formcolumnheader"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.ParameterValue'))</nobr></td>
+ <td class="formcolumnheader"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.RemoveQ'))</nobr></td>
+ <td class="formcolumnheader"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.ParameterValueExpression'))</nobr></td>
</tr>
#set($paramcounter = 0)
- #foreach($paramrecord in $PARAMETERS)
+ #foreach($paramrecord in $EXPRESSIONS)
#if(($paramcounter % 2) == 0)
<tr class="evenformrow">
#else
@@ -78,15 +36,34 @@
<nobr>$Encoder.bodyEscape($paramrecord.get('parameter'))</nobr>
</td>
<td class="formcolumncell">
+ <nobr>
+ #if($paramrecord.get('isnull') == 'true')
+ $Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.True'))
+ #end
+ #if($paramrecord.get('isnull') == 'false')
+ $Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.False'))
+ #end
+ </nobr>
+ </td>
+ <td class="formcolumncell">
<nobr>$Encoder.bodyEscape($paramrecord.get('value'))</nobr>
</td>
</tr>
#set($paramcounter = $paramcounter + 1)
#end
#if($paramcounter == 0)
- <tr class="formrow"><td colspan="2" class="formcolumnmessage"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.NoForcedMetadataSpecified'))</nobr></td></tr>
+ <tr class="formrow"><td colspan="4" class="formcolumnmessage"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.NoExpressionsSpecified'))</nobr></td></tr>
#end
</table>
</td>
</tr>
+ <tr><td class="separator" colspan="4"><hr/></td></tr>
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.KeepAllMetadata'))</nobr></td>
+ <td class="value"><nobr>$Encoder.bodyEscape($KEEPALLMETADATA)</nobr></td>
+ </tr>
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ForcedMetadata.FilterEmpty'))</nobr></td>
+ <td class="value"><nobr>$Encoder.bodyEscape($FILTEREMPTY)</nobr></td>
+ </tr>
</table>
Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jan 5 18:40:18 2015
@@ -113,4 +113,4 @@
/manifoldcf/branches/CONNECTORS-989/framework:1611600-1612101
/manifoldcf/branches/CONNECTORS-990/framework:1610284-1610707
/manifoldcf/trunk:1629122
-/manifoldcf/trunk/framework:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1632013,1632289,1632844,1633108,1633193,1633202,1633348,1633364,1634145,1634148,1634155,1634264,1634373,1634530,1635438,1635809,1636146,1636180,1636207,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1639593,1639600,1640018,1640101,1640199,1640314,1640319,1640749,1640772,1640925,1640941,1641222,1641557,1641559,1
641724,1641911,1642163,1642255,1642318,1644197,1644399,1646317,1646397,1646403,1646640,1647574,1647585,1647608
+/manifoldcf/trunk/framework:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1632013,1632289,1632844,1633108,1633193,1633202,1633348,1633364,1634145,1634148,1634155,1634264,1634373,1634530,1635438,1635809,1636146,1636180,1636207,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1639593,1639600,1640018,1640101,1640199,1640314,1640319,1640749,1640772,1640925,1640941,1641222,1641557,1641559,1
641724,1641911,1642163,1642255,1642318,1644197,1644399,1646317,1646397,1646403,1646640,1647574,1647585,1647608,1649605
Modified: manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/RepositoryDocumentFactory.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/RepositoryDocumentFactory.java?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/RepositoryDocumentFactory.java (original)
+++ manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/RepositoryDocumentFactory.java Mon Jan 5 18:40:18 2015
@@ -43,7 +43,7 @@ public class RepositoryDocumentFactory
protected BinaryInput binaryTracker;
// Readers (organized by metadata)
- protected Map<String,CharacterInput[]> metadataReaders = new HashMap<String,CharacterInput[]>();
+ protected final Map<String,CharacterInput[]> metadataReaders = new HashMap<String,CharacterInput[]>();
/** Constructor.
* Pass a RepositoryDocument. This constructor reads all streams and stores them in
Modified: manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java?rev=1649608&r1=1649607&r2=1649608&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java (original)
+++ manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java Mon Jan 5 18:40:18 2015
@@ -415,6 +415,17 @@ public class RepositoryDocument
return binaryLength;
}
+ /** Remove a field.
+ *@param fieldName is the field name.
+ */
+ public void removeField(String fieldName)
+ {
+ fields.remove(fieldName);
+ stringFields.remove(fieldName);
+ readerFields.remove(fieldName);
+ dateFields.remove(fieldName);
+ }
+
/** Add/remove a multivalue date field.
*@param fieldName is the field name.
*@param fieldData is the multi-valued data (an array of Dates). Null means
@@ -577,6 +588,12 @@ public class RepositoryDocument
newValues[i] = newValue.toString();
}
stringFields.put(fieldName,newValues);
+ // Reader is no longer useful, since we've read it to the end.
+ // Remove it from the record accordingly.
+ // NOTE WELL: This could cause side effects if the same
+ // field is accessed simultaneously two different ways!
+ readerFields.remove(fieldName);
+ fields.put(fieldName,newValues);
return newValues;
}
else