You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by ml...@apache.org on 2013/07/02 21:54:52 UTC
svn commit: r1499078 - in
/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr:
HttpPoster.java SolrConnector.java
Author: mlizewski
Date: Tue Jul 2 19:54:52 2013
New Revision: 1499078
URL: http://svn.apache.org/r1499078
Log:
improvement CONNECTORS-746: allow multiple field mappings in Solr output connector
Modified:
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1499078&r1=1499077&r2=1499078&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java Tue Jul 2 19:54:52 2013
@@ -491,7 +491,7 @@ public class HttpPoster
* @throws ManifoldCFException, ServiceInterruption
*/
public boolean indexPost(String documentURI,
- RepositoryDocument document, Map arguments, Map sourceTargets,
+ RepositoryDocument document, Map arguments, Map<String, List<String>> sourceTargets,
String authorityNameString, IOutputAddActivity activities)
throws ManifoldCFException, ServiceInterruption
{
@@ -767,7 +767,7 @@ public class HttpPoster
protected String documentURI;
protected RepositoryDocument document;
protected Map<String,List<String>> arguments;
- protected Map<String,String> sourceTargets;
+ protected Map<String,List<String>> sourceTargets;
protected String[] shareAcls;
protected String[] shareDenyAcls;
protected String[] acls;
@@ -783,7 +783,7 @@ public class HttpPoster
protected boolean rval = false;
public IngestThread(String documentURI, RepositoryDocument document,
- Map<String,List<String>> arguments, Map<String,String> sourceTargets,
+ Map<String,List<String>> arguments, Map<String, List<String>> sourceTargets,
String[] shareAcls, String[] shareDenyAcls, String[] acls, String[] denyAcls, String commitWithin)
{
super();
@@ -871,15 +871,26 @@ public class HttpPoster
while (iter.hasNext())
{
String fieldName = iter.next();
- String newFieldName = sourceTargets.get(fieldName);
- if (newFieldName == null)
- newFieldName = fieldName;
- if (newFieldName.length() > 0)
- {
- if (newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT)))
- newFieldName = ID_METADATA;
- String[] values = document.getFieldAsStrings(fieldName);
- writeField(out,LITERAL+newFieldName,values);
+ List<String> mapping = sourceTargets.get(fieldName);
+ if(mapping != null) {
+ for(String newFieldName : mapping) {
+ if(newFieldName != null && !newFieldName.isEmpty()) {
+ if (newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT))) {
+ newFieldName = ID_METADATA;
+ }
+ String[] values = document.getFieldAsStrings(fieldName);
+ writeField(out,LITERAL+newFieldName,values);
+ }
+ }
+ } else {
+ String newFieldName = fieldName;
+ if (!newFieldName.isEmpty()) {
+ if (newFieldName.toLowerCase(Locale.ROOT).equals(idAttributeName.toLowerCase(Locale.ROOT))) {
+ newFieldName = ID_METADATA;
+ }
+ String[] values = document.getFieldAsStrings(fieldName);
+ writeField(out,LITERAL+newFieldName,values);
+ }
}
}
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1499078&r1=1499077&r2=1499078&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java Tue Jul 2 19:54:52 2013
@@ -463,6 +463,7 @@ public class SolrConnector extends org.a
{
String name = sortArray[i++];
ArrayList values = (ArrayList)args.get(name);
+ java.util.Collections.sort(values);
int j = 0;
while (j < values.size())
{
@@ -477,45 +478,52 @@ public class SolrConnector extends org.a
packList(sb,nameValues,'+');
- Map fieldMap = new HashMap();
+ // Do the source/target pairs
i = 0;
- while (i < spec.getChildCount())
- {
+ Map<String, List<String>> sourceTargets = new HashMap<String, List<String>>();
+ while (i < spec.getChildCount()) {
SpecificationNode sn = spec.getChild(i++);
- if (sn.getType().equals(SolrConfig.NODE_FIELDMAP))
- {
+ if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
- if (target == null)
+ if (target == null) {
target = "";
- fieldMap.put(source,target);
+ }
+ List<String> list = (List<String>)sourceTargets.get(source);
+ if (list == null) {
+ list = new ArrayList<String>();
+ sourceTargets.put(source, list);
+ }
+ list.add(target);
}
}
- sortArray = new String[fieldMap.size()];
+ sortArray = new String[sourceTargets.size()];
+ iter = sourceTargets.keySet().iterator();
i = 0;
- iter = fieldMap.keySet().iterator();
- while (iter.hasNext())
- {
+ while (iter.hasNext()) {
sortArray[i++] = (String)iter.next();
}
java.util.Arrays.sort(sortArray);
- ArrayList sourceTargets = new ArrayList();
-
+ ArrayList sourceTargetsList = new ArrayList();
i = 0;
- while (i < sortArray.length)
- {
+ while (i < sortArray.length) {
String source = sortArray[i++];
- String target = (String)fieldMap.get(source);
- fixedList[0] = source;
- fixedList[1] = target;
- StringBuilder pairBuffer = new StringBuilder();
- packFixedList(pairBuffer,fixedList,'=');
- sourceTargets.add(pairBuffer.toString());
+ List<String> values = (List<String>)sourceTargets.get(source);
+ java.util.Collections.sort(values);
+ int j = 0;
+ while (j < values.size()) {
+ String target = (String)values.get(j++);
+ fixedList[0] = source;
+ fixedList[1] = target;
+ StringBuilder pairBuffer = new StringBuilder();
+ packFixedList(pairBuffer,fixedList,'=');
+ sourceTargetsList.add(pairBuffer.toString());
+ }
}
- packList(sb,sourceTargets,'+');
+ packList(sb,sourceTargetsList,'+');
// Here, append things which we have no intention of unpacking. This includes stuff that comes from
// the configuration information, for instance.
@@ -601,7 +609,7 @@ public class SolrConnector extends org.a
{
// Build the argument map we'll send.
Map args = new HashMap();
- Map sourceTargets = new HashMap();
+ Map<String, List<String>> sourceTargets = new HashMap<String, List<String>>();
int index = 0;
ArrayList nameValues = new ArrayList();
index = unpackList(nameValues,outputDescription,index,'+');
@@ -627,11 +635,17 @@ public class SolrConnector extends org.a
// Do the source/target pairs
i = 0;
- while (i < sts.size())
- {
+ while (i < sts.size()) {
String x = (String)sts.get(i++);
unpackFixedList(fixedBuffer,x,0,'=');
- sourceTargets.put(fixedBuffer[0],fixedBuffer[1]);
+ String source = fixedBuffer[0];
+ String target = fixedBuffer[1];
+ List<String> list = (List<String>)sourceTargets.get(source);
+ if (list == null) {
+ list = new ArrayList<String>();
+ sourceTargets.put(source, list);
+ }
+ list.add(target);
}
// Establish a session
@@ -2238,21 +2252,7 @@ public class SolrConnector extends org.a
public void outputSpecificationBody(IHTTPOutput out, Locale locale, OutputSpecification os, String tabName)
throws ManifoldCFException, IOException
{
- // Prep for field mapping tab
- HashMap fieldMap = new HashMap();
int i = 0;
- while (i < os.getChildCount())
- {
- SpecificationNode sn = os.getChild(i++);
- if (sn.getType().equals(SolrConfig.NODE_FIELDMAP))
- {
- String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
- String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
- if (target != null && target.length() == 0)
- target = null;
- fieldMap.put(source,target);
- }
- }
// Field Mapping tab
if (tabName.equals(Messages.getString(locale,"SolrConnector.SolrFieldMapping")))
@@ -2271,30 +2271,25 @@ public class SolrConnector extends org.a
" </tr>\n"
);
- String[] sourceFieldNames = new String[fieldMap.size()];
- Iterator iter = fieldMap.keySet().iterator();
- i = 0;
- while (iter.hasNext())
- {
- sourceFieldNames[i++] = (String)iter.next();
- }
- java.util.Arrays.sort(sourceFieldNames);
-
int fieldCounter = 0;
i = 0;
- while (i < sourceFieldNames.length)
- {
- String source = sourceFieldNames[i++];
- String target = (String)fieldMap.get(source);
- String targetDisplay = target;
- if (target == null)
- {
- target = "";
- targetDisplay = "(remove)";
- }
- // It's prefix will be...
- String prefix = "solr_fieldmapping_" + Integer.toString(fieldCounter);
- out.print(
+ while (i < os.getChildCount()) {
+ SpecificationNode sn = os.getChild(i++);
+ if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
+ String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
+ String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
+ if (target != null && target.length() == 0) {
+ target = null;
+ }
+ String targetDisplay = target;
+ if (target == null)
+ {
+ target = "";
+ targetDisplay = "(remove)";
+ }
+ // It's prefix will be...
+ String prefix = "solr_fieldmapping_" + Integer.toString(fieldCounter);
+ out.print(
" <tr class=\""+(((fieldCounter % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+
" <td class=\"formcolumncell\">\n"+
" <a name=\""+prefix+"\">\n"+
@@ -2311,8 +2306,9 @@ public class SolrConnector extends org.a
" <nobr>"+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(targetDisplay)+"</nobr>\n"+
" </td>\n"+
" </tr>\n"
- );
- fieldCounter++;
+ );
+ fieldCounter++;
+ }
}
if (fieldCounter == 0)
@@ -2347,25 +2343,27 @@ public class SolrConnector extends org.a
else
{
// Hiddens for field mapping
- out.print(
-"<input type=\"hidden\" name=\"solr_fieldmapping_count\" value=\""+Integer.toString(fieldMap.size())+"\"/>\n"
- );
- Iterator iter = fieldMap.keySet().iterator();
+ i = 0;
int fieldCounter = 0;
- while (iter.hasNext())
- {
- String source = (String)iter.next();
- String target = (String)fieldMap.get(source);
- if (target == null)
- target = "";
+ while (i < os.getChildCount()) {
+ SpecificationNode sn = os.getChild(i++);
+ if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
+ String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
+ String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
+ if (target == null)
+ target = "";
// It's prefix will be...
- String prefix = "solr_fieldmapping_" + Integer.toString(fieldCounter);
- out.print(
+ String prefix = "solr_fieldmapping_" + Integer.toString(fieldCounter);
+ out.print(
"<input type=\"hidden\" name=\""+prefix+"_source\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(source)+"\"/>\n"+
"<input type=\"hidden\" name=\""+prefix+"_target\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(target)+"\"/>\n"
- );
- fieldCounter++;
+ );
+ fieldCounter++;
+ }
}
+ out.print(
+"<input type=\"hidden\" name=\"solr_fieldmapping_count\" value=\""+Integer.toString(fieldCounter)+"\"/>\n"
+ );
}
}
@@ -2444,27 +2442,6 @@ public class SolrConnector extends org.a
// Prep for field mappings
HashMap fieldMap = new HashMap();
int i = 0;
- while (i < os.getChildCount())
- {
- SpecificationNode sn = os.getChild(i++);
- if (sn.getType().equals(SolrConfig.NODE_FIELDMAP))
- {
- String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
- String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
- if (target != null && target.length() == 0)
- target = null;
- fieldMap.put(source,target);
- }
- }
-
- String[] sourceFieldNames = new String[fieldMap.size()];
- Iterator iter = fieldMap.keySet().iterator();
- i = 0;
- while (iter.hasNext())
- {
- sourceFieldNames[i++] = (String)iter.next();
- }
- java.util.Arrays.sort(sourceFieldNames);
// Display field mappings
out.print(
@@ -2481,17 +2458,19 @@ public class SolrConnector extends org.a
);
int fieldCounter = 0;
- while (fieldCounter < sourceFieldNames.length)
- {
- String source = sourceFieldNames[fieldCounter++];
- String target = (String)fieldMap.get(source);
- String targetDisplay = target;
- if (target == null)
- {
- target = "";
- targetDisplay = "(remove)";
- }
- out.print(
+ i = 0;
+ while (i < os.getChildCount()) {
+ SpecificationNode sn = os.getChild(i++);
+ if (sn.getType().equals(SolrConfig.NODE_FIELDMAP)) {
+ String source = sn.getAttributeValue(SolrConfig.ATTRIBUTE_SOURCE);
+ String target = sn.getAttributeValue(SolrConfig.ATTRIBUTE_TARGET);
+ String targetDisplay = target;
+ if (target == null)
+ {
+ target = "";
+ targetDisplay = "(remove)";
+ }
+ out.print(
" <tr class=\""+(((fieldCounter % 2)==0)?"evenformrow":"oddformrow")+"\">\n"+
" <td class=\"formcolumncell\">\n"+
" <nobr>"+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(source)+"</nobr>\n"+
@@ -2500,8 +2479,9 @@ public class SolrConnector extends org.a
" <nobr>"+org.apache.manifoldcf.ui.util.Encoder.bodyEscape(targetDisplay)+"</nobr>\n"+
" </td>\n"+
" </tr>\n"
- );
- fieldCounter++;
+ );
+ fieldCounter++;
+ }
}
if (fieldCounter == 0)