You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2019/02/18 10:21:09 UTC
[lucene-solr] 01/01: SOLR-7414: Adding explicitly required fields
support to CSV and XLSX response writers
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch SOLR-7414
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 6911c86c0217a50be266fad25de188d132f07127
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Mon Feb 18 15:50:44 2019 +0530
SOLR-7414: Adding explicitly required fields support to CSV and XLSX response writers
---
.../handler/extraction/XLSXResponseWriter.java | 27 +++++++++++++++------
.../apache/solr/response/CSVResponseWriter.java | 22 ++++++++++++-----
.../java/org/apache/solr/search/ReturnFields.java | 7 ++++++
.../org/apache/solr/search/SolrReturnFields.java | 28 ++++++++++++++--------
.../solr/response/TestCSVResponseWriter.java | 22 ++++++++++++++++-
5 files changed, 82 insertions(+), 24 deletions(-)
diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
index baa1ddb..96b34e2 100644
--- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
+++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
@@ -30,6 +30,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
@@ -178,13 +179,18 @@ class XLSXWriter extends TextResponseWriter {
this.rsp = rsp;
}
+ // TODO: a lot of the code here is copied from CSVResponseWriter. We should look at
+ // refactoring into a common place.
+ // nocommit: Added support for explicitly requested fields (SOLR-7414) similar to how it
+ // was done at CSVResponseWriter. However, this hasn't been tested here.
public void writeResponse(OutputStream out, LinkedHashMap<String, String> colNamesMap,
LinkedHashMap<String, Integer> colWidthsMap) throws IOException {
SolrParams params = req.getParams();
Collection<String> fields = returnFields.getRequestedFieldNames();
+ Set<String> explicitReqFields = returnFields.getExplicitlyRequestedFieldNames();
Object responseObj = rsp.getValues().get("response");
- boolean returnOnlyStored = false;
+ boolean returnStoredOrDocValStored = false;
if (fields==null||returnFields.hasPatternMatching()) {
if (responseObj instanceof SolrDocumentList) {
// get the list of fields from the SolrDocumentList
@@ -203,12 +209,17 @@ class XLSXWriter extends TextResponseWriter {
Iterables.addAll(fields, all);
}
}
+
+ if (explicitReqFields != null) {
+ // add explicit requested fields
+ Iterables.addAll(fields, explicitReqFields);
+ }
if (returnFields.wantsScore()) {
fields.add("score");
} else {
fields.remove("score");
}
- returnOnlyStored = true;
+ returnStoredOrDocValStored = true;
}
for (String field : fields) {
@@ -223,16 +234,18 @@ class XLSXWriter extends TextResponseWriter {
}
SchemaField sf = schema.getFieldOrNull(field);
+ // Return stored fields or useDocValuesAsStored=true fields,
+ // unless an explicit field list is specified
+ if (returnStoredOrDocValStored && !(explicitReqFields != null && explicitReqFields.contains(field)) &&
+ sf!= null && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored())) {
+ continue;
+ }
+
if (sf == null) {
FieldType ft = new StrField();
sf = new SchemaField(field, ft);
}
- // Return only stored fields, unless an explicit field list is specified
- if (returnOnlyStored && sf != null && !sf.stored()) {
- continue;
- }
-
XLField xlField = new XLField();
xlField.name = field;
xlField.sf = sf;
diff --git a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java
index 5ebec77..aad4069 100644
--- a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java
@@ -28,6 +28,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
@@ -246,6 +247,7 @@ class CSVWriter extends TextResponseWriter {
}
Collection<String> fields = returnFields.getRequestedFieldNames();
+ Set<String> explicitReqFields = returnFields.getExplicitlyRequestedFieldNames();
Object responseObj = rsp.getResponse();
boolean returnStoredOrDocValStored = false;
if (fields==null||returnFields.hasPatternMatching()) {
@@ -266,6 +268,12 @@ class CSVWriter extends TextResponseWriter {
Iterables.addAll(fields, all);
}
}
+
+ if (explicitReqFields != null) {
+ // add explicit requested fields
+ Iterables.addAll(fields, explicitReqFields);
+ }
+
if (returnFields.wantsScore()) {
fields.add("score");
} else {
@@ -288,17 +296,19 @@ class CSVWriter extends TextResponseWriter {
}
SchemaField sf = schema.getFieldOrNull(field);
- if (sf == null) {
- FieldType ft = new StrField();
- sf = new SchemaField(field, ft);
- }
-
+
// Return stored fields or useDocValuesAsStored=true fields,
// unless an explicit field list is specified
- if (returnStoredOrDocValStored && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored())) {
+ if (returnStoredOrDocValStored && !(explicitReqFields != null && explicitReqFields.contains(field)) &&
+ sf!= null && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored())) {
continue;
}
+ if (sf == null) {
+ FieldType ft = new StrField();
+ sf = new SchemaField(field, ft);
+ }
+
// check for per-field overrides
sep = params.get("f." + field + '.' + CSV_SEPARATOR);
encapsulator = params.get("f." + field + '.' + CSV_ENCAPSULATOR);
diff --git a/solr/core/src/java/org/apache/solr/search/ReturnFields.java b/solr/core/src/java/org/apache/solr/search/ReturnFields.java
index c2d5973..0c35ea5 100644
--- a/solr/core/src/java/org/apache/solr/search/ReturnFields.java
+++ b/solr/core/src/java/org/apache/solr/search/ReturnFields.java
@@ -60,6 +60,13 @@ public abstract class ReturnFields {
public abstract Set<String> getRequestedFieldNames();
/**
+ * The explicitly requested field names (includes pseudo fields)
+ * <p>
+ * @return Set of explicitly requested field names or <code>null</code> (no explicit)
+ */
+ public abstract Set<String> getExplicitlyRequestedFieldNames();
+
+ /**
* Get the fields which have been renamed
* @return a mapping of renamed fields
*/
diff --git a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
index d9e681b..9a04c25 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
@@ -16,6 +16,15 @@
*/
package org.apache.solr.search;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+
import org.apache.commons.io.FilenameUtils;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.ValueSource;
@@ -35,15 +44,6 @@ import org.apache.solr.response.transform.TransformerFactory;
import org.apache.solr.response.transform.ValueSourceAugmenter;
import org.apache.solr.search.SolrDocumentFetcher.RetrieveFieldsOptimizer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Supplier;
-
/**
* The default implementation of return fields parsing for Solr.
*/
@@ -448,7 +448,15 @@ public class SolrReturnFields extends ReturnFields {
@Override
public Set<String> getRequestedFieldNames() {
- if(_wantsAllFields || reqFieldNames==null || reqFieldNames.isEmpty()) {
+ if(_wantsAllFields || reqFieldNames == null || reqFieldNames.isEmpty()) {
+ return null;
+ }
+ return reqFieldNames;
+ }
+
+ @Override
+ public Set<String> getExplicitlyRequestedFieldNames() {
+ if (reqFieldNames == null || reqFieldNames.isEmpty()) {
return null;
}
return reqFieldNames;
diff --git a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
index 979279c..204496a 100644
--- a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
+++ b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
@@ -42,7 +42,7 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
public static void createIndex() {
assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false", "foo_f","1.414","foo_d","-1.0E300","foo_dt","2000-01-02T03:04:05Z"));
assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output", "shouldbeunstored","foo"));
- assertU(adoc("id","3", "shouldbeunstored","foo"));
+ assertU(adoc("id","3", "shouldbeunstored","foo", "foo_l", "1"));
assertU(adoc("id","4", "amount_c", "1.50,EUR"));
assertU(adoc("id","5", "store", "12.434,-134.1"));
assertU(adoc("id","6", "pubyear_ii", "123", "store_iis", "12", "price_ff", "1.3"));
@@ -246,6 +246,13 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
assertEquals("exists(shouldbeunstored),XXX", singleFuncLines[0] );
assertEquals("false,1", singleFuncLines[1] );
assertEquals("true,3", singleFuncLines[3] );
+
+ // pseudo-fields with * in fl
+ txt = h.query(req("q","id:4", "wt","csv", "csv.header","true", "fl","*,YYY:[docid],FOO:amount_c"));
+ lines = txt.split("\n");
+ assertEquals(2, lines.length);
+ assertEquals(sortHeader("foo_i,foo_l,FOO,foo_s,store,store_iis," +
+ "v2_ss,pubyear_ii,foo_dt,foo_b,YYY,foo_d,id,amount_c,foo_f,v_ss"), sortHeader(lines[0]));
}
@Test
@@ -283,6 +290,19 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
assertEquals(2, singleFuncLines.length);
assertEquals("price_ff", singleFuncLines[0]);
assertEquals("1.3", singleFuncLines[1]);
+
+ // explicit price_ff with fl=*
+ singleFuncText = h.query(req("q","id:6", "wt","csv", "csv.header","true", "fl", "*,price_ff"));
+ sortedHeader = sortHeader("amount_c,store,v_ss,foo_b,v2_ss,foo_f,foo_i,foo_d,foo_s,foo_dt,id,foo_l," +
+ "pubyear_ii,store_iis,price_ff");
+ singleFuncLines = singleFuncText.split("\n");
+ assertEquals(2, singleFuncLines.length);
+ assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));
+ actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))
+ .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))
+ .collect(Collectors.toList());
+ assertEquals(4, actualVal.size());
+ assertTrue(actualVal.containsAll(Arrays.asList("6", "123", "12", "1.3")));
}