You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2019/02/18 10:21:08 UTC
[lucene-solr] branch SOLR-7414 created (now 6911c86)
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a change to branch SOLR-7414
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
at 6911c86 SOLR-7414: Adding explicitly required fields support to CSV and XLSX response writers
This branch includes the following new commits:
new 6911c86 SOLR-7414: Adding explicitly required fields support to CSV and XLSX response writers
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[lucene-solr] 01/01: SOLR-7414: Adding explicitly required fields
support to CSV and XLSX response writers
Posted by is...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch SOLR-7414
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 6911c86c0217a50be266fad25de188d132f07127
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Mon Feb 18 15:50:44 2019 +0530
SOLR-7414: Adding explicitly required fields support to CSV and XLSX response writers
---
.../handler/extraction/XLSXResponseWriter.java | 27 +++++++++++++++------
.../apache/solr/response/CSVResponseWriter.java | 22 ++++++++++++-----
.../java/org/apache/solr/search/ReturnFields.java | 7 ++++++
.../org/apache/solr/search/SolrReturnFields.java | 28 ++++++++++++++--------
.../solr/response/TestCSVResponseWriter.java | 22 ++++++++++++++++-
5 files changed, 82 insertions(+), 24 deletions(-)
diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
index baa1ddb..96b34e2 100644
--- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
+++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
@@ -30,6 +30,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
@@ -178,13 +179,18 @@ class XLSXWriter extends TextResponseWriter {
this.rsp = rsp;
}
+ // TODO: a lot of the code here is copied from CSVResponseWriter. We should look at
+ // refactoring into a common place.
+ // nocommit: Added support for explicitly requested fields (SOLR-7414) similar to how it
+ // was done at CSVResponseWriter. However, this hasn't been tested here.
public void writeResponse(OutputStream out, LinkedHashMap<String, String> colNamesMap,
LinkedHashMap<String, Integer> colWidthsMap) throws IOException {
SolrParams params = req.getParams();
Collection<String> fields = returnFields.getRequestedFieldNames();
+ Set<String> explicitReqFields = returnFields.getExplicitlyRequestedFieldNames();
Object responseObj = rsp.getValues().get("response");
- boolean returnOnlyStored = false;
+ boolean returnStoredOrDocValStored = false;
if (fields==null||returnFields.hasPatternMatching()) {
if (responseObj instanceof SolrDocumentList) {
// get the list of fields from the SolrDocumentList
@@ -203,12 +209,17 @@ class XLSXWriter extends TextResponseWriter {
Iterables.addAll(fields, all);
}
}
+
+ if (explicitReqFields != null) {
+ // add explicit requested fields
+ Iterables.addAll(fields, explicitReqFields);
+ }
if (returnFields.wantsScore()) {
fields.add("score");
} else {
fields.remove("score");
}
- returnOnlyStored = true;
+ returnStoredOrDocValStored = true;
}
for (String field : fields) {
@@ -223,16 +234,18 @@ class XLSXWriter extends TextResponseWriter {
}
SchemaField sf = schema.getFieldOrNull(field);
+ // Return stored fields or useDocValuesAsStored=true fields,
+ // unless an explicit field list is specified
+ if (returnStoredOrDocValStored && !(explicitReqFields != null && explicitReqFields.contains(field)) &&
+ sf!= null && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored())) {
+ continue;
+ }
+
if (sf == null) {
FieldType ft = new StrField();
sf = new SchemaField(field, ft);
}
- // Return only stored fields, unless an explicit field list is specified
- if (returnOnlyStored && sf != null && !sf.stored()) {
- continue;
- }
-
XLField xlField = new XLField();
xlField.name = field;
xlField.sf = sf;
diff --git a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java
index 5ebec77..aad4069 100644
--- a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java
@@ -28,6 +28,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
@@ -246,6 +247,7 @@ class CSVWriter extends TextResponseWriter {
}
Collection<String> fields = returnFields.getRequestedFieldNames();
+ Set<String> explicitReqFields = returnFields.getExplicitlyRequestedFieldNames();
Object responseObj = rsp.getResponse();
boolean returnStoredOrDocValStored = false;
if (fields==null||returnFields.hasPatternMatching()) {
@@ -266,6 +268,12 @@ class CSVWriter extends TextResponseWriter {
Iterables.addAll(fields, all);
}
}
+
+ if (explicitReqFields != null) {
+ // add explicit requested fields
+ Iterables.addAll(fields, explicitReqFields);
+ }
+
if (returnFields.wantsScore()) {
fields.add("score");
} else {
@@ -288,17 +296,19 @@ class CSVWriter extends TextResponseWriter {
}
SchemaField sf = schema.getFieldOrNull(field);
- if (sf == null) {
- FieldType ft = new StrField();
- sf = new SchemaField(field, ft);
- }
-
+
// Return stored fields or useDocValuesAsStored=true fields,
// unless an explicit field list is specified
- if (returnStoredOrDocValStored && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored())) {
+ if (returnStoredOrDocValStored && !(explicitReqFields != null && explicitReqFields.contains(field)) &&
+ sf!= null && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored())) {
continue;
}
+ if (sf == null) {
+ FieldType ft = new StrField();
+ sf = new SchemaField(field, ft);
+ }
+
// check for per-field overrides
sep = params.get("f." + field + '.' + CSV_SEPARATOR);
encapsulator = params.get("f." + field + '.' + CSV_ENCAPSULATOR);
diff --git a/solr/core/src/java/org/apache/solr/search/ReturnFields.java b/solr/core/src/java/org/apache/solr/search/ReturnFields.java
index c2d5973..0c35ea5 100644
--- a/solr/core/src/java/org/apache/solr/search/ReturnFields.java
+++ b/solr/core/src/java/org/apache/solr/search/ReturnFields.java
@@ -60,6 +60,13 @@ public abstract class ReturnFields {
public abstract Set<String> getRequestedFieldNames();
/**
+ * The explicitly requested field names (includes pseudo fields)
+ * <p>
+ * @return Set of explicitly requested field names or <code>null</code> (no explicit)
+ */
+ public abstract Set<String> getExplicitlyRequestedFieldNames();
+
+ /**
* Get the fields which have been renamed
* @return a mapping of renamed fields
*/
diff --git a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
index d9e681b..9a04c25 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
@@ -16,6 +16,15 @@
*/
package org.apache.solr.search;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+
import org.apache.commons.io.FilenameUtils;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.ValueSource;
@@ -35,15 +44,6 @@ import org.apache.solr.response.transform.TransformerFactory;
import org.apache.solr.response.transform.ValueSourceAugmenter;
import org.apache.solr.search.SolrDocumentFetcher.RetrieveFieldsOptimizer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Supplier;
-
/**
* The default implementation of return fields parsing for Solr.
*/
@@ -448,7 +448,15 @@ public class SolrReturnFields extends ReturnFields {
@Override
public Set<String> getRequestedFieldNames() {
- if(_wantsAllFields || reqFieldNames==null || reqFieldNames.isEmpty()) {
+ if(_wantsAllFields || reqFieldNames == null || reqFieldNames.isEmpty()) {
+ return null;
+ }
+ return reqFieldNames;
+ }
+
+ @Override
+ public Set<String> getExplicitlyRequestedFieldNames() {
+ if (reqFieldNames == null || reqFieldNames.isEmpty()) {
return null;
}
return reqFieldNames;
diff --git a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
index 979279c..204496a 100644
--- a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
+++ b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
@@ -42,7 +42,7 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
public static void createIndex() {
assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false", "foo_f","1.414","foo_d","-1.0E300","foo_dt","2000-01-02T03:04:05Z"));
assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output", "shouldbeunstored","foo"));
- assertU(adoc("id","3", "shouldbeunstored","foo"));
+ assertU(adoc("id","3", "shouldbeunstored","foo", "foo_l", "1"));
assertU(adoc("id","4", "amount_c", "1.50,EUR"));
assertU(adoc("id","5", "store", "12.434,-134.1"));
assertU(adoc("id","6", "pubyear_ii", "123", "store_iis", "12", "price_ff", "1.3"));
@@ -246,6 +246,13 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
assertEquals("exists(shouldbeunstored),XXX", singleFuncLines[0] );
assertEquals("false,1", singleFuncLines[1] );
assertEquals("true,3", singleFuncLines[3] );
+
+ // pseudo-fields with * in fl
+ txt = h.query(req("q","id:4", "wt","csv", "csv.header","true", "fl","*,YYY:[docid],FOO:amount_c"));
+ lines = txt.split("\n");
+ assertEquals(2, lines.length);
+ assertEquals(sortHeader("foo_i,foo_l,FOO,foo_s,store,store_iis," +
+ "v2_ss,pubyear_ii,foo_dt,foo_b,YYY,foo_d,id,amount_c,foo_f,v_ss"), sortHeader(lines[0]));
}
@Test
@@ -283,6 +290,19 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
assertEquals(2, singleFuncLines.length);
assertEquals("price_ff", singleFuncLines[0]);
assertEquals("1.3", singleFuncLines[1]);
+
+ // explicit price_ff with fl=*
+ singleFuncText = h.query(req("q","id:6", "wt","csv", "csv.header","true", "fl", "*,price_ff"));
+ sortedHeader = sortHeader("amount_c,store,v_ss,foo_b,v2_ss,foo_f,foo_i,foo_d,foo_s,foo_dt,id,foo_l," +
+ "pubyear_ii,store_iis,price_ff");
+ singleFuncLines = singleFuncText.split("\n");
+ assertEquals(2, singleFuncLines.length);
+ assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));
+ actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))
+ .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))
+ .collect(Collectors.toList());
+ assertEquals(4, actualVal.size());
+ assertTrue(actualVal.containsAll(Arrays.asList("6", "123", "12", "1.3")));
}