You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2019/03/27 10:58:46 UTC
[lucene-solr] branch master updated: SOLR-7414: CSVResponseWriter &
XLSXResponseWriter return empty field when fl alias is combined with *
selector
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new e7939d5 SOLR-7414: CSVResponseWriter & XLSXResponseWriter return empty field when fl alias is combined with * selector
e7939d5 is described below
commit e7939d5907a4af3a83235c3980dbdd3147bac304
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Wed Mar 27 14:38:36 2019 +0530
SOLR-7414: CSVResponseWriter & XLSXResponseWriter return empty field when fl alias is combined with * selector
---
solr/CHANGES.txt | 3 +
.../handler/extraction/XLSXResponseWriter.java | 95 ++-----------
.../extraction/solr/collection1/conf/schema.xml | 9 ++
.../handler/extraction/TestXLSXResponseWriter.java | 93 +++++++++++--
.../apache/solr/response/CSVResponseWriter.java | 93 ++-----------
.../solr/response/TabularResponseWriter.java | 149 +++++++++++++++++++++
.../java/org/apache/solr/search/ReturnFields.java | 7 +
.../org/apache/solr/search/SolrReturnFields.java | 28 ++--
.../solr/response/TestCSVResponseWriter.java | 24 +++-
9 files changed, 309 insertions(+), 192 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 57f1a14..c6690cc 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -141,6 +141,9 @@ Bug Fixes
* SOLR-13253: IndexSchema.getResourceLoader was being used to load non-schema things, which can be a memory leak if
"shareSchema" and other circumstances occur. Furthermore it's reference to SolrConfig was removed. (David Smiley)
+* SOLR-7414: CSVResponseWriter & XLSXResponseWriter return empty field when fl alias is combined with '*' selector
+ (Michael Lawrence, Munendra S N, Ishan Chattopadhyaya)
+
Improvements
----------------------
diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
index baa1ddb..a673d30 100644
--- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
+++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java
@@ -27,12 +27,9 @@ import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
import org.apache.lucene.index.IndexableField;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.FillPatternType;
@@ -43,19 +40,13 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFCellStyle;
import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.BasicResultContext;
import org.apache.solr.response.RawResponseWriter;
-import org.apache.solr.response.ResultContext;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.response.TextResponseWriter;
+import org.apache.solr.response.TabularResponseWriter;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.StrField;
-import org.apache.solr.search.DocList;
import org.apache.solr.search.ReturnFields;
public class XLSXResponseWriter extends RawResponseWriter {
@@ -94,10 +85,7 @@ public class XLSXResponseWriter extends RawResponseWriter {
}
}
-class XLSXWriter extends TextResponseWriter {
-
- SolrQueryRequest req;
- SolrQueryResponse rsp;
+class XLSXWriter extends TabularResponseWriter {
static class SerialWriteWorkbook {
SXSSFWorkbook swb;
@@ -172,45 +160,14 @@ class XLSXWriter extends TextResponseWriter {
private Map<String,XLField> xlFields = new LinkedHashMap<String,XLField>();
- public XLSXWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp){
+ public XLSXWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {
super(writer, req, rsp);
- this.req = req;
- this.rsp = rsp;
}
public void writeResponse(OutputStream out, LinkedHashMap<String, String> colNamesMap,
LinkedHashMap<String, Integer> colWidthsMap) throws IOException {
- SolrParams params = req.getParams();
-
- Collection<String> fields = returnFields.getRequestedFieldNames();
- Object responseObj = rsp.getValues().get("response");
- boolean returnOnlyStored = false;
- if (fields==null||returnFields.hasPatternMatching()) {
- if (responseObj instanceof SolrDocumentList) {
- // get the list of fields from the SolrDocumentList
- if(fields==null) {
- fields = new LinkedHashSet<String>();
- }
- for (SolrDocument sdoc: (SolrDocumentList)responseObj) {
- fields.addAll(sdoc.getFieldNames());
- }
- } else {
- // get the list of fields from the index
- Iterable<String> all = req.getSearcher().getFieldNames();
- if (fields == null) {
- fields = Sets.newHashSet(all);
- } else {
- Iterables.addAll(fields, all);
- }
- }
- if (returnFields.wantsScore()) {
- fields.add("score");
- } else {
- fields.remove("score");
- }
- returnOnlyStored = true;
- }
+ Collection<String> fields = getFields();
for (String field : fields) {
if (!returnFields.wantsField(field)) {
continue;
@@ -222,17 +179,15 @@ class XLSXWriter extends TextResponseWriter {
continue;
}
+ if (shouldSkipField(field)) {
+ continue;
+ }
SchemaField sf = schema.getFieldOrNull(field);
if (sf == null) {
FieldType ft = new StrField();
sf = new SchemaField(field, ft);
}
- // Return only stored fields, unless an explicit field list is specified
- if (returnOnlyStored && sf != null && !sf.stored()) {
- continue;
- }
-
XLField xlField = new XLField();
xlField.name = field;
xlField.sf = sf;
@@ -264,15 +219,7 @@ class XLSXWriter extends TextResponseWriter {
wb.setHeaderRow();
wb.addRow();
- if (responseObj instanceof ResultContext) {
- writeDocuments(null, (ResultContext)responseObj );
- }
- else if (responseObj instanceof DocList) {
- ResultContext ctx = new BasicResultContext((DocList)responseObj, returnFields, null, null, req);
- writeDocuments(null, ctx );
- } else if (responseObj instanceof SolrDocumentList) {
- writeSolrDocumentList(null, (SolrDocumentList)responseObj, returnFields );
- }
+ writeResponse(rsp.getResponse());
wb.flush(out);
wb = null;
@@ -283,23 +230,6 @@ class XLSXWriter extends TextResponseWriter {
super.close();
}
- @Override
- public void writeNamedList(String name, NamedList val) throws IOException {
- }
-
- @Override
- public void writeStartDocumentList(String name,
- long start, int size, long numFound, Float maxScore) throws IOException
- {
- // nothing
- }
-
- @Override
- public void writeEndDocumentList() throws IOException
- {
- // nothing
- }
-
//NOTE: a document cannot currently contain another document
List tmpList;
@Override
@@ -347,10 +277,6 @@ class XLSXWriter extends TextResponseWriter {
}
@Override
- public void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException {
- }
-
- @Override
public void writeArray(String name, Iterator val) throws IOException {
StringBuffer output = new StringBuffer();
while (val.hasNext()) {
@@ -404,11 +330,6 @@ class XLSXWriter extends TextResponseWriter {
}
@Override
- public void writeDate(String name, Date val) throws IOException {
- writeDate(name, val.toInstant().toString());
- }
-
- @Override
public void writeDate(String name, String val) throws IOException {
wb.writeCell(val);
}
diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml
index 475c333..9dea361 100644
--- a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml
+++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml
@@ -330,6 +330,12 @@
</analyzer>
</fieldType>
+ <!-- Point Fields -->
+ <fieldType name="pint" class="solr.IntPointField" docValues="true"/>
+ <fieldType name="plong" class="solr.LongPointField" docValues="true"/>
+ <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
+ <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
+ <fieldType name="pdate" class="solr.DatePointField" docValues="true"/>
<field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
<field name="name" type="nametext" indexed="true" stored="true"/>
@@ -462,6 +468,9 @@
<!-- ignored because not stored or indexed -->
<dynamicField name="ignored_*" type="text" indexed="false" stored="false"/>
+ <dynamicField name="*_ii" type="pint" indexed="false" stored="false" useDocValuesAsStored="true"/>
+ <dynamicField name="*_iis" type="pint" indexed="false" stored="false" useDocValuesAsStored="true"/>
+ <dynamicField name="*_ff" type="pfloat" indexed="false" stored="false" useDocValuesAsStored="false"/>
<uniqueKey>id</uniqueKey>
diff --git a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java
index 7d37844..78f2df3 100644
--- a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java
+++ b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java
@@ -18,23 +18,24 @@ package org.apache.solr.handler.extraction;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.IOException;
import java.time.Instant;
+import java.util.Arrays;
import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
-import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFSheet;
-
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.QueryResponseWriter;
-import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.response.RawResponseWriter;
+import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrReturnFields;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -64,6 +65,7 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {
assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output", "shouldbeunstored","foo"));
assertU(adoc("id","3", "shouldbeunstored","foo"));
assertU(adoc("id","4", "foo_s1","foo"));
+ assertU(adoc("id","5", "pubyear_ii", "123", "store_iis", "12", "price_ff", "1.3"));
assertU(commit());
}
@@ -73,7 +75,7 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {
}
@Test
- public void testStructuredDataViaBaseWriters() throws IOException, Exception {
+ public void testStructuredDataViaBaseWriters() throws Exception {
SolrQueryResponse rsp = new SolrQueryResponse();
// Don't send a ContentStream back, this will fall back to the configured base writer.
// But abuse the CONTENT key to ensure writer is also checking type
@@ -220,19 +222,85 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {
//assertions specific to multiple pseudofields functions like abs, div, exists, etc.. (SOLR-5423)
String funcText = getStringFromSheet(getWSResultForQuery(req("df", "text", "q","*", "wt","xlsx", "fl","XXX:id,YYY:exists(foo_s1)")));
String[] funcLines = funcText.split("\n");
- assertEquals(5, funcLines.length);
+ assertEquals(6, funcLines.length);
assertEquals("XXX,YYY", funcLines[0] );
assertEquals("1,false", funcLines[1] );
assertEquals("3,false", funcLines[3] );
+
+ //assertions specific to single function without alias (SOLR-5423)
+ String singleFuncText = getStringFromSheet(
+ getWSResultForQuery(req("df", "text", "q","*", "wt","xlsx", "fl","exists(foo_s1),XXX:id")));
+ String[] singleFuncLines = singleFuncText.split("\n");
+ assertEquals(6, singleFuncLines.length);
+ assertEquals("exists(foo_s1),XXX", singleFuncLines[0] );
+ assertEquals("false,1", singleFuncLines[1] );
+ assertEquals("false,3", singleFuncLines[3] );
+
+ // pseudo-fields with * in fl
+ txt = getStringFromSheet(
+ getWSResultForQuery(req("df", "text", "q","id:4", "wt","xlsx", "fl","*,YYY:[docid],FOO:foo_s1")));
+ lines = txt.split("\n");
+ assertEquals(2, lines.length);
+ assertEquals(sortHeader("foo_i,foo_l,FOO,foo_s,pubyear_ii,store_iis," +
+ "v2_ss,multiDefault,timestamp,foo_dt1,foo_b,YYY,foo_d,id,foo_f,v_ss,foo_s1,intDefault"), sortHeader(lines[0]));
+ }
+
+ @Test
+ public void testForDVEnabledFields() throws Exception {
+ // for dv enabled and useDocValueAsStored=true
+ // returns pubyear_ii, store_iis but not price_ff
+ String singleFuncText = getStringFromSheet(
+ getWSResultForQuery(req("df", "text", "q","id:5", "wt","xlsx")));
+ String sortedHeader = sortHeader("foo_i,foo_l,foo_s,pubyear_ii,store_iis," +
+ "v2_ss,multiDefault,timestamp,foo_dt1,foo_b,foo_d,id,foo_f,v_ss,foo_s1,intDefault");
+ String[] singleFuncLines = singleFuncText.split("\n");
+ assertEquals(2, singleFuncLines.length);
+ assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));
+ List<String> actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))
+ .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))
+ .collect(Collectors.toList());
+ assertTrue(actualVal.containsAll(Arrays.asList("5", "123", "12")));
+
+ // explicit fl=*
+ singleFuncText = getStringFromSheet(
+ getWSResultForQuery(req("df", "text", "q","id:5", "wt","xlsx", "fl", "*")));
+ singleFuncLines = singleFuncText.split("\n");
+ assertEquals(2, singleFuncLines.length);
+ assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));
+ actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))
+ .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))
+ .collect(Collectors.toList());
+ assertTrue(actualVal.containsAll(Arrays.asList("5", "123", "12")));
+
+ // explicit price_ff
+ singleFuncText = getStringFromSheet(
+ getWSResultForQuery(req("df", "text", "q","id:5", "wt","xlsx", "fl", "price_ff")));
+ singleFuncLines = singleFuncText.split("\n");
+ assertEquals(2, singleFuncLines.length);
+ assertEquals("price_ff", singleFuncLines[0]);
+ assertEquals("1.3", singleFuncLines[1]);
+
+ // explicit price_ff with fl=*
+ singleFuncText = getStringFromSheet(
+ getWSResultForQuery(req("df", "text", "q","id:5", "wt","xlsx", "csv.header","true", "fl", "*,price_ff")));
+ sortedHeader = sortHeader("foo_i,foo_l,foo_b,foo_s,pubyear_ii,store_iis," +
+ "v2_ss,multiDefault,timestamp,foo_dt1,id,foo_d,foo_f,v_ss,foo_s1,intDefault,price_ff");
+ singleFuncLines = singleFuncText.split("\n");
+ assertEquals(2, singleFuncLines.length);
+ assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));
+ actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))
+ .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))
+ .collect(Collectors.toList());
+ assertTrue(actualVal.containsAll(Arrays.asList("5", "123", "12", "1.3")));
}
// returns first worksheet as XLSXResponseWriter only returns one sheet
- private XSSFSheet getWSResultForQuery(SolrQueryRequest req) throws IOException, Exception {
+ private XSSFSheet getWSResultForQuery(SolrQueryRequest req) throws Exception {
SolrQueryResponse rsp = h.queryAndResponse("", req);
return getWSResultForQuery(req, rsp);
}
- private XSSFSheet getWSResultForQuery(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, Exception {
+ private XSSFSheet getWSResultForQuery(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
ByteArrayOutputStream xmlBout = new ByteArrayOutputStream();
writerXlsx.write(xmlBout, req, rsp);
XSSFWorkbook output = new XSSFWorkbook(new ByteArrayInputStream(xmlBout.toByteArray()));
@@ -254,4 +322,13 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {
}
return output.toString();
}
+
+ /*
+ * Utility method to sort a comma separated list of strings, for easier comparison regardless of platform
+ */
+ private String sortHeader(String input) {
+ String[] output = input.trim().split(",");
+ Arrays.sort(output);
+ return Arrays.toString(output);
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java
index 5ebec77..e35e3ed 100644
--- a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java
@@ -22,18 +22,12 @@ import java.io.Writer;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
-import java.util.Date;
-import java.util.Iterator;
import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
import org.apache.lucene.index.IndexableField;
import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.FastWriter;
@@ -44,11 +38,10 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.StrField;
-import org.apache.solr.search.DocList;
import org.apache.solr.search.ReturnFields;
/**
- *
+ * Response writer for csv data
*/
public class CSVResponseWriter implements QueryResponseWriter {
@@ -75,7 +68,7 @@ public class CSVResponseWriter implements QueryResponseWriter {
}
-class CSVWriter extends TextResponseWriter {
+class CSVWriter extends TabularResponseWriter {
static String SEPARATOR = "separator";
static String ENCAPSULATOR = "encapsulator";
static String ESCAPE = "escape";
@@ -245,35 +238,7 @@ class CSVWriter extends TextResponseWriter {
// encapsulator will already be disabled if it wasn't specified
}
- Collection<String> fields = returnFields.getRequestedFieldNames();
- Object responseObj = rsp.getResponse();
- boolean returnStoredOrDocValStored = false;
- if (fields==null||returnFields.hasPatternMatching()) {
- if (responseObj instanceof SolrDocumentList) {
- // get the list of fields from the SolrDocumentList
- if(fields==null) {
- fields = new LinkedHashSet<>();
- }
- for (SolrDocument sdoc: (SolrDocumentList)responseObj) {
- fields.addAll(sdoc.getFieldNames());
- }
- } else {
- // get the list of fields from the index
- Iterable<String> all = req.getSearcher().getFieldNames();
- if (fields == null) {
- fields = Sets.newHashSet(all);
- } else {
- Iterables.addAll(fields, all);
- }
- }
- if (returnFields.wantsScore()) {
- fields.add("score");
- } else {
- fields.remove("score");
- }
- returnStoredOrDocValStored = true;
- }
-
+ Collection<String> fields = getFields();
CSVSharedBufPrinter csvPrinterMV = new CSVSharedBufPrinter(mvWriter, mvStrategy);
for (String field : fields) {
@@ -287,17 +252,15 @@ class CSVWriter extends TextResponseWriter {
continue;
}
+ if (shouldSkipField(field)) {
+ continue;
+ }
+
SchemaField sf = schema.getFieldOrNull(field);
if (sf == null) {
FieldType ft = new StrField();
sf = new SchemaField(field, ft);
}
-
- // Return stored fields or useDocValuesAsStored=true fields,
- // unless an explicit field list is specified
- if (returnStoredOrDocValStored && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored())) {
- continue;
- }
// check for per-field overrides
sep = params.get("f." + field + '.' + CSV_SEPARATOR);
@@ -350,17 +313,7 @@ class CSVWriter extends TextResponseWriter {
printer.println();
}
- if (responseObj instanceof ResultContext) {
- writeDocuments(null, (ResultContext)responseObj );
- }
- else if (responseObj instanceof DocList) {
-
- ResultContext ctx = new BasicResultContext((DocList)responseObj, returnFields, null, null, req);
- writeDocuments(null, ctx );
- } else if (responseObj instanceof SolrDocumentList) {
- writeSolrDocumentList(null, (SolrDocumentList)responseObj, returnFields );
- }
-
+ writeResponse(rsp.getResponse());
}
@Override
@@ -369,23 +322,6 @@ class CSVWriter extends TextResponseWriter {
super.close();
}
- @Override
- public void writeNamedList(String name, NamedList val) throws IOException {
- }
-
- @Override
- public void writeStartDocumentList(String name,
- long start, int size, long numFound, Float maxScore) throws IOException
- {
- // nothing
- }
-
- @Override
- public void writeEndDocumentList() throws IOException
- {
- // nothing
- }
-
//NOTE: a document cannot currently contain another document
List tmpList;
@Override
@@ -457,14 +393,6 @@ class CSVWriter extends TextResponseWriter {
}
@Override
- public void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException {
- }
-
- @Override
- public void writeArray(String name, Iterator val) throws IOException {
- }
-
- @Override
public void writeNull(String name) throws IOException {
printer.print(NullValue);
}
@@ -495,11 +423,6 @@ class CSVWriter extends TextResponseWriter {
}
@Override
- public void writeDate(String name, Date val) throws IOException {
- writeDate(name, val.toInstant().toString());
- }
-
- @Override
public void writeDate(String name, String val) throws IOException {
printer.print(val, false);
}
diff --git a/solr/core/src/java/org/apache/solr/response/TabularResponseWriter.java b/solr/core/src/java/org/apache/solr/response/TabularResponseWriter.java
new file mode 100644
index 0000000..a6e9e94
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/response/TabularResponseWriter.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.response;
+
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.DocList;
+
+/**
+ * Base response writer for table-oriented data
+ */
+public abstract class TabularResponseWriter extends TextResponseWriter {
+
+ private boolean returnStoredOrDocValStored;
+
+ public TabularResponseWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse resp) {
+ super(writer, req, resp);
+ // fl=* or globs specified in fl
+ returnStoredOrDocValStored = ((returnFields.getRequestedFieldNames() == null) ||
+ returnFields.hasPatternMatching());
+ }
+
+ /**
+ * Returns fields to be returned in the response
+ */
+ public Collection<String> getFields() {
+ Collection<String> fields = returnFields.getRequestedFieldNames();
+ Set<String> explicitReqFields = returnFields.getExplicitlyRequestedFieldNames();
+ Object responseObj = rsp.getResponse();
+ if (fields==null||returnFields.hasPatternMatching()) {
+ if (responseObj instanceof SolrDocumentList) {
+ // get the list of fields from the SolrDocumentList
+ if(fields==null) {
+ fields = new LinkedHashSet<>();
+ }
+ for (SolrDocument sdoc: (SolrDocumentList)responseObj) {
+ fields.addAll(sdoc.getFieldNames());
+ }
+ } else {
+ // get the list of fields from the index
+ Iterable<String> all = req.getSearcher().getFieldNames();
+ if (fields == null) {
+ fields = Sets.newHashSet(all);
+ } else {
+ Iterables.addAll(fields, all);
+ }
+ }
+
+ if (explicitReqFields != null) {
+ // add explicit requested fields
+ Iterables.addAll(fields, explicitReqFields);
+ }
+
+ if (returnFields.wantsScore()) {
+ fields.add("score");
+ } else {
+ fields.remove("score");
+ }
+ }
+ return fields;
+ }
+
+ /**
+ * Returns true if field needs to be skipped else false
+ * @param field name of the field
+ * @return boolean value
+ */
+ public boolean shouldSkipField(String field) {
+ Set<String> explicitReqFields = returnFields.getExplicitlyRequestedFieldNames();
+ SchemaField sf = schema.getFieldOrNull(field);
+
+ // Return stored fields or useDocValuesAsStored=true fields,
+ // unless an explicit field list is specified
+ return (returnStoredOrDocValStored && !(explicitReqFields != null && explicitReqFields.contains(field)) &&
+ sf!= null && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored()));
+ }
+
+ public void writeResponse(Object responseObj) throws IOException {
+ if (responseObj instanceof ResultContext) {
+ writeDocuments(null, (ResultContext)responseObj );
+ }
+ else if (responseObj instanceof DocList) {
+ ResultContext ctx = new BasicResultContext((DocList)responseObj, returnFields, null, null, req);
+ writeDocuments(null, ctx );
+ } else if (responseObj instanceof SolrDocumentList) {
+ writeSolrDocumentList(null, (SolrDocumentList)responseObj, returnFields );
+ }
+ }
+
+ @Override
+ public void writeNamedList(String name, NamedList val) throws IOException {
+ }
+
+ @Override
+ public void writeStartDocumentList(String name,
+ long start, int size, long numFound, Float maxScore) throws IOException
+ {
+ // nothing
+ }
+
+ @Override
+ public void writeEndDocumentList() throws IOException
+ {
+ // nothing
+ }
+
+ @Override
+ public void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException {
+ }
+
+ @Override
+ public void writeArray(String name, Iterator val) throws IOException {
+ }
+
+ @Override
+ public void writeDate(String name, Date val) throws IOException {
+ writeDate(name, val.toInstant().toString());
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/ReturnFields.java b/solr/core/src/java/org/apache/solr/search/ReturnFields.java
index c2d5973..dcfbcd6 100644
--- a/solr/core/src/java/org/apache/solr/search/ReturnFields.java
+++ b/solr/core/src/java/org/apache/solr/search/ReturnFields.java
@@ -60,6 +60,13 @@ public abstract class ReturnFields {
public abstract Set<String> getRequestedFieldNames();
/**
+ * The explicitly requested field names (includes pseudo fields)
+ * <p>
+ * @return Set of explicitly requested field names or <code>null</code> (no explict)
+ */
+ public abstract Set<String> getExplicitlyRequestedFieldNames();
+
+ /**
* Get the fields which have been renamed
* @return a mapping of renamed fields
*/
diff --git a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
index d9e681b..9a04c25 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java
@@ -16,6 +16,15 @@
*/
package org.apache.solr.search;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+
import org.apache.commons.io.FilenameUtils;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.ValueSource;
@@ -35,15 +44,6 @@ import org.apache.solr.response.transform.TransformerFactory;
import org.apache.solr.response.transform.ValueSourceAugmenter;
import org.apache.solr.search.SolrDocumentFetcher.RetrieveFieldsOptimizer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Supplier;
-
/**
* The default implementation of return fields parsing for Solr.
*/
@@ -448,7 +448,15 @@ public class SolrReturnFields extends ReturnFields {
@Override
public Set<String> getRequestedFieldNames() {
- if(_wantsAllFields || reqFieldNames==null || reqFieldNames.isEmpty()) {
+ if(_wantsAllFields || reqFieldNames == null || reqFieldNames.isEmpty()) {
+ return null;
+ }
+ return reqFieldNames;
+ }
+
+ @Override
+ public Set<String> getExplicitlyRequestedFieldNames() {
+ if (reqFieldNames == null || reqFieldNames.isEmpty()) {
return null;
}
return reqFieldNames;
diff --git a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
index 979279c..b26c8de 100644
--- a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
+++ b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
@@ -42,7 +42,7 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
public static void createIndex() {
assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false", "foo_f","1.414","foo_d","-1.0E300","foo_dt","2000-01-02T03:04:05Z"));
assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output", "shouldbeunstored","foo"));
- assertU(adoc("id","3", "shouldbeunstored","foo"));
+ assertU(adoc("id","3", "shouldbeunstored","foo", "foo_l", "1"));
assertU(adoc("id","4", "amount_c", "1.50,EUR"));
assertU(adoc("id","5", "store", "12.434,-134.1"));
assertU(adoc("id","6", "pubyear_ii", "123", "store_iis", "12", "price_ff", "1.3"));
@@ -246,12 +246,19 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
assertEquals("exists(shouldbeunstored),XXX", singleFuncLines[0] );
assertEquals("false,1", singleFuncLines[1] );
assertEquals("true,3", singleFuncLines[3] );
+
+ // pseudo-fields with * in fl
+ txt = h.query(req("q","id:4", "wt","csv", "csv.header","true", "fl","*,YYY:[docid],FOO:amount_c"));
+ lines = txt.split("\n");
+ assertEquals(2, lines.length);
+ assertEquals(sortHeader("foo_i,foo_l,FOO,foo_s,store,store_iis," +
+ "v2_ss,pubyear_ii,foo_dt,foo_b,YYY,foo_d,id,amount_c,foo_f,v_ss"), sortHeader(lines[0]));
}
@Test
public void testForDVEnabledFields() throws Exception {
// for dv enabled and useDocValueAsStored=true
- // returns pubyear_i, store_iis but not price_ff
+ // returns pubyear_ii, store_iis but not price_ff
String singleFuncText = h.query(req("q","id:6", "wt","csv", "csv.header","true"));
String sortedHeader = sortHeader("amount_c,store,v_ss,foo_b,v2_ss,foo_f,foo_i,foo_d,foo_s,foo_dt,id,foo_l," +
"pubyear_ii,store_iis");
@@ -283,6 +290,19 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
assertEquals(2, singleFuncLines.length);
assertEquals("price_ff", singleFuncLines[0]);
assertEquals("1.3", singleFuncLines[1]);
+
+ // explicit price_ff with fl=*
+ singleFuncText = h.query(req("q","id:6", "wt","csv", "csv.header","true", "fl", "*,price_ff"));
+ sortedHeader = sortHeader("amount_c,store,v_ss,foo_b,v2_ss,foo_f,foo_i,foo_d,foo_s,foo_dt,id,foo_l," +
+ "pubyear_ii,store_iis,price_ff");
+ singleFuncLines = singleFuncText.split("\n");
+ assertEquals(2, singleFuncLines.length);
+ assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));
+ actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))
+ .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))
+ .collect(Collectors.toList());
+ assertEquals(4, actualVal.size());
+ assertTrue(actualVal.containsAll(Arrays.asList("6", "123", "12", "1.3")));
}