[solr] branch main updated: SOLR-15570: Include declared but unused fields from the schema in table metadata (#241)

[th...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

thelabdude pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git


The following commit(s) were added to refs/heads/main by this push:
     new 6056b68  SOLR-15570: Include declared but unused fields from the schema in table metadata (#241)
6056b68 is described below

commit 6056b6814de1a93cfbf14858bfa5e6026e86d16d
Author: Timothy Potter <th...@gmail.com>
AuthorDate: Fri Jul 30 08:47:41 2021 -0600

    SOLR-15570: Include declared but unused fields from the schema in table metadata (#241)
---
 .../org/apache/solr/handler/sql/SolrSchema.java    | 29 +++++++++++++---------
 .../org/apache/solr/handler/TestSQLHandler.java    | 16 ++++++++++++
 .../solr-ref-guide/src/parallel-sql-interface.adoc |  2 +-
 3 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java b/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java
index 413bbf7..d527018 100644
--- a/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java
+++ b/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java
@@ -23,6 +23,8 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import com.google.common.collect.ImmutableMap;
 import org.apache.calcite.rel.type.RelDataType;
@@ -111,33 +113,40 @@ class SolrSchema extends AbstractSchema implements Closeable {
     }
   }
 
-  private Map<String, LukeResponse.FieldTypeInfo> getFieldTypeInfo(final String collection) {
+  private LukeResponse getSchema(final String collection) {
     final String zk = this.properties.getProperty("zk");
     PKIAuthenticationPlugin.withServerIdentity(true);
     try {
       LukeRequest lukeRequest = new LukeRequest();
-      lukeRequest.setShowSchema(true); // for custom type info ...
+      lukeRequest.setShowSchema(true); // for empty fields and custom type info ...
       lukeRequest.setNumTerms(0);
-      return lukeRequest.process(solrClientCache.getCloudSolrClient(zk), collection).getFieldTypeInfo();
+      return lukeRequest.process(solrClientCache.getCloudSolrClient(zk), collection);
     } catch (SolrServerException | IOException e) {
       throw new RuntimeException(e);
     } finally {
       PKIAuthenticationPlugin.withServerIdentity(false);
     }
   }
-
+  
   RelProtoDataType getRelDataType(String collection) {
     // Temporary type factory, just for the duration of this method. Allowable
     // because we're creating a proto-type, not a type; before being used, the
     // proto-type will be copied into a real type factory.
     final RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
     final RelDataTypeFactory.Builder fieldInfo = typeFactory.builder();
-    Map<String, LukeResponse.FieldInfo> luceneFieldInfoMap = getFieldInfo(collection);
+    
+    // Get fields that have data, including dynamic field instances
+    Map<String, LukeResponse.FieldInfo> fieldsInUseMap = getFieldInfo(collection);
+
+    LukeResponse schema = getSchema(collection);
+    // merge the actual fields in use returned by Luke with the declared fields in the schema that are empty
+    Map<String, LukeResponse.FieldInfo> combinedFields = Stream.of(fieldsInUseMap, schema.getFieldInfo())
+            .flatMap(map -> map.entrySet().stream())
+            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (v1, v2) -> v1));
 
-    Map<String, LukeResponse.FieldTypeInfo> fieldTypeInfoMap = null; // loaded lazily if needed
     Map<String, Class<?>> javaClassForTypeMap = new HashMap<>(); // local cache for custom field types we've already resolved
 
-    for (Map.Entry<String, LukeResponse.FieldInfo> entry : luceneFieldInfoMap.entrySet()) {
+    for (Map.Entry<String, LukeResponse.FieldInfo> entry : combinedFields.entrySet()) {
       LukeResponse.FieldInfo luceneFieldInfo = entry.getValue();
 
       String luceneFieldType = luceneFieldInfo.getType();
@@ -173,11 +182,7 @@ class SolrSchema extends AbstractSchema implements Closeable {
         default:
           Class<?> javaClass = javaClassForTypeMap.get(luceneFieldType);
           if (javaClass == null) {
-            if (fieldTypeInfoMap == null) {
-              // lazily go to luke for the field type info ...
-              fieldTypeInfoMap = getFieldTypeInfo(collection);
-            }
-            javaClass = guessJavaClassForFieldType(fieldTypeInfoMap.get(luceneFieldType));
+            javaClass = guessJavaClassForFieldType(schema.getFieldTypeInfo().get(luceneFieldType));
             javaClassForTypeMap.put(luceneFieldType, javaClass);
           }
           type = typeFactory.createJavaType(javaClass);
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
index c3420fd..f7209a1 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
@@ -2232,4 +2232,20 @@ public class TestSQLHandler extends SolrCloudTestCase {
     // select * w/o limit is not supported by Solr SQL
     expectThrows(IOException.class, () -> expectResults("SELECT * FROM $ALIAS", -1));
   }
+
+  @Test
+  public void testSelectEmptyField() throws Exception {
+    new UpdateRequest()
+        .add("id", "01")
+        .add("id", "02")
+        .add("id", "03")
+        .add("id", "04")
+        .add("id", "05")
+        .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+    // stringx is declared in the schema but has no docs
+    expectResults("SELECT id, stringx FROM $ALIAS", 5);
+    // notafield_i matches a dynamic field pattern but has no docs, so don't allow this
+    expectThrows(IOException.class, () -> expectResults("SELECT id, stringx, notafield_i FROM $ALIAS", 5));
+  }
 }
diff --git a/solr/solr-ref-guide/src/parallel-sql-interface.adoc b/solr/solr-ref-guide/src/parallel-sql-interface.adoc
index 7837b95..68caf06 100644
--- a/solr/solr-ref-guide/src/parallel-sql-interface.adoc
+++ b/solr/solr-ref-guide/src/parallel-sql-interface.adoc
@@ -40,7 +40,7 @@ Column names in the SQL query map directly to fields in the Solr index for the c
 These identifiers are case sensitive.
 Aliases are supported, and can be referenced in the `ORDER BY` clause.
 
-The `*` syntax to indicate all fields is not supported in either limited or unlimited queries.
+The `SELECT *` syntax to indicate all fields is only supported for queries with a `LIMIT` clause.
 The `score` field can be used only with queries that contain a `LIMIT` clause.
 
 For example, we could index Solr's sample documents and then construct an SQL query like this: