You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2024/03/29 16:58:09 UTC
(couchdb) branch nouveau-numeric-queries created (now 177ee3a8d)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a change to branch nouveau-numeric-queries
in repository https://gitbox.apache.org/repos/asf/couchdb.git
at 177ee3a8d use PointsConfigMap so numeric query and numeric range query work cleanly
This branch includes the following new commits:
new 177ee3a8d use PointsConfigMap so numeric query and numeric range query work cleanly
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(couchdb) 01/01: use PointsConfigMap so numeric query and numeric range query work cleanly
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch nouveau-numeric-queries
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 177ee3a8da03cf7f68d0e84c93ecb8e06162dbed
Author: Robert Newson <rn...@apache.org>
AuthorDate: Fri Mar 29 16:57:53 2024 +0000
use PointsConfigMap so numeric query and numeric range query work cleanly
---
.../couchdb/nouveau/lucene9/Lucene9Index.java | 55 +++++-
.../nouveau/lucene9/NouveauQueryParser.java | 184 ---------------------
.../nouveau/lucene9/NouveauQueryParserTest.java | 49 ++----
3 files changed, 67 insertions(+), 221 deletions(-)
diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java
index 587b1fa4f..c44a12999 100644
--- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java
+++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java
@@ -23,6 +23,7 @@ import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.file.NoSuchFileException;
+import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -65,11 +66,14 @@ import org.apache.lucene.facet.LabelAndValue;
import org.apache.lucene.facet.StringDocValuesReaderState;
import org.apache.lucene.facet.StringValueFacetCounts;
import org.apache.lucene.facet.range.DoubleRangeFacetCounts;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
+import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser;
+import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CollectorManager;
@@ -98,17 +102,20 @@ public class Lucene9Index extends Index {
private final Analyzer analyzer;
private final IndexWriter writer;
private final SearcherManager searcherManager;
+ private volatile Map<String, PointsConfig> pointsConfigMap;
public Lucene9Index(
final Analyzer analyzer,
final IndexWriter writer,
final long updateSeq,
final long purgeSeq,
- final SearcherManager searcherManager) {
+ final SearcherManager searcherManager)
+ throws IOException {
super(updateSeq, purgeSeq);
this.analyzer = Objects.requireNonNull(analyzer);
this.writer = Objects.requireNonNull(writer);
this.searcherManager = Objects.requireNonNull(searcherManager);
+ initPointsConfigMap();
}
@Override
@@ -134,6 +141,7 @@ public class Lucene9Index extends Index {
public void doUpdate(final String docId, final DocumentUpdateRequest request) throws IOException {
final Term docIdTerm = docIdTerm(docId);
final Document doc = toDocument(docId, request);
+ replacePointsConfigMap(doc);
writer.updateDocument(docIdTerm, doc);
}
@@ -489,7 +497,9 @@ public class Lucene9Index extends Index {
}
private Query parse(final SearchRequest request) {
- var queryParser = new NouveauQueryParser(analyzer, request.getLocale());
+ var queryParser = new StandardQueryParser(analyzer);
+ queryParser.setPointsConfigMap(pointsConfigMap);
+
Query result;
try {
result = queryParser.parse(request.getQuery(), "default");
@@ -505,6 +515,47 @@ public class Lucene9Index extends Index {
return result;
}
+ private void initPointsConfigMap() throws IOException {
+ var pointsConfigMap = new HashMap<String, PointsConfig>();
+ var dir = writer.getDirectory();
+ if (!DirectoryReader.indexExists(dir)) {
+ this.pointsConfigMap = pointsConfigMap;
+ return;
+ }
+ var doublePointsConfig = new PointsConfig(NumberFormat.getInstance(), Double.class);
+ try (var reader = DirectoryReader.open(dir)) {
+ for (var l : reader.leaves()) {
+ for (var fi : l.reader().getFieldInfos()) {
+ if (fi.getPointDimensionCount() != 0) {
+ pointsConfigMap.put(fi.name, doublePointsConfig);
+ }
+ }
+ }
+ }
+ this.pointsConfigMap = pointsConfigMap;
+ }
+
+ // Replace PointsConfigMap if we have just seen at least one new double field.
+ private void replacePointsConfigMap(final Document doc) throws IOException {
+ Map<String, PointsConfig> pointsConfigMap = null;
+ PointsConfig doublePointsConfig = null;
+
+ for (var field : doc) {
+ var isDoubleField = field instanceof org.apache.lucene.document.DoubleField;
+ var isNew = !this.pointsConfigMap.containsKey(field.name());
+ if (isDoubleField && isNew) {
+ if (pointsConfigMap == null) {
+ pointsConfigMap = new HashMap<String, PointsConfig>(this.pointsConfigMap);
+ doublePointsConfig = new PointsConfig(NumberFormat.getInstance(), Double.class);
+ }
+ pointsConfigMap.put(field.name(), doublePointsConfig);
+ }
+ }
+ if (pointsConfigMap != null) {
+ this.pointsConfigMap = pointsConfigMap;
+ }
+ }
+
@Override
public String toString() {
return "Lucene9Index [analyzer=" + analyzer + ", writer=" + writer + ", searcherManager=" + searcherManager
diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java
deleted file mode 100644
index f1cf60f0f..000000000
--- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java
+++ /dev/null
@@ -1,184 +0,0 @@
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package org.apache.couchdb.nouveau.lucene9;
-
-import java.text.NumberFormat;
-import java.text.ParseException;
-import java.util.List;
-import java.util.Locale;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
-import org.apache.lucene.queryparser.flexible.core.QueryParserHelper;
-import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode;
-import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
-import org.apache.lucene.queryparser.flexible.core.nodes.RangeQueryNode;
-import org.apache.lucene.queryparser.flexible.core.processors.NoChildOptimizationQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessorImpl;
-import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessorPipeline;
-import org.apache.lucene.queryparser.flexible.core.processors.RemoveDeletedQueryNodesProcessor;
-import org.apache.lucene.queryparser.flexible.standard.builders.StandardQueryTreeBuilder;
-import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
-import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler;
-import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler.ConfigurationKeys;
-import org.apache.lucene.queryparser.flexible.standard.nodes.PointQueryNode;
-import org.apache.lucene.queryparser.flexible.standard.nodes.PointRangeQueryNode;
-import org.apache.lucene.queryparser.flexible.standard.nodes.TermRangeQueryNode;
-import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser;
-import org.apache.lucene.queryparser.flexible.standard.processors.AllowLeadingWildcardProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.AnalyzerQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.BooleanQuery2ModifierNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.BooleanSingleChildOptimizationQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.BoostQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.DefaultPhraseSlopQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.FuzzyQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.IntervalQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.MatchAllDocsQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.MultiFieldQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.MultiTermRewriteMethodProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.OpenRangeQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.PhraseSlopQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.RegexpQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.RemoveEmptyNonLeafQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.TermRangeQueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.standard.processors.WildcardQueryNodeProcessor;
-import org.apache.lucene.search.Query;
-
-public final class NouveauQueryParser extends QueryParserHelper {
-
- public NouveauQueryParser(final Analyzer analyzer, final Locale locale) {
- super(
- new StandardQueryConfigHandler(),
- new StandardSyntaxParser(),
- new NouveauQueryNodeProcessorPipeline(locale),
- new StandardQueryTreeBuilder());
- getQueryConfigHandler().set(ConfigurationKeys.ENABLE_POSITION_INCREMENTS, true);
- getQueryConfigHandler().set(ConfigurationKeys.ANALYZER, analyzer);
- }
-
- @Override
- public Query parse(String query, String defaultField) throws QueryNodeException {
- return (Query) super.parse(query, defaultField);
- }
-
- /**
- * Same pipeline as StandardQueryParser but we substitute
- * PointQueryNodeProcessor and PointRangeQueryNodeProcessor for
- * NouveauPointProcessor below.
- */
- public static class NouveauQueryNodeProcessorPipeline extends QueryNodeProcessorPipeline {
-
- public NouveauQueryNodeProcessorPipeline(final Locale locale) {
- super(null);
- add(new WildcardQueryNodeProcessor());
- add(new MultiFieldQueryNodeProcessor());
- add(new FuzzyQueryNodeProcessor());
- add(new RegexpQueryNodeProcessor());
- add(new MatchAllDocsQueryNodeProcessor());
- add(new OpenRangeQueryNodeProcessor());
- add(new NouveauPointProcessor(locale));
- add(new TermRangeQueryNodeProcessor());
- add(new AllowLeadingWildcardProcessor());
- add(new AnalyzerQueryNodeProcessor());
- add(new PhraseSlopQueryNodeProcessor());
- add(new BooleanQuery2ModifierNodeProcessor());
- add(new NoChildOptimizationQueryNodeProcessor());
- add(new RemoveDeletedQueryNodesProcessor());
- add(new RemoveEmptyNonLeafQueryNodeProcessor());
- add(new BooleanSingleChildOptimizationQueryNodeProcessor());
- add(new DefaultPhraseSlopQueryNodeProcessor());
- add(new BoostQueryNodeProcessor());
- add(new MultiTermRewriteMethodProcessor());
- add(new IntervalQueryNodeProcessor());
- }
- }
-
- /**
- * If it looks like a number, treat it as a number.
- */
- public static class NouveauPointProcessor extends QueryNodeProcessorImpl {
-
- private final Locale locale;
-
- NouveauPointProcessor(final Locale locale) {
- this.locale = locale != null ? locale : Locale.getDefault();
- }
-
- @Override
- protected QueryNode postProcessNode(final QueryNode node) throws QueryNodeException {
- final var numberFormat = NumberFormat.getInstance(locale);
- final var pointsConfig = new PointsConfig(numberFormat, Double.class);
-
- if (node instanceof FieldQueryNode && !(node.getParent() instanceof RangeQueryNode)) {
- final var fieldNode = (FieldQueryNode) node;
- String text = fieldNode.getTextAsString();
- if (text.isEmpty()) {
- return node;
- }
- final Number number;
- try {
- number = numberFormat.parse(text).doubleValue();
- } catch (final ParseException e) {
- return node;
- }
- final var lowerNode = new PointQueryNode(fieldNode.getField(), number, numberFormat);
- final var upperNode = new PointQueryNode(fieldNode.getField(), number, numberFormat);
- return new PointRangeQueryNode(lowerNode, upperNode, true, true, pointsConfig);
- }
-
- if (node instanceof TermRangeQueryNode) {
- final var termRangeNode = (TermRangeQueryNode) node;
- final var lower = termRangeNode.getLowerBound();
- final var upper = termRangeNode.getUpperBound();
- final var lowerText = lower.getTextAsString();
- final var upperText = upper.getTextAsString();
- Number lowerNumber = null, upperNumber = null;
-
- if (lowerText.length() > 0 && !lowerText.equals("-Infinity")) {
- try {
- lowerNumber = numberFormat.parse(lowerText).doubleValue();
- } catch (final ParseException e) {
- return node;
- }
- }
-
- if (upperText.length() > 0 && !upperText.equals("Infinity")) {
- try {
- upperNumber = numberFormat.parse(upperText).doubleValue();
- } catch (final ParseException e) {
- return node;
- }
- }
-
- final var lowerNode = new PointQueryNode(termRangeNode.getField(), lowerNumber, numberFormat);
- final var upperNode = new PointQueryNode(termRangeNode.getField(), upperNumber, numberFormat);
- final var lowerInclusive = termRangeNode.isLowerInclusive();
- final var upperInclusive = termRangeNode.isUpperInclusive();
-
- return new PointRangeQueryNode(lowerNode, upperNode, lowerInclusive, upperInclusive, pointsConfig);
- }
-
- return node;
- }
-
- @Override
- protected QueryNode preProcessNode(final QueryNode node) throws QueryNodeException {
- return node;
- }
-
- @Override
- protected List<QueryNode> setChildrenOrder(final List<QueryNode> children) throws QueryNodeException {
- return children;
- }
- }
-}
diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java
index 6dbcceeb8..667b5363a 100644
--- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java
+++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java
@@ -15,10 +15,13 @@ package org.apache.couchdb.nouveau.lucene9;
import static org.assertj.core.api.Assertions.assertThat;
-import java.util.Locale;
+import java.text.NumberFormat;
+import java.util.Map;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser;
+import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
@@ -31,11 +34,12 @@ public class NouveauQueryParserTest {
private static final String DEFAULT_FIELD = "foo";
- private static NouveauQueryParser qp;
+ private static StandardQueryParser qp;
@BeforeAll
public static void setup() {
- qp = new NouveauQueryParser(new StandardAnalyzer(), Locale.US);
+ qp = new StandardQueryParser(new StandardAnalyzer());
+ qp.setPointsConfigMap(Map.of("num", new PointsConfig(NumberFormat.getInstance(), Double.class)));
}
@Test
@@ -67,50 +71,25 @@ public class NouveauQueryParserTest {
@Test
public void testInferredPointQuery() throws Exception {
- assertThat(qp.parse("foo:12", DEFAULT_FIELD)).isEqualTo(DoublePoint.newExactQuery("foo", 12.0));
+ assertThat(qp.parse("num:12", DEFAULT_FIELD)).isEqualTo(DoublePoint.newExactQuery("num", 12.0));
}
@Test
public void testInferredPointRangeQuery() throws Exception {
- assertThat(qp.parse("foo:[1 TO 12]", DEFAULT_FIELD))
- .isEqualTo(DoublePoint.newRangeQuery("foo", new double[] {1}, new double[] {12}));
+ assertThat(qp.parse("num:[1 TO 12]", DEFAULT_FIELD))
+ .isEqualTo(DoublePoint.newRangeQuery("num", new double[] {1}, new double[] {12}));
}
@Test
public void testOpenLeftPointRangeQuery() throws Exception {
- assertThat(qp.parse("foo:[* TO 100.0]", DEFAULT_FIELD))
+ assertThat(qp.parse("num:[* TO 100.0]", DEFAULT_FIELD))
.isEqualTo(
- DoublePoint.newRangeQuery("foo", new double[] {Double.NEGATIVE_INFINITY}, new double[] {100}));
+ DoublePoint.newRangeQuery("num", new double[] {Double.NEGATIVE_INFINITY}, new double[] {100}));
}
@Test
public void testOpenRightPointRangeQuery() throws Exception {
- assertThat(qp.parse("foo:[1.0 TO *]", DEFAULT_FIELD))
- .isEqualTo(DoublePoint.newRangeQuery("foo", new double[] {1}, new double[] {Double.POSITIVE_INFINITY}));
- }
-
- @Test
- public void testOpenLeftPointRangeQueryLegacy() throws Exception {
- assertThat(qp.parse("foo:[-Infinity TO 100.0]", DEFAULT_FIELD))
- .isEqualTo(
- DoublePoint.newRangeQuery("foo", new double[] {Double.NEGATIVE_INFINITY}, new double[] {100}));
- }
-
- @Test
- public void testOpenRightPointRangeQueryLegacy() throws Exception {
- assertThat(qp.parse("foo:[1.0 TO Infinity]", DEFAULT_FIELD))
- .isEqualTo(DoublePoint.newRangeQuery("foo", new double[] {1}, new double[] {Double.POSITIVE_INFINITY}));
- }
-
- @Test
- public void testLocales() throws Exception {
- var us = new NouveauQueryParser(new StandardAnalyzer(), Locale.US);
- var de = new NouveauQueryParser(new StandardAnalyzer(), Locale.GERMAN);
-
- assertThat(us.parse("foo:[10.0 TO 20.0]", DEFAULT_FIELD))
- .isEqualTo(DoublePoint.newRangeQuery("foo", new double[] {10}, new double[] {20}));
-
- assertThat(de.parse("foo:[10.0 TO 20.0]", DEFAULT_FIELD))
- .isEqualTo(DoublePoint.newRangeQuery("foo", new double[] {100}, new double[] {200}));
+ assertThat(qp.parse("num:[1.0 TO *]", DEFAULT_FIELD))
+ .isEqualTo(DoublePoint.newRangeQuery("num", new double[] {1}, new double[] {Double.POSITIVE_INFINITY}));
}
}